From 882d9ca8a4fbc0bc22d670e6d33ce758fd1eaf4b Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 15:46:13 +0800 Subject: [PATCH 01/12] refactor(ecstore): isolate metadata quorum decisions (#7165) * refactor(ecstore): isolate metadata quorum decisions * test(ecstore): match sealed context fixture map type * test(ecstore): count decommission faults across retry restarts --- .../src/set_disk/core/io_primitives.rs | 390 +----------------- .../src/set_disk/core/metadata_quorum.rs | 385 +++++++++++++++++ crates/ecstore/src/set_disk/core/mod.rs | 1 + 3 files changed, 407 insertions(+), 369 deletions(-) create mode 100644 crates/ecstore/src/set_disk/core/metadata_quorum.rs diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index a7c52549d..18b04c781 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -59,27 +59,34 @@ use super::super::{ send_heal_request_with_admission, should_prevent_write, to_object_err, try_read_inline_data_shards_direct, warn, }; #[cfg(test)] +pub(in crate::set_disk) use super::metadata_quorum::MetadataEarlyStopDecision; +pub(in crate::set_disk) use super::metadata_quorum::{ + MetadataQuorumAccumulator, is_metadata_fanout_ignored_error, metadata_early_stop_candidate_matches, +}; +#[cfg(test)] use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; #[cfg(test)] use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH; #[cfg(test)] use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD; use crate::diagnostics::get::{ - GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY, - GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, - GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, - GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, - GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, + GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, + GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, + GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_DIRECT_MEMORY, + GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_READER_SETUP_DROP_PENDING, + GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT, + GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, get_stage_timer_if_enabled, + record_get_stage_duration_if_enabled, +}; +#[cfg(test)] +use crate::diagnostics::get::{ GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR, - GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, - GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, - GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, - GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR, - GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID, - GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_INTERNAL_META, - GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, - GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, - GET_STAGE_READER_TASK_READER_CONSTRUCTION, get_stage_timer_if_enabled, record_get_stage_duration_if_enabled, + GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, }; #[cfg(test)] use crate::disk::CHECK_PART_FILE_NOT_FOUND; @@ -690,324 +697,6 @@ impl MetadataFanoutDiagnostics { } } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(in crate::set_disk) struct MetadataEarlyStopDecision { - pub(in crate::set_disk) reason: &'static str, -} - -#[derive(Clone, Debug)] -pub(in crate::set_disk) struct MetadataQuorumAccumulator { - pub(in crate::set_disk) total_disks: usize, - pub(in crate::set_disk) default_parity_count: usize, - pub(in crate::set_disk) allow_early_stop: bool, - pub(in crate::set_disk) valid_responses: usize, - pub(in crate::set_disk) not_found_responses: usize, - pub(in crate::set_disk) version_not_found_responses: usize, - pub(in crate::set_disk) ignored_errors: usize, - pub(in crate::set_disk) hard_errors: usize, - pub(in crate::set_disk) candidate: Option, - pub(in crate::set_disk) candidate_votes: usize, - // Bitset of shard indexes whose metadata matches the candidate. Erasure - // layouts are capped at 16 shards, so this stays allocation-free on the - // GET metadata hot path. - candidate_shard_mask: u16, - pub(in crate::set_disk) conflicting_metadata: bool, - pub(in crate::set_disk) delete_marker_seen: bool, - pub(in crate::set_disk) delete_marker_candidates: Vec<(FileInfo, usize)>, - pub(in crate::set_disk) delete_marker_votes: usize, - pub(in crate::set_disk) requested_version_id: String, - pub(in crate::set_disk) matching_version_votes: usize, -} - -impl MetadataQuorumAccumulator { - pub(in crate::set_disk) fn new(total_disks: usize, default_parity_count: usize, allow_early_stop: bool) -> Self { - Self { - total_disks, - default_parity_count, - allow_early_stop, - valid_responses: 0, - not_found_responses: 0, - version_not_found_responses: 0, - ignored_errors: 0, - hard_errors: 0, - candidate: None, - candidate_votes: 0, - candidate_shard_mask: 0, - conflicting_metadata: false, - delete_marker_seen: false, - delete_marker_candidates: Vec::new(), - delete_marker_votes: 0, - requested_version_id: String::new(), - matching_version_votes: 0, - } - } - - pub(in crate::set_disk) fn with_requested_version_id(mut self, version_id: &str) -> Self { - self.requested_version_id = version_id.to_string(); - self - } - - pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) { - self.observe_file_info_with_index(None, file_info); - } - - pub(in crate::set_disk) fn observe_file_info_at(&mut self, disk_index: usize, file_info: &FileInfo) { - self.observe_file_info_with_index(Some(disk_index), file_info); - } - - fn observe_file_info_with_index(&mut self, disk_index: Option, file_info: &FileInfo) { - if !file_info_is_valid_for_metadata(file_info) { - self.hard_errors = self.hard_errors.saturating_add(1); - return; - } - - self.valid_responses = self.valid_responses.saturating_add(1); - - // Track version match for versioned requests - if !self.requested_version_id.is_empty() - && let Some(ref vid) = file_info.version_id - && vid.to_string() == self.requested_version_id - { - self.matching_version_votes = self.matching_version_votes.saturating_add(1); - } - - if file_info.is_canonical_delete_marker() { - self.delete_marker_seen = true; - if let Some((_, votes)) = self - .delete_marker_candidates - .iter_mut() - .find(|(candidate, _)| metadata_early_stop_candidate_matches(candidate, file_info)) - { - *votes = votes.saturating_add(1); - } else { - self.delete_marker_candidates.push((file_info.clone(), 1)); - } - self.delete_marker_votes = self - .delete_marker_candidates - .iter() - .map(|(_, votes)| *votes) - .max() - .unwrap_or_default(); - self.conflicting_metadata |= self.delete_marker_candidates.len() > 1; - return; - } - - match &self.candidate { - Some(candidate) if metadata_early_stop_candidate_matches(candidate, file_info) => { - self.candidate_votes = self.candidate_votes.saturating_add(1); - if let Some(disk_index) = disk_index - && let Some(bit) = Self::candidate_shard_bit(candidate, file_info, disk_index) - { - self.candidate_shard_mask |= bit; - } - } - Some(_) => { - self.conflicting_metadata = true; - } - None => { - self.candidate = Some(file_info.clone()); - self.candidate_votes = 1; - if let Some(disk_index) = disk_index - && let Some(bit) = Self::candidate_shard_bit(file_info, file_info, disk_index) - { - self.candidate_shard_mask |= bit; - } - } - } - } - - fn candidate_shard_bit(candidate: &FileInfo, file_info: &FileInfo, disk_index: usize) -> Option { - let &erasure_index = candidate.erasure.distribution.get(disk_index)?; - if erasure_index == 0 || erasure_index > u16::BITS as usize || file_info.erasure.index != erasure_index { - return None; - } - Some(1u16 << (erasure_index - 1)) - } - - pub(in crate::set_disk) fn candidate_has_read_reserve(&self) -> bool { - self.candidate_read_reserve_target() - .is_some_and(|required| self.candidate_shard_mask.count_ones() as usize >= required) - } - - pub(in crate::set_disk) fn candidate_read_reserve_target(&self) -> Option { - let candidate = self.candidate.as_ref()?; - Some( - candidate - .erasure - .data_blocks - .saturating_add(usize::from(candidate.erasure.parity_blocks > 0)), - ) - } - - pub(in crate::set_disk) fn observe_error(&mut self, err: &DiskError) { - match err { - DiskError::FileNotFound | DiskError::VolumeNotFound => { - self.not_found_responses = self.not_found_responses.saturating_add(1); - } - DiskError::FileVersionNotFound => { - self.version_not_found_responses = self.version_not_found_responses.saturating_add(1); - } - _ if is_metadata_fanout_ignored_error(err) => { - self.ignored_errors = self.ignored_errors.saturating_add(1); - } - _ => { - self.hard_errors = self.hard_errors.saturating_add(1); - } - } - } - - pub(in crate::set_disk) fn early_stop_decision(&self) -> Option { - if !self.allow_early_stop { - return None; - } - if self.delete_marker_votes >= self.default_write_quorum() { - return Some(MetadataEarlyStopDecision { - reason: GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, - }); - } - if self.conflicting_metadata - || self.delete_marker_seen - || self.not_found_responses > 0 - || self.version_not_found_responses > 0 - || self.hard_errors > 0 - { - return None; - } - if self - .candidate - .as_ref() - .and_then(|candidate| self.candidate_latest_quorum(candidate)) - .is_some_and(|latest_quorum| self.candidate_votes >= latest_quorum) - { - return Some(MetadataEarlyStopDecision { - reason: GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, - }); - } - None - } - - /// Check if a versioned request can early-stop because the requested - /// version_id has reached quorum across disks. - pub(in crate::set_disk) fn version_early_stop_decision(&self) -> Option { - if !self.allow_early_stop { - return None; - } - if self.requested_version_id.is_empty() { - return None; - } - if self.conflicting_metadata - || self.delete_marker_seen - || self.not_found_responses > 0 - || self.version_not_found_responses > 0 - || self.hard_errors > 0 - { - return None; - } - if self.matching_version_votes >= self.read_quorum_for_version() { - return Some(MetadataEarlyStopDecision { - reason: GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, - }); - } - None - } - - pub(in crate::set_disk) fn can_still_reach_early_stop_with_pending(&self, pending: usize) -> bool { - if !self.allow_early_stop { - return false; - } - if self.delete_marker_votes.saturating_add(pending) >= self.default_write_quorum() { - return true; - } - if self.conflicting_metadata - || self.delete_marker_seen - || self.not_found_responses > 0 - || self.version_not_found_responses > 0 - || self.hard_errors > 0 - { - return false; - } - if !self.requested_version_id.is_empty() - && self.matching_version_votes.saturating_add(pending) >= self.read_quorum_for_version() - { - return true; - } - match &self.candidate { - Some(candidate) => self - .candidate_latest_quorum(candidate) - .is_some_and(|latest_quorum| self.candidate_votes.saturating_add(pending) >= latest_quorum), - None => pending >= self.default_write_quorum(), - } - } - - /// Compute the read quorum threshold for version-aware early-stop. - /// Uses `total_disks / 2` (like `missing_response_quorum`) when - /// `default_parity_count` is set, otherwise requires all disks. - pub(in crate::set_disk) fn read_quorum_for_version(&self) -> usize { - self.missing_response_quorum() - } - - pub(in crate::set_disk) fn final_miss_reason(&self) -> &'static str { - if !self.allow_early_stop { - return GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST; - } - if self.conflicting_metadata { - return GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA; - } - if self.delete_marker_seen { - return GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER; - } - let missing_response_quorum = self.missing_response_quorum(); - if self.version_not_found_responses >= missing_response_quorum { - return GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND; - } - if self.not_found_responses >= missing_response_quorum { - return GET_METADATA_EARLY_STOP_REASON_NOT_FOUND; - } - if self.hard_errors > 0 { - return GET_METADATA_EARLY_STOP_REASON_ERROR; - } - if self.ignored_errors > 0 { - return GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM; - } - GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM - } - - pub(in crate::set_disk) fn candidate_latest_quorum(&self, candidate: &FileInfo) -> Option { - if self.default_parity_count == 0 { - return Some(self.total_disks); - } - if candidate.is_canonical_delete_marker() || candidate.size == 0 || candidate.erasure.parity_blocks >= self.total_disks { - return None; - } - let data_blocks = candidate.erasure.data_blocks; - Some(if data_blocks == candidate.erasure.parity_blocks { - data_blocks.saturating_add(1) - } else { - data_blocks - }) - } - - pub(crate) fn default_write_quorum(&self) -> usize { - if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks { - return self.total_disks; - } - let data_blocks = self.total_disks.saturating_sub(self.default_parity_count); - if data_blocks == self.default_parity_count { - data_blocks.saturating_add(1) - } else { - data_blocks - } - } - - pub(in crate::set_disk) fn missing_response_quorum(&self) -> usize { - if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks { - self.total_disks - } else { - self.total_disks / 2 - } - } -} - #[derive(Clone, Debug)] pub(in crate::set_disk) enum MetadataCacheLookup { Hit(Arc), @@ -1015,39 +704,6 @@ pub(in crate::set_disk) enum MetadataCacheLookup { RejectedInsufficientQuorum, } -pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo, right: &FileInfo) -> bool { - left.volume == right.volume - && left.name == right.name - && left.version_id == right.version_id - && left.is_latest == right.is_latest - && left.deleted == right.deleted - && left.mark_deleted == right.mark_deleted - && left.transition_status == right.transition_status - && left.transitioned_objname == right.transitioned_objname - && left.transition_tier == right.transition_tier - && left.transition_version_id == right.transition_version_id - && left.transition_version == right.transition_version - && left.transition_version_state == right.transition_version_state - && left.expire_restored == right.expire_restored - && left.size == right.size - && left.mod_time == right.mod_time - && left.mode == right.mode - && left.written_by_version == right.written_by_version - && left.metadata == right.metadata - && left.replication_state_internal == right.replication_state_internal - && left.parts == right.parts - && left.checksum == right.checksum - && left.versioned == right.versioned - && left.num_versions == right.num_versions - && left.successor_mod_time == right.successor_mod_time - && left.data_dir == right.data_dir - && left.erasure.algorithm == right.erasure.algorithm - && left.erasure.data_blocks == right.erasure.data_blocks - && left.erasure.parity_blocks == right.erasure.parity_blocks - && left.erasure.block_size == right.erasure.block_size - && left.erasure.distribution == right.erasure.distribution -} - pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason( bucket: &str, object: &str, @@ -1247,10 +903,6 @@ pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) -> } } -pub(in crate::set_disk) fn is_metadata_fanout_ignored_error(err: &DiskError) -> bool { - OBJECT_OP_IGNORED_ERRS.iter().any(|ignored| ignored == err) -} - pub(in crate::set_disk) fn is_confirmed_missing_part_error(err: Option<&str>) -> bool { let Some(err) = err else { return false; diff --git a/crates/ecstore/src/set_disk/core/metadata_quorum.rs b/crates/ecstore/src/set_disk/core/metadata_quorum.rs new file mode 100644 index 000000000..2922b6fc6 --- /dev/null +++ b/crates/ecstore/src/set_disk/core/metadata_quorum.rs @@ -0,0 +1,385 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Pure metadata quorum and early-stop decisions for `SetDisks` reads. +//! +//! Disk scheduling, coalescing, cancellation, and late shard materialization +//! remain with their existing owners; this module only classifies observations. + +use crate::diagnostics::get::{ + GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, + GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, + GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, + GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, + GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, +}; +use crate::disk::error::DiskError; +use crate::disk::error_reduce::OBJECT_OP_IGNORED_ERRS; +use crate::set_disk::file_info_is_valid_for_metadata; +use rustfs_filemeta::FileInfo; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(in crate::set_disk) struct MetadataEarlyStopDecision { + pub(in crate::set_disk) reason: &'static str, +} + +#[derive(Clone, Debug)] +pub(in crate::set_disk) struct MetadataQuorumAccumulator { + pub(in crate::set_disk) total_disks: usize, + pub(in crate::set_disk) default_parity_count: usize, + pub(in crate::set_disk) allow_early_stop: bool, + pub(in crate::set_disk) valid_responses: usize, + pub(in crate::set_disk) not_found_responses: usize, + pub(in crate::set_disk) version_not_found_responses: usize, + pub(in crate::set_disk) ignored_errors: usize, + pub(in crate::set_disk) hard_errors: usize, + pub(in crate::set_disk) candidate: Option, + pub(in crate::set_disk) candidate_votes: usize, + // Bitset of shard indexes whose metadata matches the candidate. Erasure + // layouts are capped at 16 shards, so this stays allocation-free on the + // GET metadata hot path. + candidate_shard_mask: u16, + pub(in crate::set_disk) conflicting_metadata: bool, + pub(in crate::set_disk) delete_marker_seen: bool, + pub(in crate::set_disk) delete_marker_candidates: Vec<(FileInfo, usize)>, + pub(in crate::set_disk) delete_marker_votes: usize, + pub(in crate::set_disk) requested_version_id: String, + pub(in crate::set_disk) matching_version_votes: usize, +} + +impl MetadataQuorumAccumulator { + pub(in crate::set_disk) fn new(total_disks: usize, default_parity_count: usize, allow_early_stop: bool) -> Self { + Self { + total_disks, + default_parity_count, + allow_early_stop, + valid_responses: 0, + not_found_responses: 0, + version_not_found_responses: 0, + ignored_errors: 0, + hard_errors: 0, + candidate: None, + candidate_votes: 0, + candidate_shard_mask: 0, + conflicting_metadata: false, + delete_marker_seen: false, + delete_marker_candidates: Vec::new(), + delete_marker_votes: 0, + requested_version_id: String::new(), + matching_version_votes: 0, + } + } + + pub(in crate::set_disk) fn with_requested_version_id(mut self, version_id: &str) -> Self { + self.requested_version_id = version_id.to_string(); + self + } + + pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) { + self.observe_file_info_with_index(None, file_info); + } + + pub(in crate::set_disk) fn observe_file_info_at(&mut self, disk_index: usize, file_info: &FileInfo) { + self.observe_file_info_with_index(Some(disk_index), file_info); + } + + fn observe_file_info_with_index(&mut self, disk_index: Option, file_info: &FileInfo) { + if !file_info_is_valid_for_metadata(file_info) { + self.hard_errors = self.hard_errors.saturating_add(1); + return; + } + + self.valid_responses = self.valid_responses.saturating_add(1); + + // Track version match for versioned requests + if !self.requested_version_id.is_empty() + && let Some(ref vid) = file_info.version_id + && vid.to_string() == self.requested_version_id + { + self.matching_version_votes = self.matching_version_votes.saturating_add(1); + } + + if file_info.is_canonical_delete_marker() { + self.delete_marker_seen = true; + if let Some((_, votes)) = self + .delete_marker_candidates + .iter_mut() + .find(|(candidate, _)| metadata_early_stop_candidate_matches(candidate, file_info)) + { + *votes = votes.saturating_add(1); + } else { + self.delete_marker_candidates.push((file_info.clone(), 1)); + } + self.delete_marker_votes = self + .delete_marker_candidates + .iter() + .map(|(_, votes)| *votes) + .max() + .unwrap_or_default(); + self.conflicting_metadata |= self.delete_marker_candidates.len() > 1; + return; + } + + match &self.candidate { + Some(candidate) if metadata_early_stop_candidate_matches(candidate, file_info) => { + self.candidate_votes = self.candidate_votes.saturating_add(1); + if let Some(disk_index) = disk_index + && let Some(bit) = Self::candidate_shard_bit(candidate, file_info, disk_index) + { + self.candidate_shard_mask |= bit; + } + } + Some(_) => { + self.conflicting_metadata = true; + } + None => { + self.candidate = Some(file_info.clone()); + self.candidate_votes = 1; + if let Some(disk_index) = disk_index + && let Some(bit) = Self::candidate_shard_bit(file_info, file_info, disk_index) + { + self.candidate_shard_mask |= bit; + } + } + } + } + + fn candidate_shard_bit(candidate: &FileInfo, file_info: &FileInfo, disk_index: usize) -> Option { + let &erasure_index = candidate.erasure.distribution.get(disk_index)?; + if erasure_index == 0 || erasure_index > u16::BITS as usize || file_info.erasure.index != erasure_index { + return None; + } + Some(1u16 << (erasure_index - 1)) + } + + pub(in crate::set_disk) fn candidate_has_read_reserve(&self) -> bool { + self.candidate_read_reserve_target() + .is_some_and(|required| self.candidate_shard_mask.count_ones() as usize >= required) + } + + pub(in crate::set_disk) fn candidate_read_reserve_target(&self) -> Option { + let candidate = self.candidate.as_ref()?; + Some( + candidate + .erasure + .data_blocks + .saturating_add(usize::from(candidate.erasure.parity_blocks > 0)), + ) + } + + pub(in crate::set_disk) fn observe_error(&mut self, err: &DiskError) { + match err { + DiskError::FileNotFound | DiskError::VolumeNotFound => { + self.not_found_responses = self.not_found_responses.saturating_add(1); + } + DiskError::FileVersionNotFound => { + self.version_not_found_responses = self.version_not_found_responses.saturating_add(1); + } + _ if is_metadata_fanout_ignored_error(err) => { + self.ignored_errors = self.ignored_errors.saturating_add(1); + } + _ => { + self.hard_errors = self.hard_errors.saturating_add(1); + } + } + } + + pub(in crate::set_disk) fn early_stop_decision(&self) -> Option { + if !self.allow_early_stop { + return None; + } + if self.delete_marker_votes >= self.default_write_quorum() { + return Some(MetadataEarlyStopDecision { + reason: GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, + }); + } + if self.conflicting_metadata + || self.delete_marker_seen + || self.not_found_responses > 0 + || self.version_not_found_responses > 0 + || self.hard_errors > 0 + { + return None; + } + if self + .candidate + .as_ref() + .and_then(|candidate| self.candidate_latest_quorum(candidate)) + .is_some_and(|latest_quorum| self.candidate_votes >= latest_quorum) + { + return Some(MetadataEarlyStopDecision { + reason: GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, + }); + } + None + } + + /// Check if a versioned request can early-stop because the requested + /// version_id has reached quorum across disks. + pub(in crate::set_disk) fn version_early_stop_decision(&self) -> Option { + if !self.allow_early_stop { + return None; + } + if self.requested_version_id.is_empty() { + return None; + } + if self.conflicting_metadata + || self.delete_marker_seen + || self.not_found_responses > 0 + || self.version_not_found_responses > 0 + || self.hard_errors > 0 + { + return None; + } + if self.matching_version_votes >= self.read_quorum_for_version() { + return Some(MetadataEarlyStopDecision { + reason: GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, + }); + } + None + } + + pub(in crate::set_disk) fn can_still_reach_early_stop_with_pending(&self, pending: usize) -> bool { + if !self.allow_early_stop { + return false; + } + if self.delete_marker_votes.saturating_add(pending) >= self.default_write_quorum() { + return true; + } + if self.conflicting_metadata + || self.delete_marker_seen + || self.not_found_responses > 0 + || self.version_not_found_responses > 0 + || self.hard_errors > 0 + { + return false; + } + if !self.requested_version_id.is_empty() + && self.matching_version_votes.saturating_add(pending) >= self.read_quorum_for_version() + { + return true; + } + match &self.candidate { + Some(candidate) => self + .candidate_latest_quorum(candidate) + .is_some_and(|latest_quorum| self.candidate_votes.saturating_add(pending) >= latest_quorum), + None => pending >= self.default_write_quorum(), + } + } + + /// Compute the read quorum threshold for version-aware early-stop. + /// Uses `total_disks / 2` (like `missing_response_quorum`) when + /// `default_parity_count` is set, otherwise requires all disks. + pub(in crate::set_disk) fn read_quorum_for_version(&self) -> usize { + self.missing_response_quorum() + } + + pub(in crate::set_disk) fn final_miss_reason(&self) -> &'static str { + if !self.allow_early_stop { + return GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST; + } + if self.conflicting_metadata { + return GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA; + } + if self.delete_marker_seen { + return GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER; + } + let missing_response_quorum = self.missing_response_quorum(); + if self.version_not_found_responses >= missing_response_quorum { + return GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND; + } + if self.not_found_responses >= missing_response_quorum { + return GET_METADATA_EARLY_STOP_REASON_NOT_FOUND; + } + if self.hard_errors > 0 { + return GET_METADATA_EARLY_STOP_REASON_ERROR; + } + if self.ignored_errors > 0 { + return GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM; + } + GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM + } + + pub(in crate::set_disk) fn candidate_latest_quorum(&self, candidate: &FileInfo) -> Option { + if self.default_parity_count == 0 { + return Some(self.total_disks); + } + if candidate.is_canonical_delete_marker() || candidate.size == 0 || candidate.erasure.parity_blocks >= self.total_disks { + return None; + } + let data_blocks = candidate.erasure.data_blocks; + Some(if data_blocks == candidate.erasure.parity_blocks { + data_blocks.saturating_add(1) + } else { + data_blocks + }) + } + + pub(crate) fn default_write_quorum(&self) -> usize { + if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks { + return self.total_disks; + } + let data_blocks = self.total_disks.saturating_sub(self.default_parity_count); + if data_blocks == self.default_parity_count { + data_blocks.saturating_add(1) + } else { + data_blocks + } + } + + pub(in crate::set_disk) fn missing_response_quorum(&self) -> usize { + if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks { + self.total_disks + } else { + self.total_disks / 2 + } + } +} + +pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo, right: &FileInfo) -> bool { + left.volume == right.volume + && left.name == right.name + && left.version_id == right.version_id + && left.is_latest == right.is_latest + && left.deleted == right.deleted + && left.mark_deleted == right.mark_deleted + && left.transition_status == right.transition_status + && left.transitioned_objname == right.transitioned_objname + && left.transition_tier == right.transition_tier + && left.transition_version_id == right.transition_version_id + && left.transition_version == right.transition_version + && left.transition_version_state == right.transition_version_state + && left.expire_restored == right.expire_restored + && left.size == right.size + && left.mod_time == right.mod_time + && left.mode == right.mode + && left.written_by_version == right.written_by_version + && left.metadata == right.metadata + && left.replication_state_internal == right.replication_state_internal + && left.parts == right.parts + && left.checksum == right.checksum + && left.versioned == right.versioned + && left.num_versions == right.num_versions + && left.successor_mod_time == right.successor_mod_time + && left.data_dir == right.data_dir + && left.erasure.algorithm == right.erasure.algorithm + && left.erasure.data_blocks == right.erasure.data_blocks + && left.erasure.parity_blocks == right.erasure.parity_blocks + && left.erasure.block_size == right.erasure.block_size + && left.erasure.distribution == right.erasure.distribution +} + +pub(in crate::set_disk) fn is_metadata_fanout_ignored_error(err: &DiskError) -> bool { + OBJECT_OP_IGNORED_ERRS.iter().any(|ignored| ignored == err) +} diff --git a/crates/ecstore/src/set_disk/core/mod.rs b/crates/ecstore/src/set_disk/core/mod.rs index 903487e64..b8507603e 100644 --- a/crates/ecstore/src/set_disk/core/mod.rs +++ b/crates/ecstore/src/set_disk/core/mod.rs @@ -18,3 +18,4 @@ //! duplicating read/write/erasure logic. pub(crate) mod io_primitives; +mod metadata_quorum; From 15e9bc5ed0f8558ed6bcdce559eeb1419c479753 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 15:54:56 +0800 Subject: [PATCH 02/12] test(ecstore): count injected faults across retry restarts (#7170) test(ecstore): count decommission faults across retry restarts From bbd7b9ef17b5b9bce2dd2134fcfc18510bb8c5f8 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 5 Sep 2026 15:55:54 +0800 Subject: [PATCH 03/12] fix(site-replication): bound and order outage recovery (#7148) * fix(site-replication): wake retry drain after peer recovery * fix(site-replication): replay configure after bucket make * fix(site-replication): serialize retry replay state * fix(site-replication): persist destructive retry intents * fix(site-replication): bound retry recovery rounds * fix(site-replication): keep recovery replay live * fix(site-replication): preserve retry ordering * fix(site-replication): bound retry coordination * fix(site-replication): serialize topology replay * fix(site-replication): fence distributed retry state * fix(site-replication): bound outage retry drain * fix(site-replication): drop unsafe delete retry intents * fix(site-replication): order bucket mutation replay * fix(site-replication): harden outage retry replay * fix(site-replication): fence destructive peer delivery * fix(site-replication): avoid peer edit retry deadlock * fix(site-replication): fence retry error classification * fix(site-replication): classify connect timeouts * fix(site-replication): close recovery review races * test(site-replication): cover timeout endpoint text * fix(site-replication): close destructive recovery gaps * fix(site-replication): fence recovery revisions * fix(site-replication): replay bucket metadata on recovery * fix(site-replication): preserve s3gate boundary --------- Co-authored-by: overtrue --- .../src/replication_extension_test.rs | 93 ++ rustfs/src/admin/handlers/site_replication.rs | 662 +++++++++----- rustfs/src/app/bucket_usecase.rs | 100 ++- rustfs/src/site_replication/hooks.rs | 462 +++++++++- rustfs/src/site_replication/mod.rs | 13 +- rustfs/src/site_replication/repair.rs | 4 +- rustfs/src/site_replication/retry.rs | 849 +++++++++++++++--- rustfs/src/site_replication/state_lock.rs | 37 +- rustfs/src/site_replication/tests.rs | 609 ++++++++++++- rustfs/src/site_replication/transport.rs | 30 +- rustfs/src/site_replication_reconcile.rs | 48 +- rustfs/src/storage_api.rs | 4 +- 12 files changed, 2375 insertions(+), 536 deletions(-) diff --git a/crates/e2e_test/src/replication_extension_test.rs b/crates/e2e_test/src/replication_extension_test.rs index f5d308c7e..198fa5166 100644 --- a/crates/e2e_test/src/replication_extension_test.rs +++ b/crates/e2e_test/src/replication_extension_test.rs @@ -6743,6 +6743,99 @@ async fn test_site_replication_replicates_object_with_bucket_versioning_real_dua Ok(()) } +#[tokio::test] +async fn test_site_replication_replays_bucket_created_during_peer_outage_real_dual_node() -> TestResult { + init_logging(); + + // Keep compilation outside the scenario timeout. Recovery itself waits + // for the production 30-second lightweight retry tick. + let _rustfs_binary = rustfs_binary_path(); + + match timeout(Duration::from_secs(150), async { + let mut site_env = replication_fast_env(); + site_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV); + + let mut site_a_env = RustFSTestEnvironment::new().await?; + site_a_env.start_rustfs_server_with_env(vec![], &site_env).await?; + + let mut site_b_env = RustFSTestEnvironment::new().await?; + site_b_env.start_rustfs_server_without_cleanup_with_env(&site_env).await?; + + let site_a_client = site_a_env.create_s3_client(); + let site_b_client = site_b_env.create_s3_client(); + let bucket = "site-repl-peer-outage"; + let key = "after-recovery.txt"; + let payload = b"site replication recovered the missed bucket".to_vec(); + + let add_status = site_replication_add( + &site_a_env, + &[ + PeerSite { + name: "outage-site-a".to_string(), + endpoint: site_a_env.url.clone(), + access_key: site_a_env.access_key.clone(), + secret_key: site_a_env.secret_key.clone(), + ..Default::default() + }, + PeerSite { + name: "outage-site-b".to_string(), + endpoint: site_b_env.url.clone(), + access_key: site_b_env.access_key.clone(), + secret_key: site_b_env.secret_key.clone(), + ..Default::default() + }, + ], + ) + .await?; + assert!(add_status.success, "unexpected site add result: {add_status:?}"); + wait_for_site_replication_enabled(&site_a_env, 2).await?; + wait_for_site_replication_enabled(&site_b_env, 2).await?; + + site_b_env.stop_server(); + site_a_client.create_bucket().bucket(bucket).send().await?; + site_a_client.head_bucket().bucket(bucket).send().await?; + + let queued = site_replication_info(&site_a_env) + .await? + .retry_stats + .ok_or("peer outage did not persist a site replication retry event")?; + assert!(queued.pending + queued.failed > 0, "peer outage retry queue was unexpectedly empty"); + + site_b_env.restart_server_preserving_data(vec![], &site_env).await?; + let recovery_deadline = tokio::time::Instant::now() + Duration::from_secs(75); + loop { + let bucket_recovered = site_b_client.head_bucket().bucket(bucket).send().await.is_ok(); + let queue_empty = site_replication_info(&site_a_env).await?.retry_stats.is_none(); + if bucket_recovered && queue_empty { + break; + } + if tokio::time::Instant::now() >= recovery_deadline { + return Err(format!( + "site replication retry did not settle after peer recovery; bucket_recovered={bucket_recovered}, queue_empty={queue_empty}" + ) + .into()); + } + sleep(Duration::from_millis(250)).await; + } + + site_a_client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from(payload.clone())) + .send() + .await?; + assert_eq!(wait_for_object_on_target(&site_b_client, bucket, key).await?, payload); + + Ok(()) + }) + .await + { + Ok(result) => result, + Err(_) => Err("site replication peer-outage recovery timed out after 150 seconds".into()), + } +} + /// Re-applying a site's own replication config must not disable the peer's reverse direction. /// /// `PutBucketReplication` broadcasts the config to every peer — the console's replication diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index d75e3b850..51c7dda99 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -190,7 +190,8 @@ fn site_replicator_service_account_policy() -> S3Result { .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("parse site replicator policy failed: {e}"))) } -// Lock order: lifecycle -> bucket operation -> repair admission -> state -> per-bucket metadata. +// Lock order: lifecycle -> bucket-mutation admission -> per-bucket mutation +// -> bucket operation -> repair admission -> state -> per-bucket metadata. // "state" is the distributed state-object lock in // crate::site_replication::state_lock, entered through // update_site_replication_state (P1-15). There is no process-local state @@ -434,6 +435,7 @@ pub fn register_site_replication_route(r: &mut S3Router) -> std: // into this module: startup sits below this layer and must not depend upwards. The admin // router is built before startup reconciles, so the hook is always installed in time. crate::site_replication_reconcile::register_site_replication_reconciler(reconcile_site_replication_wiring); + crate::site_replication_reconcile::register_site_replication_retry_drainer(reconcile_site_replication_retry_drain); for (method, path, operation) in [ (Method::PUT, "/v3/site-replication/add", AdminOperation(&SiteReplicationAddHandler {})), @@ -1803,28 +1805,61 @@ async fn reconcile_site_replication_buckets() -> S3Result<()> { /// (`SiteReplicationEditHandler`), so a tick landing between them would rewrite the targets /// from the stale endpoint. The pending marker in the persisted state closes that window. /// Skipping costs nothing — the timer comes back. +async fn site_replication_reconcile_prerequisites_ready() -> bool { + if current_iam_handle().is_none() || current_object_store_handle().is_none() { + return false; + } + if let Err(err) = migrate_collapsed_retry_queue_paths().await { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "retry_queue_migration_failed", + error = ?err, + "admin site replication state" + ); + return false; + } + true +} + +fn reconcile_site_replication_retry_drain() -> std::pin::Pin + Send>> { + Box::pin(async { + let Some(lifecycle) = SiteReplicationLifecycleGuard::try_acquire() else { + return; + }; + if !site_replication_reconcile_prerequisites_ready().await { + return; + } + match load_site_replication_state().await { + Ok(state) => { + if state.pending_endpoint_refresh.is_some() || state.pending_rotation.is_some() || state.pending_remove.is_some() + { + return; + } + } + Err(_) => return, + } + // Admission above observes a lifecycle-stable state. The lightweight + // drain itself handles only idempotent bucket setup, reloads state + // under the distributed repair lock, and shares that lock with bucket + // deletion. Do not hold this process-local guard across peer I/O: an + // outage recovery must not make admin add/edit/remove time out. + drop(lifecycle); + drain_site_replication_retry_queue_lightweight().await; + }) +} + fn reconcile_site_replication_wiring() -> std::pin::Pin + Send>> { Box::pin(async { // The scheduler starts before IAM and the object store are guaranteed ready (IAM // bootstrap may still be recovering), so an early tick returns quietly instead of // logging a failure for every reconciler. - if current_iam_handle().is_none() || current_object_store_handle().is_none() { - return; - } - - let Some(_lifecycle) = SiteReplicationLifecycleGuard::try_acquire() else { + let Some(lifecycle) = SiteReplicationLifecycleGuard::try_acquire() else { return; }; - if let Err(err) = migrate_collapsed_retry_queue_paths().await { - warn!( - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - result = "retry_queue_migration_failed", - error = ?err, - "admin site replication state" - ); + if !site_replication_reconcile_prerequisites_ready().await { return; } @@ -1878,8 +1913,9 @@ fn reconcile_site_replication_wiring() -> std::pin::Pin bool { let (origin, generation) = fence; if origin != local_deployment_id && state.peers.contains_key(origin) { @@ -4791,105 +4827,135 @@ async fn backfill_existing_buckets_after_add( let resync_id = Uuid::new_v4().to_string(); for bucket in &buckets { - let name = &bucket.name; + let operation_name = bucket.name.clone(); + let lock_bucket = operation_name.clone(); + let operation_state = state.clone(); + let operation_local_peer = local_peer.clone(); + let operation_resync_id = resync_id.clone(); + let operation_bootstrap_token = bootstrap_token.map(str::to_owned); + let bucket_errors = with_site_replication_bucket_mutation_lock(store.clone(), &lock_bucket, move || async move { + let mut errors = SiteReplicationErrorSummary::default(); + let name = &operation_name; - if let Err(err) = ensure_site_replication_bucket_versioning(name).await { - warn!( - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - bucket = %name, - result = "backfill_versioning_setup_failed", - error = ?err, - "admin site replication state" - ); - errors.push(format!("{name}: versioning setup failed: {err}")); - continue; - } - match ensure_site_replication_bucket_setup(name).await { - Ok(true) => {} - Ok(false) => { - // Runtime targets unavailable: the setup silently no-ops, which would make the - // downstream make-bucket broadcast and resync fail. Record it and skip so the - // operator sees this bucket was not propagated instead of an unqualified success. + if let Err(err) = ensure_site_replication_bucket_versioning(name).await { warn!( event = EVENT_ADMIN_SITE_REPLICATION_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, bucket = %name, - result = "backfill_bucket_setup_skipped", - "admin site replication state" - ); - errors.push(format!("{name}: replication setup skipped (site replication runtime unavailable)")); - continue; - } - Err(err) => { - warn!( - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - bucket = %name, - result = "backfill_bucket_setup_failed", + result = "backfill_versioning_setup_failed", error = ?err, "admin site replication state" ); - errors.push(format!("{name}: bucket setup failed: {err}")); + errors.push(format!("{name}: versioning setup failed: {err}")); + return errors; } - } - // Broadcast the bucket to peers so they create it too (idempotent on the peer side). - // Read the real lock_enabled flag so peers recreate the bucket with the same object-lock - // setting — object lock cannot be added after bucket creation. - let lock_enabled = match metadata_sys::get(name).await { - Ok(bm) => bm.lock_enabled, - Err(err) => { - warn!( - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - bucket = %name, - result = "backfill_bucket_metadata_read_failed", - fallback = "lock_enabled=false", - error = ?err, - "admin site replication state" - ); - false + match ensure_site_replication_bucket_setup(name).await { + Ok(true) => {} + Ok(false) => { + // Runtime targets unavailable: the setup silently no-ops, which would make the + // downstream make-bucket broadcast and resync fail. Record it and skip so the + // operator sees this bucket was not propagated instead of an unqualified success. + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + bucket = %name, + result = "backfill_bucket_setup_skipped", + "admin site replication state" + ); + errors.push(format!("{name}: replication setup skipped (site replication runtime unavailable)")); + return errors; + } + Err(err) => { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + bucket = %name, + result = "backfill_bucket_setup_failed", + error = ?err, + "admin site replication state" + ); + errors.push(format!("{name}: bucket setup failed: {err}")); + } } - }; - if let Err(err) = broadcast_site_replication_make_bucket(name, lock_enabled, None, bootstrap_token).await { - warn!( - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - bucket = %name, - result = "backfill_make_bucket_broadcast_failed", - error = ?err, - "admin site replication state" - ); - errors.push(format!("{name}: make-bucket broadcast failed: {err}")); - } - // Kick a resync toward every remote peer so existing objects travel across. - for peer in state.peers.values() { - if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) { - continue; - } - let manifest = site_bucket_resync_manifest_entry(name, peer, OffsetDateTime::now_utc()).await; - let result = if manifest.target_arn.is_empty() { - manifest - } else { - start_site_bucket_resync(name, &manifest.target_arn, &resync_id).await + // Broadcast the bucket to peers so they create it too (idempotent on the peer side). + // Read the real lock_enabled flag so peers recreate the bucket with the same object-lock + // setting — object lock cannot be added after bucket creation. + let lock_enabled = match metadata_sys::get(name).await { + Ok(bm) => bm.lock_enabled, + Err(err) => { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + bucket = %name, + result = "backfill_bucket_metadata_read_failed", + fallback = "lock_enabled=false", + error = ?err, + "admin site replication state" + ); + false + } }; - if result.status == "failed" { + if let Err(err) = + broadcast_site_replication_make_bucket(name, lock_enabled, None, operation_bootstrap_token.as_deref()).await + { warn!( event = EVENT_ADMIN_SITE_REPLICATION_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, bucket = %name, - peer = %peer.endpoint, - result = "backfill_resync_kick_failed", - detail = %result.err_detail, + result = "backfill_make_bucket_broadcast_failed", + error = ?err, "admin site replication state" ); - errors.push(format!("{name} -> {}: resync kick failed: {}", peer.endpoint, result.err_detail)); + errors.push(format!("{name}: make-bucket broadcast failed: {err}")); + } + // Kick a resync toward every remote peer so existing objects travel across. + for peer in operation_state.peers.values() { + if peer.deployment_id == operation_local_peer.deployment_id + || same_identity_endpoint(&peer.endpoint, &operation_local_peer.endpoint) + { + continue; + } + let manifest = site_bucket_resync_manifest_entry(name, peer, OffsetDateTime::now_utc()).await; + let result = if manifest.target_arn.is_empty() { + manifest + } else { + start_site_bucket_resync(name, &manifest.target_arn, &operation_resync_id).await + }; + if result.status == "failed" { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + bucket = %name, + peer = %peer.endpoint, + result = "backfill_resync_kick_failed", + detail = %result.err_detail, + "admin site replication state" + ); + errors.push(format!("{name} -> {}: resync kick failed: {}", peer.endpoint, result.err_detail)); + } + } + errors + }) + .await; + match bucket_errors { + Ok(bucket_errors) => errors.extend(bucket_errors), + Err(err) => { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + bucket = %lock_bucket, + result = "backfill_bucket_mutation_lock_failed", + error = ?err, + "admin site replication state" + ); + errors.push(format!("{lock_bucket}: bucket mutation lock failed: {err}")); } } } @@ -6072,146 +6138,204 @@ fn parse_peer_join_response(body: &[u8], fallback_peer: PeerInfo) -> Result, present: &HashSet) -> S3Result<()> { + let mut missing = expected.difference(present).cloned().collect::>(); + if !missing.is_empty() { + missing.sort_unstable(); + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!( + "bucket `{}` disappeared while site replication was being added; peers may already be joined — re-run replicate add", + missing[0] + ), + )); + } + + let mut unexpected = present.difference(expected).cloned().collect::>(); + if !unexpected.is_empty() { + unexpected.sort_unstable(); + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!( + "bucket `{}` appeared while site replication was being added; peers may already be joined — re-run replicate add", + unexpected[0] + ), + )); + } + + Ok(()) +} + #[async_trait::async_trait] impl Operation for SiteReplicationAddHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationAddAction).await?; reject_site_replicator_on_public_admin(&cred)?; let replicate_ilm_expiry = sr_add_replicate_ilm_expiry(&req.uri); + let local_endpoint = site_replication_local_endpoint(&req.uri, &req.headers); let lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await?; - // Everything up to the commit below is preflight: peer probes, IAM - // work and the join fan-out all talk to the network, so none of it may - // run inside the state transaction. The snapshot read here is what the - // `updated_at` CAS in the commit validates. - let current_state = load_site_replication_state().await?; - if pending_endpoint_refresh(¤t_state).is_some() { - return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending")); - } - let local_peer = current_local_peer(&req, ¤t_state); let mut sites: Vec = read_site_replication_json(req, &cred.secret_key, true).await?; - // The web console's "Set Up Site Replication" omits the local deployment from the payload; - // inject it so the add preflight (which requires the local deployment) succeeds. No-op for `mc`. - ensure_local_site_present(&mut sites, &local_peer); - validate_add_sites(&sites, &local_peer)?; - let preflight_infos = add_preflight_infos(&sites, ¤t_state, &local_peer).await?; - validate_add_preflight_topology(&preflight_infos, &local_peer)?; - let expected_updated_at = current_state.updated_at; - require_add_peer_tls_capability(&sites, &local_peer).await?; - // Early exit on a state that moved under the preflight probes, BEFORE - // the IAM write and the join fan-out change anything remote. Advisory - // only — the binding check is the CAS inside the commit — but it fences - // the common race off the side-effect path and refreshes the merge - // base so the CAS window is only the join round trips. - let latest_state = load_site_replication_state().await?; - ensure_edit_precondition(&latest_state, expected_updated_at, None, "add preflight")?; - let current_state = latest_state; - let (service_account_access_key, service_account_secret_key) = - ensure_site_replicator_service_account(&cred.access_key, false).await?; - let bootstrap_buckets = preflight_infos - .iter() - .filter(|info| !same_identity_endpoint(&info.endpoint, &local_peer.endpoint)) - .flat_map(|info| info.buckets.keys().cloned()) - .collect(); - let add_in_progress_guard = SiteReplicationAddInProgressGuard::start(lifecycle_guard, bootstrap_buckets)?; - let mut state = merge_add_sites( - current_state, - local_peer.clone(), - sites.clone(), - service_account_access_key.clone(), - cred.access_key.clone(), - replicate_ilm_expiry, - ); - state.sync_state_initialized = true; - let join_req = SRPeerJoinEnvelope { - request: SRPeerJoinReq { - svc_acct_access_key: service_account_access_key, - svc_acct_secret_key: service_account_secret_key.clone(), - svc_acct_parent: String::new(), - peers: state.peers.clone(), - updated_at: state.updated_at, - }, - defer_sync_state_enable: true, - }; - let peer_join_path = - with_site_replication_bootstrap_token(SITE_REPLICATION_PEER_JOIN_PATH, &add_in_progress_guard.token.to_string()); + let admin_access_key = cred.access_key.clone(); + let admission_store = current_object_store_handle() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; + let list_store = admission_store.clone(); + let (state, edit_generation, local_peer, service_account_secret_key, mut initial_sync_errors, _add_guard) = + with_site_replication_bucket_mutation_admission_lock(admission_store, move || async move { + // The writer starts before the local bucket snapshot and stays + // held through every peer join and the topology commit. A + // delete followed by a same-name create therefore cannot hide + // behind an unchanged final name set. Peer bootstrap callbacks + // use their internal path and do not acquire this public- + // mutation admission lock. + let current_state = load_site_replication_state().await?; + if pending_endpoint_refresh(¤t_state).is_some() { + return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending")); + } + let local_peer = local_peer_at_endpoint(local_endpoint, ¤t_state); + // The web console's "Set Up Site Replication" omits the local deployment from the payload; + // inject it so the add preflight (which requires the local deployment) succeeds. No-op for `mc`. + ensure_local_site_present(&mut sites, &local_peer); + validate_add_sites(&sites, &local_peer)?; + let preflight_infos = add_preflight_infos(&sites, ¤t_state, &local_peer).await?; + validate_add_preflight_topology(&preflight_infos, &local_peer)?; + let expected_updated_at = current_state.updated_at; + require_add_peer_tls_capability(&sites, &local_peer).await?; + // Early exit on a state that moved under the preflight probes, BEFORE + // the IAM write and the join fan-out change anything remote. Advisory + // only — the binding check is the CAS inside the commit — but it fences + // the common race off the side-effect path and refreshes the merge + // base so the CAS window is only the join round trips. + let latest_state = load_site_replication_state().await?; + ensure_edit_precondition(&latest_state, expected_updated_at, None, "add preflight")?; + let current_state = latest_state; + let (service_account_access_key, service_account_secret_key) = + ensure_site_replicator_service_account(&admin_access_key, false).await?; + let expected_buckets: HashSet = + preflight_infos.iter().flat_map(|info| info.buckets.keys().cloned()).collect(); + let bootstrap_buckets: HashSet = preflight_infos + .iter() + .filter(|info| !same_identity_endpoint(&info.endpoint, &local_peer.endpoint)) + .flat_map(|info| info.buckets.keys().cloned()) + .collect(); + let add_in_progress_guard = + SiteReplicationAddInProgressGuard::start(lifecycle_guard, bootstrap_buckets.clone())?; + let mut state = merge_add_sites( + current_state, + local_peer.clone(), + sites.clone(), + service_account_access_key.clone(), + admin_access_key, + replicate_ilm_expiry, + ); + state.sync_state_initialized = true; + let join_req = SRPeerJoinEnvelope { + request: SRPeerJoinReq { + svc_acct_access_key: service_account_access_key, + svc_acct_secret_key: service_account_secret_key.clone(), + svc_acct_parent: String::new(), + peers: state.peers.clone(), + updated_at: state.updated_at, + }, + defer_sync_state_enable: true, + }; + let peer_join_path = with_site_replication_bootstrap_token( + SITE_REPLICATION_PEER_JOIN_PATH, + &add_in_progress_guard.token.to_string(), + ); - let mut joined_endpoints = HashSet::new(); - let mut initial_sync_errors = SiteReplicationErrorSummary::default(); - for (site, preflight) in sites.iter().zip(preflight_infos.iter()) { - if same_identity_endpoint(&site.endpoint, &local_peer.endpoint) - || !joined_endpoints.insert(site_identity_key(&site.endpoint)) - { - continue; - } + let mut joined_endpoints = HashSet::new(); + let mut initial_sync_errors = SiteReplicationErrorSummary::default(); + for (site, preflight) in sites.iter().zip(preflight_infos.iter()) { + if same_identity_endpoint(&site.endpoint, &local_peer.endpoint) + || !joined_endpoints.insert(site_identity_key(&site.endpoint)) + { + continue; + } - let mut peer_join_req = join_req.clone(); - peer_join_req.request.svc_acct_parent = site.access_key.clone(); - let connection = PeerConnection::try_from(site)?; - let body = PeerAdminRequest::put(&connection, &peer_join_path, &site.access_key) - .send(&site.secret_key, &peer_join_req) + let mut peer_join_req = join_req.clone(); + peer_join_req.request.svc_acct_parent = site.access_key.clone(); + let connection = PeerConnection::try_from(site)?; + let body = PeerAdminRequest::put(&connection, &peer_join_path, &site.access_key) + .send(&site.secret_key, &peer_join_req) + .await?; + + let mut fallback_peer = existing_peer_for_endpoint(&state, &site.endpoint) + .unwrap_or_else(|| normalize_peer_site(site.clone(), replicate_ilm_expiry)); + fallback_peer.deployment_id = preflight.deployment_id.clone(); + let join_response = parse_peer_join_response(&body, fallback_peer).map_err(|e| { + S3Error::with_message( + S3ErrorCode::InternalError, + format!("parse peer join response from {} failed: {e}", site.endpoint), + ) + })?; + if !join_response.initial_sync_error_message.is_empty() { + initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message)); + } + // An explicit no-op join. The peer answered 200 but wrote nothing — + // its persisted state is already newer than the snapshot it was + // sent — so the add is only PARTIALLY configured and saying + // "configured successfully" would be a lie (rustfs/rustfs#5963). + // `None` (a MinIO peer, or one older than the field) is not a + // no-op signal and is deliberately not reported. + if join_response.applied == Some(false) { + initial_sync_errors.push(format!( + "{}: peer did not apply the join (its site replication state is newer than the snapshot it was sent); \ + the site is not configured against this peer", + site.endpoint + )); + } + state = reconcile_peer_with_actual_identity(state, join_response.peer); + let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| { + S3Error::with_message( + S3ErrorCode::InternalError, + format!("peer join response from {} did not identify the requested site", site.endpoint), + ) + })?; + validate_proposed_peer(&reconciled_peer).map_err(|err| { + S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!("invalid peer join response from {}: {err}", site.endpoint), + ) + })?; + } + + mark_unknown_peer_sync_enabled(&mut state.peers); + + // Commit. The state transaction's CAS still fences topology + // writers that do not use bucket admission. By this point + // remote sites may already have accepted their joins, so a + // mismatch asks the operator to re-run add and reconverge. + let next_state = state; + let present = list_store + .list_bucket(&BucketOptions::default()) + .await + .map_err(ApiError::from)? + .into_iter() + .map(|bucket| bucket.name) + .collect::>(); + ensure_add_bucket_set_matches_preflight(&expected_buckets, &present)?; + let (state, edit_generation) = update_site_replication_state(move |state| { + if state.updated_at != expected_updated_at || pending_endpoint_refresh(state).is_some() { + return Err(s3_error!( + InvalidRequest, + "site replication state changed during peer join; the peers may already be joined — re-run replicate add" + )); + } + adopt_add_commit_state(state, next_state); + let edit_generation = next_peer_edit_generation(state); + Ok((state.clone(), edit_generation)) + }) .await?; - - let mut fallback_peer = existing_peer_for_endpoint(&state, &site.endpoint) - .unwrap_or_else(|| normalize_peer_site(site.clone(), replicate_ilm_expiry)); - fallback_peer.deployment_id = preflight.deployment_id.clone(); - let join_response = parse_peer_join_response(&body, fallback_peer).map_err(|e| { - S3Error::with_message( - S3ErrorCode::InternalError, - format!("parse peer join response from {} failed: {e}", site.endpoint), - ) - })?; - if !join_response.initial_sync_error_message.is_empty() { - initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message)); - } - // An explicit no-op join. The peer answered 200 but wrote nothing — - // its persisted state is already newer than the snapshot it was - // sent — so the add is only PARTIALLY configured and saying - // "configured successfully" would be a lie (rustfs/rustfs#5963). - // `None` (a MinIO peer, or one older than the field) is not a - // no-op signal and is deliberately not reported. - if join_response.applied == Some(false) { - initial_sync_errors.push(format!( - "{}: peer did not apply the join (its site replication state is newer than the snapshot it was sent); \ - the site is not configured against this peer", - site.endpoint - )); - } - state = reconcile_peer_with_actual_identity(state, join_response.peer); - let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| { - S3Error::with_message( - S3ErrorCode::InternalError, - format!("peer join response from {} did not identify the requested site", site.endpoint), - ) - })?; - validate_proposed_peer(&reconciled_peer).map_err(|err| { - S3Error::with_message( - S3ErrorCode::InvalidRequest, - format!("invalid peer join response from {}: {err}", site.endpoint), - ) - })?; - } - - mark_unknown_peer_sync_enabled(&mut state.peers); - - // Commit. The CAS runs inside the transaction, against the state the - // transaction itself loaded — the peer round trips above took however - // long they took, and only this check can tell whether the topology - // this add was planned against is still the current one. The error - // says so: by this point the remote sites already accepted their - // joins, and re-running the add is what reconverges the local side. - let next_state = state; - let (state, edit_generation) = update_site_replication_state(move |state| { - if state.updated_at != expected_updated_at || pending_endpoint_refresh(state).is_some() { - return Err(s3_error!( - InvalidRequest, - "site replication state changed during peer join; the peers may already be joined — re-run replicate add" - )); - } - adopt_add_commit_state(state, next_state); - let edit_generation = next_peer_edit_generation(state); - Ok((state.clone(), edit_generation)) - }) - .await?; + Ok(( + state, + edit_generation, + local_peer, + service_account_secret_key, + initial_sync_errors, + add_in_progress_guard, + )) + }) + .await?; // The finalize fan-out delivers peer-edit payloads, so it carries the // generation allocated in the commit above: the receiving site orders @@ -7185,8 +7309,14 @@ impl Operation for SRPeerEditHandler { // The fence is self-reported — the shared service account means // the sender cannot be identified — so it is honoured only after // the admissibility check, against the same state it will gate. - let commit_fence = - commit_fence.filter(|fence| peer_edit_fence_is_admissible(state, &local_peer.deployment_id, fence)); + let commit_fence = match commit_fence { + Some(fence) if peer_edit_fence_is_admissible(state, &local_peer.deployment_id, &fence) => Some(fence), + // A fenced edit can only come from a current remote peer. If + // that origin left while the retry was in flight, applying + // its body here would resurrect the removed topology. + Some(_) => return Ok(StateCommit::Unchanged(PeerEditOutcome::Acked)), + None => None, + }; // Ordering fence: the sending site allocates the generation under // its state-object lock, so a delivery that lost the race carries // a generation this site has already passed. Applying it would @@ -8886,6 +9016,41 @@ mod tests { ); } + #[test] + fn add_admission_starts_before_preflight_and_rejects_bucket_set_changes() { + let expected = HashSet::from(["remote-owned".to_string(), "shared".to_string()]); + let present = HashSet::from(["shared".to_string()]); + + let err = ensure_add_bucket_set_matches_preflight(&expected, &present) + .expect_err("a missing bootstrap bucket must reject the topology commit"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + + let present = HashSet::from([ + "remote-owned".to_string(), + "shared".to_string(), + "created-during-add".to_string(), + ]); + let err = ensure_add_bucket_set_matches_preflight(&expected, &present) + .expect_err("a bucket created during add must reject the topology commit"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + + let src = include_str!("site_replication.rs"); + let add = src + .split("impl Operation for SiteReplicationAddHandler") + .nth(1) + .and_then(|rest| rest.split("pub struct SiteReplicationRemoveHandler").next()) + .expect("add handler block"); + let admission = add + .find("with_site_replication_bucket_mutation_admission_lock") + .expect("distributed mutation admission"); + let preflight = add.find("add_preflight_infos").expect("bucket preflight"); + let validation = add + .find("ensure_add_bucket_set_matches_preflight") + .expect("bucket-set validation"); + let commit = add.find("adopt_add_commit_state").expect("topology commit"); + assert!(admission < preflight && preflight < validation && validation < commit); + } + #[test] fn test_tls_capability_gates_run_before_add_or_edit_state_side_effects() { let src = include_str!("site_replication.rs"); @@ -9182,13 +9347,19 @@ mod tests { ); // Fence hardening: origin and generation are self-reported by a // caller the shared service account cannot identify, so the handler - // must pass the fence through the admissibility check — against the - // same state the fence gates, i.e. inside the transaction — before - // reading or raising any high-water mark. + // must admit the fence against the same state it gates. An origin + // removed while a retry was in flight is acknowledged without + // applying the stale body; otherwise it could recreate topology. assert!( - handler_block.contains(".filter(|fence| peer_edit_fence_is_admissible(state, &local_peer.deployment_id, fence))"), + handler_block.contains( + "Some(fence) if peer_edit_fence_is_admissible(state, &local_peer.deployment_id, &fence) => Some(fence)" + ), "SRPeerEditHandler must admit a fence only through peer_edit_fence_is_admissible inside the state transaction" ); + assert!( + handler_block.contains("Some(_) => return Ok(StateCommit::Unchanged(PeerEditOutcome::Acked))"), + "SRPeerEditHandler must not apply a fenced edit after its origin leaves the current topology" + ); // P1-15 PR2: both halves of the fence and the edit they fence share // ONE transaction. Checking the fence against a state read outside the // lock would let the check pass on one snapshot and the write land on @@ -10352,8 +10523,9 @@ mod tests { /// A fence is self-reported: every site authenticates peer traffic with /// the same site-replicator credential, so a compromised peer can stamp /// ANY origin with ANY generation. An origin the receiver does not - /// replicate with — or the receiver itself — is ignored and plants no - /// mark; a mark a compromised peer plants for a CURRENT origin cannot + /// replicate with — or the receiver itself — is inadmissible and plants + /// no mark; the handler acknowledges such a request without applying its + /// body. A mark a compromised peer plants for a CURRENT origin cannot /// silence that origin, because the staleness window refuses to fence on /// a mark implausibly far above the genuine deliveries. #[test] @@ -12791,6 +12963,7 @@ mod tests { last_error: "site replication is not enabled".to_string(), updated_at: Some(OffsetDateTime::now_utc()), edit_generation: None, + peer_unreachable: false, deletions_recorded: false, }], ..Default::default() @@ -12989,6 +13162,7 @@ mod tests { last_error: "peer offline".to_string(), updated_at: Some(OffsetDateTime::now_utc()), edit_generation: None, + peer_unreachable: false, deletions_recorded: false, }], ..Default::default() diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 7dada3492..e1951cd69 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -75,7 +75,8 @@ use crate::auth::get_condition_values_with_client_info; use crate::error::ApiError; use crate::shared_types::RemoteAddr; use crate::site_replication::{ - site_replication_bucket_meta_hook, site_replication_delete_bucket_hook, site_replication_make_bucket_hook, + cancel_site_replication_delete_bucket, commit_site_replication_delete_bucket, prepare_site_replication_delete_bucket, + site_replication_bucket_meta_hook, site_replication_make_bucket_hook, with_site_replication_bucket_mutation_lock, }; use crate::storage::storage_api::lock_bucket_targets_metadata; use http::StatusCode; @@ -1331,23 +1332,34 @@ impl DefaultBucketUsecase { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - let make_result = store - .make_bucket( - &bucket, - &MakeBucketOptions { - force_create: false, - lock_enabled, - ..Default::default() - }, - ) - .await; + // Keep the local namespace mutation and its peer hook ordered across + // every node in this site. Otherwise a delete waiting for repair + // coordination can arrive after this create on remote sites. + let operation_bucket = bucket.clone(); + let operation_store = store.clone(); + let make_result = with_site_replication_bucket_mutation_lock(store, &bucket, move || async move { + let make_result = operation_store + .make_bucket( + &operation_bucket, + &MakeBucketOptions { + force_create: false, + lock_enabled, + ..Default::default() + }, + ) + .await; + if make_result.is_ok() { + crate::storage::invalidate_bucket_validation_cache(&operation_bucket); + if let Err(err) = site_replication_make_bucket_hook(&operation_bucket, lock_enabled).await { + warn!(bucket = %operation_bucket, error = ?err, "site replication make bucket hook failed"); + } + } + make_result + }) + .await?; match make_result { - Ok(()) => { - // Invalidate the bucket validation cache so subsequent GETs - // see the newly created bucket immediately. - crate::storage::invalidate_bucket_validation_cache(&bucket); - } + Ok(()) => {} Err(StorageError::BucketExists(_)) => { // Per S3 spec: bucket namespace is global. Owner recreating returns 200 OK; // non-owner gets 409 BucketAlreadyExists. @@ -1358,10 +1370,6 @@ impl DefaultBucketUsecase { Err(e) => return Err(ApiError::from(e).into()), } - if let Err(err) = site_replication_make_bucket_hook(&bucket, lock_enabled).await { - warn!(bucket = %bucket, error = ?err, "site replication make bucket hook failed"); - } - let output = CreateBucketOutput::default(); counter!("rustfs_create_bucket_total").increment(1); let result = Ok(S3Response::new(output)); @@ -1397,16 +1405,41 @@ impl DefaultBucketUsecase { authorize_request(&mut req, Action::S3Action(S3Action::ForceDeleteBucketAction)).await?; } - store - .delete_bucket( - &input.bucket, - &DeleteBucketOptions { - force, - ..Default::default() - }, - ) - .await - .map_err(ApiError::from)?; + // Keep the local namespace mutation and its peer hook ordered across + // every node in this site so an older delete cannot overtake a new + // same-name make while it waits for repair coordination. + let operation_bucket = input.bucket.clone(); + let operation_store = store.clone(); + with_site_replication_bucket_mutation_lock(store, &input.bucket, move || async move { + let intent = prepare_site_replication_delete_bucket(&operation_bucket, force).await?; + let delete_result = operation_store + .delete_bucket( + &operation_bucket, + &DeleteBucketOptions { + force, + ..Default::default() + }, + ) + .await; + match delete_result { + Ok(()) => { + crate::storage::invalidate_bucket_validation_cache(&operation_bucket); + if let Some(intent) = intent + && let Err(err) = commit_site_replication_delete_bucket(&intent).await + { + warn!(bucket = %operation_bucket, error = ?err, "site replication delete bucket hook failed"); + } + Ok::<(), S3Error>(()) + } + Err(err) => { + if let Some(intent) = intent { + cancel_site_replication_delete_bucket(intent).await; + } + Err(S3Error::from(ApiError::from(err))) + } + } + }) + .await??; // Drop every cached object body for the now-deleted bucket so dead // bytes do not sit resident until TTL. Covers both the normal and the @@ -1415,16 +1448,9 @@ impl DefaultBucketUsecase { let cache_adapter = current_object_data_cache_for_context(self.context.as_deref()); let _ = invalidate_object_data_cache_bucket_after_delete(&cache_adapter, &input.bucket).await; - // Invalidate bucket validation cache - crate::storage::invalidate_bucket_validation_cache(&input.bucket); - // Re-evaluate lifecycle and replication after bucket removal. rustfs_scanner::record_scanner_maintenance_change(&input.bucket); - if let Err(err) = site_replication_delete_bucket_hook(&input.bucket, force).await { - warn!(bucket = %input.bucket, error = ?err, "site replication delete bucket hook failed"); - } - // Notify peers to drop their cached metadata for the now-deleted bucket. let request_context = req.extensions.get::().cloned(); notify_bucket_metadata_delete(input.bucket.clone(), request_context); diff --git a/rustfs/src/site_replication/hooks.rs b/rustfs/src/site_replication/hooks.rs index cd62704ee..95a830500 100644 --- a/rustfs/src/site_replication/hooks.rs +++ b/rustfs/src/site_replication/hooks.rs @@ -22,6 +22,57 @@ pub(crate) const SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION: &str = "confi pub(crate) static SITE_REPLICATION_BUCKET_OP_LOCK: LazyLock> = LazyLock::new(|| RwLock::new(())); +const SITE_REPLICATION_BUCKET_MUTATION_LOCK_PREFIX: &str = "config/site-replication/bucket-mutation"; +pub(crate) const SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH: &str = + "config/site-replication/bucket-mutation-admission.lock"; + +pub(crate) fn site_replication_bucket_mutation_lock_path(bucket: &str) -> String { + format!("{SITE_REPLICATION_BUCKET_MUTATION_LOCK_PREFIX}/{bucket}.lock") +} + +pub(crate) async fn with_site_replication_bucket_mutation_lock( + store: Arc, + bucket: &str, + operation: F, +) -> S3Result +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future + Send + 'static, + T: Send + 'static, +{ + let mutation_store = store.clone(); + let mutation_path = site_replication_bucket_mutation_lock_path(bucket); + with_config_object_read_lock( + store, + SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH.to_string(), + move || async move { + with_config_object_write_lock(mutation_store, mutation_path, operation) + .await + .map_err(|err| S3Error::from(ApiError::from(err))) + }, + ) + .await + .map_err(|err| S3Error::from(ApiError::from(err)))? +} + +/// Exclude every local bucket namespace mutation from an add's local preflight +/// snapshot until its topology commit. Peer bootstrap callbacks do not enter +/// this public-mutation admission path, so they can finish while the writer is +/// held; post-commit fan-out and backfill must run after it is released. +pub(crate) async fn with_site_replication_bucket_mutation_admission_lock( + store: Arc, + operation: F, +) -> S3Result +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future> + Send + 'static, + T: Send + 'static, +{ + with_config_object_write_lock(store, SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH.to_string(), operation) + .await + .map_err(|err| S3Error::from(ApiError::from(err)))? +} + #[derive(Debug, Default)] pub(crate) struct SiteReplicationBootstrapPlan { pub(crate) iam_items: Vec, @@ -329,6 +380,91 @@ pub(crate) fn site_replication_bootstrap_plan(info: &SRInfo) -> S3Result S3Result { + let mut plan = SiteReplicationBootstrapPlan { + bucket_make_ops: vec![bootstrap_bucket_make_op_path(bucket)], + bucket_configure_ops: vec![bootstrap_bucket_op_path( + &bucket.bucket, + SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION, + )], + ..Default::default() + }; + append_bootstrap_bucket_items(&mut plan, bucket, replicate_ilm_expiry)?; + Ok(plan) +} + +pub(crate) fn site_replication_bucket_retry_plan_from_info( + bucket: &SRBucketInfo, + replicate_ilm_expiry: bool, +) -> S3Result { + let mut plan = site_replication_bucket_retry_plan_for(bucket, replicate_ilm_expiry)?; + // Omit only metadata the make/configure operations can reproduce exactly. + // Non-default versioning fields and operator-authored replication rules + // remain in the plan; their extra request cost intentionally defers the + // event to the complete drain when the lightweight budget is too small. + plan.bucket_items.retain(|item| !retry_bucket_metadata_is_redundant(item)); + Ok(plan) +} + +fn retry_bucket_metadata_is_redundant(item: &SRBucketMeta) -> bool { + match item.r#type.as_str() { + "version-config" => item.versioning.as_deref().is_some_and(|raw| { + deserialize::(&decode_bucket_meta_wire_value(raw)).is_ok_and(|config| { + config + == VersioningConfiguration { + status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)), + ..Default::default() + } + }) + }), + "replication-config" => item.replication_config.as_deref().is_some_and(|raw| { + deserialize::(&decode_bucket_meta_wire_value(raw)) + .is_ok_and(|config| config.role.trim().is_empty() && config.rules.iter().all(is_derived_site_replication_rule)) + }), + // `Some("")` is the in-memory sentinel used when the bucket is lock + // enabled but has no object-lock configuration body. The make query + // carries lockEnabled=true; sending an empty metadata body is neither + // useful nor parseable. + "object-lock-config" => item.object_lock_config.as_deref() == Some(""), + _ => false, + } +} + +pub(crate) async fn site_replication_bucket_retry_plan( + bucket: &str, + replicate_ilm_expiry: bool, +) -> S3Result { + let Some(store) = current_object_store_handle() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + let bucket_info = match store.get_bucket_info(bucket, &BucketOptions::default()).await { + Ok(bucket_info) => bucket_info, + Err(err) if is_err_bucket_not_found(&err) => return Ok(SiteReplicationBootstrapPlan::default()), + Err(err) => return Err(ApiError::from(err).into()), + }; + let lock_enabled = bucket_info.object_locking; + let metadata = metadata_sys::get(bucket).await.map_err(ApiError::from)?; + let mut bucket_info = SRBucketInfo { + bucket: bucket.to_string(), + created_at: bucket_info.created, + location: current_region().map(|region| region.to_string()).unwrap_or_default(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }; + populate_sr_bucket_info_from_metadata(&mut bucket_info, &metadata).await; + if lock_enabled && bucket_info.object_lock_config.is_none() { + bucket_info.object_lock_config = Some(String::new()); + } + site_replication_bucket_retry_plan_from_info(&bucket_info, replicate_ilm_expiry) +} + pub async fn site_replication_make_bucket_hook(bucket: &str, lock_enabled: bool) -> S3Result<()> { let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.read().await; let runtime = { @@ -393,20 +529,273 @@ pub(crate) async fn broadcast_site_replication_make_bucket( broadcast_site_replication_json_using_runtime(runtime, &configure_path, &serde_json::json!({})).await } -pub async fn site_replication_delete_bucket_hook(bucket: &str, force_delete: bool) -> S3Result<()> { +const SITE_REPLICATION_DELETE_INTENT_PENDING: &str = + "bucket deletion reserved; local completion and peer delivery are not yet known"; + +#[derive(Clone)] +struct SiteReplicationDeleteBucketReservation { + peer: PeerInfo, + previous: Option, + observed: SiteReplicationRetryEvent, +} + +pub(crate) struct SiteReplicationDeleteBucketIntent { + path: String, + reservations: Vec, + displaced: Vec, +} + +fn site_replication_delete_bucket_path(bucket: &str, force_delete: bool) -> String { let operation = if force_delete { "force-delete-bucket" } else { "delete-bucket" }; - let path = format!( + format!( "/rustfs/admin/v3/site-replication/peer/bucket-ops?{}", form_urlencoded::Serializer::new(String::new()) .append_pair("bucket", bucket) .append_pair("operation", operation) .finish() - ); - broadcast_site_replication_json(&path, &serde_json::json!({})).await + ) +} + +/// Reserve every destructive peer delivery before the local namespace is +/// changed. The state transaction either persists the complete set or writes +/// nothing, so a full/unreadable queue fails the S3 delete closed. +pub(crate) async fn prepare_site_replication_delete_bucket( + bucket: &str, + force_delete: bool, +) -> S3Result> { + let path = site_replication_delete_bucket_path(bucket, force_delete); + let reservation_path = path.clone(); + update_site_replication_state_when_changed(move |state| { + if !state.enabled() { + return Ok(StateCommit::Unchanged(None)); + } + let local_peer = current_local_runtime_peer(state); + let peers = state + .peers + .values() + .filter(|peer| { + peer.deployment_id != local_peer.deployment_id && !same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) + }) + .cloned() + .collect::>(); + if peers.is_empty() { + return Ok(StateCommit::Unchanged(None)); + } + + let mut reservations = Vec::with_capacity(peers.len()); + let mut displaced = Vec::new(); + for peer in peers { + let previous = state + .retry_queue + .iter() + .find(|event| retry_event_matches(event, &peer, &reservation_path)) + .cloned(); + displaced.extend(upsert_site_replication_retry_event( + &mut state.retry_queue, + &peer, + &reservation_path, + SITE_REPLICATION_DELETE_INTENT_PENDING, + None, + )?); + let observed = state + .retry_queue + .iter() + .find(|event| retry_event_matches(event, &peer, &reservation_path)) + .cloned() + .ok_or_else(|| { + S3Error::with_message( + S3ErrorCode::InternalError, + "site replication delete reservation disappeared before commit".to_string(), + ) + })?; + reservations.push(SiteReplicationDeleteBucketReservation { + peer, + previous, + observed, + }); + } + Ok(StateCommit::Changed(Some(SiteReplicationDeleteBucketIntent { + path: reservation_path, + reservations, + displaced, + }))) + }) + .await +} + +/// Roll back a reservation when the local storage delete definitively failed. +/// A concurrently revised reservation is preserved; it belongs to a newer +/// observation and this operation has no authority to settle it. +pub(crate) async fn cancel_site_replication_delete_bucket(intent: SiteReplicationDeleteBucketIntent) { + let path = intent.path.clone(); + let result = update_site_replication_state_when_changed(move |state| { + let mut changed = false; + for reservation in intent.reservations { + let Some(index) = state.retry_queue.iter().position(|event| { + retry_event_matches(event, &reservation.peer, &reservation.observed.path) + && event.id == reservation.observed.id + && event.updated_at == reservation.observed.updated_at + }) else { + continue; + }; + if let Some(previous) = reservation.previous { + state.retry_queue[index] = previous; + } else { + state.retry_queue.remove(index); + } + changed = true; + } + + let mut restored_all = true; + for displaced in intent.displaced { + let duplicate = state.retry_queue.iter().any(|event| { + event.id == displaced.id + || (event.peer_deployment_id == displaced.peer_deployment_id && event.path == displaced.path) + }); + if duplicate { + continue; + } + if state.retry_queue.len() >= SITE_REPLICATION_RETRY_QUEUE_LIMIT { + restored_all = false; + continue; + } + state.retry_queue.push(displaced); + changed = true; + } + Ok(if changed { + StateCommit::Changed(restored_all) + } else { + StateCommit::Unchanged(restored_all) + }) + }) + .await; + + match result { + Ok(true) => {} + Ok(false) => warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + path, + result = "delete_intent_cancel_incomplete", + "admin site replication state" + ), + Err(err) => warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + path, + result = "delete_intent_cancel_failed", + error = ?err, + "admin site replication state" + ), + } +} + +async fn broadcast_site_replication_delete_bucket(intent: &SiteReplicationDeleteBucketIntent) -> S3Result<()> { + let sends = intent.reservations.iter().cloned().map(|reservation| { + let request_path = intent.path.clone(); + async move { + let fallback_peer = reservation.peer.clone(); + let observed = reservation.observed.clone(); + let delivery_path = request_path.clone(); + let delivery = with_site_replication_state_read_lock(move |state| async move { + let Some(current_peer) = state.peers.get(&fallback_peer.deployment_id).cloned() else { + return Ok(None); + }; + let service_account_secret_key = + match site_replicator_service_account_secret(&state.service_account_access_key).await { + Ok(secret) => secret, + Err(err) => { + let Some(secret) = legacy_site_replicator_state_secret(&state) else { + return Err(err); + }; + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "legacy_state_service_account_secret_fallback", + error = ?err, + "admin site replication state" + ); + secret + } + }; + let result = async { + let transport = PeerTransport::for_runtime_peer(¤t_peer).await?; + PeerAdminRequest::put(&transport.connection, &delivery_path, &state.service_account_access_key) + .with_client(&transport.client) + .send(&service_account_secret_key, &serde_json::json!({})) + .await + } + .await; + Ok(Some((current_peer, result))) + }) + .await; + match delivery { + Ok(Some((current_peer, Ok(_)))) => { + dequeue_observed_site_replication_retry_event(¤t_peer, &observed).await; + None + } + Ok(Some((current_peer, Err(err)))) => { + // Keep the failed deletion operator-visible, but never + // replay it automatically: without a bucket-incarnation + // fence, a delayed delete could erase a recreated bucket. + enqueue_site_replication_retry_event(¤t_peer, &request_path, &err).await; + Some(err) + } + Ok(None) => { + dequeue_observed_site_replication_retry_event(&reservation.peer, &observed).await; + None + } + Err(err) => { + enqueue_site_replication_retry_event(&reservation.peer, &request_path, &err).await; + Some(err) + } + } + } + }); + futures::future::join_all(sends) + .await + .into_iter() + .flatten() + .next() + .map_or(Ok(()), Err) +} + +pub(crate) async fn commit_site_replication_delete_bucket(intent: &SiteReplicationDeleteBucketIntent) -> S3Result<()> { + let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.read().await; + let store = + current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; + let retry_peers = intent + .reservations + .iter() + .map(|reservation| reservation.peer.clone()) + .collect::>(); + let retry_path = intent.path.clone(); + let delivery_intent = SiteReplicationDeleteBucketIntent { + path: intent.path.clone(), + reservations: intent.reservations.clone(), + displaced: Vec::new(), + }; + match with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move { + broadcast_site_replication_delete_bucket(&delivery_intent).await + }) + .await + { + Ok(result) => result, + Err(err) => { + let err: S3Error = ApiError::from(err).into(); + for peer in &retry_peers { + enqueue_site_replication_retry_event(peer, &retry_path, &err).await; + } + Err(err) + } + } } pub async fn site_replication_bucket_meta_hook(mut item: SRBucketMeta) -> S3Result<()> { @@ -515,6 +904,39 @@ pub(crate) fn maybe_time(value: OffsetDateTime) -> Option { (value != OffsetDateTime::UNIX_EPOCH).then_some(value) } +async fn populate_sr_bucket_info_from_metadata(entry: &mut SRBucketInfo, metadata: &BucketMetadata) { + entry.policy = raw_config_to_string(&metadata.policy_config_json).and_then(|raw| serde_json::from_str(&raw).ok()); + entry.versioning = raw_config_to_base64(&metadata.versioning_config_xml); + entry.tags = raw_config_to_base64(&metadata.tagging_config_xml); + entry.object_lock_config = raw_config_to_base64(&metadata.object_lock_config_xml); + entry.sse_config = raw_config_to_base64(&metadata.encryption_config_xml); + entry.replication_config = raw_config_to_base64(&metadata.replication_config_xml); + entry.quota_config = raw_config_to_base64(&metadata.quota_config_json); + // Expiry subset only: this entry feeds both the bootstrap/repair plan + // (peers must not receive transition rules) and cross-site consistency + // views (transition rules are site-local and would read as false + // mismatches). A deleted expiry state is a `None` value with the + // deletion's axis so repair can converge peers that missed the live + // delete. + let expiry_statement = lifecycle_expiry_statement(metadata); + entry.expiry_lc_config = expiry_statement.as_ref().and_then(|(subset, _)| subset.clone()); + entry.cors_config = raw_config_to_base64(&metadata.cors_config_xml); + entry.policy_updated_at = maybe_time(metadata.policy_config_updated_at); + entry.tag_config_updated_at = maybe_time(metadata.tagging_config_updated_at); + entry.object_lock_config_updated_at = maybe_time(metadata.object_lock_config_updated_at); + entry.sse_config_updated_at = maybe_time(metadata.encryption_config_updated_at); + entry.versioning_config_updated_at = maybe_time(metadata.versioning_config_updated_at); + entry.replication_config_updated_at = maybe_time(metadata.replication_config_updated_at); + entry.quota_config_updated_at = maybe_time(metadata.quota_config_updated_at); + // The expiry axis, not the whole-config write time: local transition-only + // edits inflate the latter, and a repair item stamped with it could + // out-rank a newer real expiry edit on a third site. + entry.expiry_lc_config_updated_at = expiry_statement.map(|(_, axis)| axis); + entry.cors_config_updated_at = maybe_time(metadata.cors_config_updated_at); + entry.replication_targets_online = + Some(site_replication_targets_online(&entry.bucket, &metadata.replication_config_xml).await); +} + pub(crate) async fn build_sr_info(state: &SiteReplicationState, local_peer: &PeerInfo) -> S3Result { let Some(store) = current_object_store_handle() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); @@ -546,37 +968,7 @@ pub(crate) async fn build_sr_info(state: &SiteReplicationState, local_peer: &Pee }; if let Some(metadata) = metadata { - entry.policy = raw_config_to_string(&metadata.policy_config_json).and_then(|raw| serde_json::from_str(&raw).ok()); - entry.versioning = raw_config_to_base64(&metadata.versioning_config_xml); - entry.tags = raw_config_to_base64(&metadata.tagging_config_xml); - entry.object_lock_config = raw_config_to_base64(&metadata.object_lock_config_xml); - entry.sse_config = raw_config_to_base64(&metadata.encryption_config_xml); - entry.replication_config = raw_config_to_base64(&metadata.replication_config_xml); - entry.quota_config = raw_config_to_base64(&metadata.quota_config_json); - // Expiry subset only: this entry feeds both the bootstrap/repair - // plan (peers must not receive transition rules) and cross-site - // consistency views (transition rules are site-local and would - // read as false mismatches). A deleted expiry state is a `None` - // value with the deletion's axis so repair can converge peers - // that missed the live delete. - let expiry_statement = lifecycle_expiry_statement(&metadata); - entry.expiry_lc_config = expiry_statement.as_ref().and_then(|(subset, _)| subset.clone()); - entry.cors_config = raw_config_to_base64(&metadata.cors_config_xml); - entry.policy_updated_at = maybe_time(metadata.policy_config_updated_at); - entry.tag_config_updated_at = maybe_time(metadata.tagging_config_updated_at); - entry.object_lock_config_updated_at = maybe_time(metadata.object_lock_config_updated_at); - entry.sse_config_updated_at = maybe_time(metadata.encryption_config_updated_at); - entry.versioning_config_updated_at = maybe_time(metadata.versioning_config_updated_at); - entry.replication_config_updated_at = maybe_time(metadata.replication_config_updated_at); - entry.quota_config_updated_at = maybe_time(metadata.quota_config_updated_at); - // The expiry axis, not the whole-config write time: local - // transition-only edits inflate the latter, and a repair item - // stamped with it could out-rank a newer real expiry edit on a - // third site. - entry.expiry_lc_config_updated_at = expiry_statement.map(|(_, axis)| axis); - entry.cors_config_updated_at = maybe_time(metadata.cors_config_updated_at); - entry.replication_targets_online = - Some(site_replication_targets_online(&bucket.name, &metadata.replication_config_xml).await); + populate_sr_bucket_info_from_metadata(&mut entry, &metadata).await; } info.buckets.insert(bucket.name, entry); diff --git a/rustfs/src/site_replication/mod.rs b/rustfs/src/site_replication/mod.rs index 4c3ad07d9..630c50576 100644 --- a/rustfs/src/site_replication/mod.rs +++ b/rustfs/src/site_replication/mod.rs @@ -47,6 +47,7 @@ use self::identity::{ canonical_endpoint, deployment_id_for_endpoint, mark_unknown_peer_sync_enabled, normalize_peer_map_by_identity_with, same_identity_endpoint, }; +pub(crate) use self::state_lock::with_site_replication_state_read_lock; use self::state_lock::{SITE_REPLICATION_STATE_PATH, with_site_replication_state_lock}; use crate::auth::constant_time_eq; use crate::config::get_config_snapshot; @@ -64,12 +65,12 @@ use crate::storage_api::site_replication::s3::{ #[cfg(test)] use crate::storage_api::site_replication::save_config as save_admin_config; use crate::storage_api::site_replication::{ - ARN, BUCKET_REPLICATION_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BucketOperations, BucketOptions, BucketTarget, - BucketTargetSys, BucketTargetType, BucketTargets, Credentials, ECStore, OperatorRuleContract, StorageError, - VersioningApi as _, assign_site_replication_rule_priorities, delete_config_no_lock, deserialize, is_site_replication_role, - lock_bucket_targets_metadata, metadata_sys, read_config as read_admin_config, read_config_no_lock, - replication_target_arn_deployment_id, save_config_no_lock, serialize, site_replication_rule_deployment_id, - with_config_object_read_lock, with_config_object_write_lock, + ARN, BUCKET_REPLICATION_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BucketMetadata, BucketOperations, + BucketOptions, BucketTarget, BucketTargetSys, BucketTargetType, BucketTargets, Credentials, ECStore, OperatorRuleContract, + StorageError, VersioningApi as _, assign_site_replication_rule_priorities, delete_config_no_lock, deserialize, + is_err_bucket_not_found, is_site_replication_role, lock_bucket_targets_metadata, metadata_sys, + read_config as read_admin_config, read_config_no_lock, replication_target_arn_deployment_id, save_config_no_lock, serialize, + site_replication_rule_deployment_id, with_config_object_read_lock, with_config_object_write_lock, }; use base64_simd::STANDARD as BASE64_STANDARD; use base64_simd::URL_SAFE_NO_PAD; diff --git a/rustfs/src/site_replication/repair.rs b/rustfs/src/site_replication/repair.rs index 5b3c260e3..b2785c0fd 100644 --- a/rustfs/src/site_replication/repair.rs +++ b/rustfs/src/site_replication/repair.rs @@ -649,7 +649,9 @@ pub(crate) async fn persist_site_replication_repair_task( let path = path.to_string(); update_site_replication_state(move |state| { match failure.as_deref() { - Some(error) => upsert_site_replication_retry_event(&mut state.retry_queue, &peer, &path, error, None), + Some(error) => { + upsert_site_replication_retry_event(&mut state.retry_queue, &peer, &path, error, None)?; + } None => { dequeue_site_replication_retry_events_including_escalated(&mut state.retry_queue, &peer, &path); // A repair is the operator's accountability transfer for the diff --git a/rustfs/src/site_replication/retry.rs b/rustfs/src/site_replication/retry.rs index 818fbee40..4b818c752 100644 --- a/rustfs/src/site_replication/retry.rs +++ b/rustfs/src/site_replication/retry.rs @@ -13,6 +13,7 @@ // limitations under the License. use super::*; +use futures::{StreamExt, stream}; pub(crate) const SITE_REPLICATION_RETRY_QUEUE_LIMIT: usize = 256; @@ -41,6 +42,12 @@ pub(crate) struct SiteReplicationRetryEvent { /// [`settle_site_replication_retry_events`]. #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) edit_generation: Option, + /// The latest delivery failure happened before an authenticated peer + /// response was received (connect, DNS, or TLS). Such failures + /// may bypass the expensive replay backoff only after a cheap devnull + /// reachability probe proves the peer is back. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub(crate) peer_unreachable: bool, /// Whether every failure folded into this collapsed IAM entry had its /// deletion body (if it was a deletion) recorded in /// [`SiteReplicationState::iam_deletion_replays`]. Only then may a @@ -196,27 +203,69 @@ pub(crate) fn settle_site_replication_retry_events( before.saturating_sub(queue.len()) } +pub(crate) fn settle_observed_site_replication_retry_event( + queue: &mut Vec, + peer: &PeerInfo, + observed: &SiteReplicationRetryEvent, +) -> usize { + let before = queue.len(); + queue.retain(|current| { + !(retry_event_matches(current, peer, &observed.path) + && current.id == observed.id + && current.updated_at == observed.updated_at) + }); + before.saturating_sub(queue.len()) +} + pub(crate) fn upsert_site_replication_retry_event( queue: &mut Vec, peer: &PeerInfo, path: &str, error: &str, generation: Option, -) { +) -> S3Result> { let path = collapsed_retry_queue_path(path).unwrap_or(path); let now = OffsetDateTime::now_utc(); let detail = summarize_peer_error_detail(error); + let peer_unreachable = retry_error_indicates_peer_unreachable(error); if let Some(event) = queue.iter_mut().find(|event| retry_event_matches(event, peer, path)) { + // The id is the event revision used by probe promotion and replay + // settlement. Refresh it on every failure so an older in-flight + // success can never acknowledge the newer observation. + event.id = Uuid::new_v4().to_string(); event.retry_count = event.retry_count.saturating_add(1); event.failed = event.retry_count >= SITE_REPLICATION_RETRY_FAILED_AFTER; event.last_error = detail; event.updated_at = Some(now); + event.peer_unreachable = peer_unreachable; // Keep the newest generation: an older delivery that fails afterwards // must not lower the fence and let its own success settle the event. event.edit_generation = event.edit_generation.max(generation); - return; + return Ok(Vec::new()); } + let slots_needed = queue + .len() + .saturating_add(1) + .saturating_sub(SITE_REPLICATION_RETRY_QUEUE_LIMIT); + let mut evict_indices = queue + .iter() + .enumerate() + .filter_map(|(index, event)| retry_event_is_safely_replayable(event).then_some(index)) + .take(slots_needed) + .collect::>(); + if evict_indices.len() != slots_needed { + return Err(S3Error::with_message( + S3ErrorCode::ServiceUnavailable, + "site replication retry queue is full of non-evictable liabilities; repair them before recording more failures" + .to_string(), + )); + } + let mut evicted = Vec::with_capacity(evict_indices.len()); + while let Some(index) = evict_indices.pop() { + evicted.push(queue.remove(index)); + } + evicted.reverse(); queue.push(SiteReplicationRetryEvent { id: Uuid::new_v4().to_string(), peer_deployment_id: peer.deployment_id.clone(), @@ -227,12 +276,38 @@ pub(crate) fn upsert_site_replication_retry_event( last_error: detail, updated_at: Some(now), edit_generation: generation, + peer_unreachable, deletions_recorded: false, }); - if queue.len() > SITE_REPLICATION_RETRY_QUEUE_LIMIT { - let overflow = queue.len() - SITE_REPLICATION_RETRY_QUEUE_LIMIT; - queue.drain(0..overflow); + Ok(evicted) +} + +pub(crate) fn is_destructive_bucket_retry_path(path: &str) -> bool { + matches!( + retry_bucket_operation(path).as_deref(), + Some("delete-bucket" | "force-delete-bucket" | "purge-deleted-bucket") + ) +} + +fn retry_event_is_safely_replayable(event: &SiteReplicationRetryEvent) -> bool { + if is_destructive_bucket_retry_path(&event.path) { + return false; } + matches!( + classify_site_replication_retry_event(event), + Some(RetryDrainAction::PeerEdit | RetryDrainAction::BucketOpReplay { .. }) + ) +} + +pub(crate) fn retry_error_indicates_peer_unreachable(error: &str) -> bool { + let error = error.to_ascii_lowercase(); + let Some((_, request)) = error.split_once("peer request to ") else { + return false; + }; + let Some((_, failure)) = request.split_once(" failed ") else { + return false; + }; + failure.starts_with("(connect):") || failure.starts_with("(dns resolution):") || failure.starts_with("(tls handshake):") } pub(crate) fn retry_stats_for_state(state: &SiteReplicationState) -> Option { @@ -271,7 +346,7 @@ pub(crate) async fn enqueue_site_replication_retry_event_for_generation( // (remove_sites already pruned them); recording a late failure for it // would only pollute retry_stats until the queue cap evicts it. if state.peers.contains_key(&peer_owned.deployment_id) { - upsert_site_replication_retry_event(&mut state.retry_queue, &peer_owned, &path_owned, &error_text, generation); + upsert_site_replication_retry_event(&mut state.retry_queue, &peer_owned, &path_owned, &error_text, generation)?; } Ok(()) }) @@ -356,12 +431,17 @@ pub(crate) fn iam_item_deletion_entity(item: &SRIAMItem) -> Option { /// live in the same state so the caller commits them in one transaction — a /// retry entry can never exist whose deletion body was lost to a separate /// failed write. -pub(crate) fn record_failed_iam_delivery(state: &mut SiteReplicationState, peer: &PeerInfo, item: &SRIAMItem, error: &str) { +pub(crate) fn record_failed_iam_delivery( + state: &mut SiteReplicationState, + peer: &PeerInfo, + item: &SRIAMItem, + error: &str, +) -> S3Result<()> { let existed = state .retry_queue .iter() .any(|event| retry_event_matches(event, peer, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH)); - upsert_site_replication_retry_event(&mut state.retry_queue, peer, SITE_REPLICATION_PEER_IAM_ITEM_WIRE_PATH, error, None); + upsert_site_replication_retry_event(&mut state.retry_queue, peer, SITE_REPLICATION_PEER_IAM_ITEM_WIRE_PATH, error, None)?; if !existed && let Some(event) = state .retry_queue @@ -375,13 +455,13 @@ pub(crate) fn record_failed_iam_delivery(state: &mut SiteReplicationState, peer: } let Some(entity) = iam_item_deletion_entity(item) else { - return; + return Ok(()); }; let item_value = match serde_json::to_value(item) { Ok(value) => value, Err(_) => { degrade_iam_retry_event_to_escalation(state, peer); - return; + return Ok(()); } }; let now = OffsetDateTime::now_utc(); @@ -390,9 +470,13 @@ pub(crate) fn record_failed_iam_delivery(state: &mut SiteReplicationState, peer: .iter_mut() .find(|record| iam_deletion_replay_matches(record, peer) && record.entity == entity) { + // This id is the replay-record revision. A settlement that sent the + // previous body must not remove a same-entity deletion that failed + // while its snapshot was in flight. + existing.id = Uuid::new_v4().to_string(); existing.item = item_value; existing.recorded_at = Some(now); - return; + return Ok(()); } let per_peer = state @@ -424,6 +508,7 @@ pub(crate) fn record_failed_iam_delivery(state: &mut SiteReplicationState, peer: item: item_value, recorded_at: Some(now), }); + Ok(()) } pub(crate) fn degrade_iam_retry_event_to_escalation(state: &mut SiteReplicationState, peer: &PeerInfo) { @@ -467,7 +552,7 @@ pub(crate) async fn record_failed_site_replication_iam_delivery(peer: &PeerInfo, // A departed peer can never drain its entries again (remove_sites // already pruned them) — mirror enqueue_site_replication_retry_event. if state.peers.contains_key(&peer_owned.deployment_id) { - record_failed_iam_delivery(state, &peer_owned, &item_owned, &error_text); + record_failed_iam_delivery(state, &peer_owned, &item_owned, &error_text)?; } Ok(()) }) @@ -511,15 +596,14 @@ pub(crate) async fn record_failed_site_replication_iam_delivery(peer: &PeerInfo, pub(crate) fn settle_replayed_iam_retry_events( state: &mut SiteReplicationState, peer: &PeerInfo, - path: &str, - snapshot_updated_at: Option, + observed: &SiteReplicationRetryEvent, replayed_record_ids: &[String], ) -> bool { state .iam_deletion_replays .retain(|record| !(iam_deletion_replay_matches(record, peer) && replayed_record_ids.contains(&record.id))); - if collapsed_retry_queue_path(path) != Some(SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH) { + if collapsed_retry_queue_path(&observed.path) != Some(SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH) { return false; } let Some(index) = state @@ -530,11 +614,7 @@ pub(crate) fn settle_replayed_iam_retry_events( return false; }; let event = &state.retry_queue[index]; - let newer_failure_recorded = match (event.updated_at, snapshot_updated_at) { - (Some(current), Some(seen)) => current > seen, - (Some(_), None) => true, - (None, _) => false, - }; + let newer_failure_recorded = event.id != observed.id || event.updated_at != observed.updated_at; if newer_failure_recorded && event.last_error != SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER { // The newer failure's own deletion (if any) has its own record; the // next drain pass replays it. @@ -548,24 +628,22 @@ pub(crate) fn settle_replayed_iam_retry_events( state.retry_queue.remove(index); return true; } - escalate_site_replication_retry_events_up_to(&mut state.retry_queue, peer, path, snapshot_updated_at); + escalate_site_replication_retry_events_up_to(&mut state.retry_queue, peer, &observed.path, observed.updated_at); false } pub(crate) async fn settle_replayed_site_replication_iam_retry_event( peer: &PeerInfo, - path: &str, - snapshot_updated_at: Option, + observed: &SiteReplicationRetryEvent, replayed_record_ids: Vec, ) { let peer_owned = peer.clone(); - let path_owned = path.to_string(); + let observed_owned = observed.clone(); let result = update_site_replication_state(move |state| { Ok(settle_replayed_iam_retry_events( state, &peer_owned, - &path_owned, - snapshot_updated_at, + &observed_owned, &replayed_record_ids, )) }) @@ -591,7 +669,7 @@ pub(crate) async fn settle_replayed_site_replication_iam_retry_event( event = EVENT_ADMIN_SITE_REPLICATION_STATE, peer = %peer.endpoint, deployment_id = %peer.deployment_id, - path, + path = %observed.path, error = ?err, "failed to settle replayed site replication IAM retry event" ); @@ -652,8 +730,30 @@ pub(crate) const SITE_REPLICATION_RETRY_DRAIN_BASE_BACKOFF_SECS: i64 = 600; /// converges at the next tick instead of waiting out this ceiling. pub(crate) const SITE_REPLICATION_RETRY_DRAIN_MAX_BACKOFF_SECS: i64 = 86_400; -/// What the background drain may do for one retry event. Everything not -/// representable here is operator territory (manual repair). +/// Bound a background drain round to one complete bucket bootstrap chain: +/// make, at most nine metadata records, then replication configuration. +/// Larger snapshots and topology edits remain queued for operator repair. +pub(crate) const SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER: usize = 11; + +/// Bound sockets for every retry pass. The lightweight pass also admits at +/// most this many peer request chains per round, bounding its lock hold time. +pub(crate) const SITE_REPLICATION_RETRY_DRAIN_PEER_CONCURRENCY: usize = 4; + +/// Rotate the bounded lightweight window instead of always admitting the +/// lexicographically first peers. Advancing by one full window per scheduler +/// round gives every queued peer a turn within `ceil(peer_count / limit)` +/// rounds, even when earlier peers each have a large bucket backlog. +pub(crate) fn lightweight_retry_peer_rotation(peer_count: usize, round: i64) -> usize { + if peer_count == 0 { + return 0; + } + let window = SITE_REPLICATION_RETRY_DRAIN_PEER_CONCURRENCY.min(peer_count); + (round.rem_euclid(peer_count as i64) as usize * window) % peer_count +} + +/// A replay shape the retry machinery can derive from persisted state. The +/// lightweight 30-second scheduler admits only bounded bucket-op chains; +/// snapshot and topology-wide work remains operator territory. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum RetryDrainAction { /// Constant-path IAM item deliveries collapse into one queue entry per @@ -670,6 +770,10 @@ pub(crate) enum RetryDrainAction { PeerEdit, } +pub(crate) fn is_lightweight_retry_drain_action(action: &RetryDrainAction) -> bool { + matches!(action, RetryDrainAction::BucketOpReplay { .. }) +} + #[derive(Clone)] pub(crate) enum RetrySnapshot { Iam(Vec), @@ -724,27 +828,126 @@ impl RetrySnapshot { } } - pub(crate) async fn send(&self, transport: &PeerTransport, access_key: &str, secret_key: &str) -> S3Result<()> { + pub(crate) async fn send( + &self, + peer: &PeerInfo, + transport: &PeerTransport, + access_key: &str, + secret_key: &str, + ) -> S3Result { match self { Self::Iam(items) => { for item in items { - SiteReplicationRepairTask::Iam(item) - .send(transport, access_key, secret_key) - .await?; + if !send_retry_task_if_peer_current( + peer, + &SiteReplicationRepairTask::Iam(item), + transport, + access_key, + secret_key, + ) + .await? + { + return Ok(false); + } } } Self::BucketMetadata(items) => { for item in items { - SiteReplicationRepairTask::BucketMetadata(item) - .send(transport, access_key, secret_key) - .await?; + if !send_retry_task_if_peer_current( + peer, + &SiteReplicationRepairTask::BucketMetadata(item), + transport, + access_key, + secret_key, + ) + .await? + { + return Ok(false); + } } } } - Ok(()) + Ok(true) } } +pub(crate) async fn send_retry_task_if_peer_current( + peer: &PeerInfo, + task: &SiteReplicationRepairTask<'_>, + transport: &PeerTransport, + access_key: &str, + secret_key: &str, +) -> S3Result { + let body = match task { + SiteReplicationRepairTask::Iam(item) => serde_json::to_value(item), + SiteReplicationRepairTask::BucketMetadata(item) => serde_json::to_value(item), + SiteReplicationRepairTask::BucketMake(_) | SiteReplicationRepairTask::Replication(_) => Ok(serde_json::json!({})), + } + .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize retry task failed: {err}")))?; + send_retry_request_if_peer_current(peer, transport, task.path(), access_key, secret_key, body).await +} + +pub(crate) async fn send_retry_request_if_peer_current( + peer: &PeerInfo, + transport: &PeerTransport, + path: &str, + access_key: &str, + secret_key: &str, + body: Value, +) -> S3Result { + let peer = peer.clone(); + let transport = transport.clone(); + let path = path.to_string(); + let access_key = access_key.to_string(); + let secret_key = secret_key.to_string(); + with_site_replication_state_read_lock(move |state| async move { + let current = state + .peers + .get(&peer.deployment_id) + .is_some_and(|current| same_identity_endpoint(¤t.endpoint, &peer.endpoint)); + if !current { + return Ok(false); + } + PeerAdminRequest::put(&transport.connection, &path, &access_key) + .with_client(&transport.client) + .send(&secret_key, &body) + .await?; + Ok(true) + }) + .await +} + +async fn send_peer_edit_retry_if_peer_current( + peer: &PeerInfo, + transport: &PeerTransport, + path: &str, + access_key: &str, + secret_key: &str, + body: Value, +) -> S3Result { + let peer_owned = peer.clone(); + let current = with_site_replication_state_read_lock(move |state| async move { + Ok(state + .peers + .get(&peer_owned.deployment_id) + .is_some_and(|current| same_identity_endpoint(¤t.endpoint, &peer_owned.endpoint))) + }) + .await?; + if !current { + return Ok(false); + } + // A peer-edit handler takes its own site's state write lock. Releasing + // this site's read lock before the request prevents simultaneous A -> B + // and B -> A retries from waiting on each other's write lock. The edit + // generation carried by `path` fences a delivery overtaken by a newer + // topology commit after this check. + PeerAdminRequest::put(&transport.connection, path, access_key) + .with_client(&transport.client) + .send(secret_key, &body) + .await?; + Ok(true) +} + #[derive(Hash, PartialEq, Eq)] pub(crate) enum IamSnapshotKey { Policy(String), @@ -872,6 +1075,66 @@ pub(crate) fn retry_bucket_name(path: &str) -> Option { .find_map(|(key, value)| (key == "bucket" && !value.is_empty()).then(|| value.into_owned())) } +pub(crate) fn bucket_op_retry_replay_tasks<'a>( + plan: &'a SiteReplicationBootstrapPlan, + operation: &str, + bucket: &str, +) -> S3Result>> { + let matches_bucket = |path: &&String| retry_bucket_name(path).as_deref() == Some(bucket); + match operation { + SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING => { + let mut tasks = plan + .bucket_make_ops + .iter() + .filter(matches_bucket) + .map(|path| SiteReplicationRepairTask::BucketMake(path.as_str())) + .collect::>(); + if tasks.is_empty() { + return Ok(tasks); + } + let configure_tasks = plan + .bucket_configure_ops + .iter() + .filter(matches_bucket) + .map(|path| SiteReplicationRepairTask::Replication(path.as_str())) + .collect::>(); + if configure_tasks.is_empty() { + return Err(S3Error::with_message( + S3ErrorCode::InternalError, + format!("site replication retry plan has no configure operation for bucket {bucket:?}"), + )); + } + tasks.extend( + plan.bucket_items + .iter() + .filter(|item| item.bucket == bucket) + .map(SiteReplicationRepairTask::BucketMetadata), + ); + tasks.extend(configure_tasks); + Ok(tasks) + } + SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION => Ok(plan + .bucket_configure_ops + .iter() + .filter(matches_bucket) + .map(|path| SiteReplicationRepairTask::Replication(path.as_str())) + .collect()), + _ => Err(S3Error::with_message( + S3ErrorCode::InvalidArgument, + format!("unsupported site replication retry bucket operation {operation:?}"), + )), + } +} + +pub(crate) fn retry_drain_request_count(action: &RetryDrainAction, plan: Option<&SiteReplicationBootstrapPlan>) -> usize { + match action { + RetryDrainAction::BucketOpReplay { operation, bucket } => plan + .and_then(|plan| bucket_op_retry_replay_tasks(plan, operation, bucket).ok()) + .map_or(0, |tasks| tasks.len()), + RetryDrainAction::IamSnapshot | RetryDrainAction::BucketMetadataSnapshot | RetryDrainAction::PeerEdit => usize::MAX, + } +} + /// A collapsed retry event after a stable snapshot resend is escalated with /// this marker instead of being cleared: the snapshot contains no task for a /// failed deletion, so remote absence remains operator-visible. Collapsed @@ -989,10 +1252,10 @@ pub(crate) fn actionable_site_replication_retry_events( /// backoff exists to spare a *dead* peer the expensive replay (plan build, /// snapshot resend) — it must not delay convergence to a peer that has /// already RECOVERED, or a failure window ends in up to a day of silent -/// divergence (backlog#2071). The drain probes each such peer with one cheap -/// request per tick and promotes its backlog when the probe succeeds. The -/// base backoff still floors individual re-attempts so a reachable peer that -/// keeps failing a delivery is not hammered faster than before. +/// divergence (backlog#2071). Peer connection failures may be probed before +/// the normal replay backoff elapses; request timeouts and application +/// failures still wait at least one base interval so a reachable peer that +/// keeps rejecting a replay is not hammered faster than before. pub(crate) fn deferred_site_replication_retry_events( state: &SiteReplicationState, now: OffsetDateTime, @@ -1005,7 +1268,14 @@ pub(crate) fn deferred_site_replication_retry_events( .filter(|event| !site_replication_retry_backoff_elapsed(event, now)) .filter(|event| { event.updated_at.is_none_or(|updated_at| { - now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) >= SITE_REPLICATION_RETRY_DRAIN_BASE_BACKOFF_SECS + event.peer_unreachable + // Older binaries do not persist `peer_unreachable`. Parse + // only the locally-produced outer transport-error shape so + // rolling upgrades retain fast recovery without trusting + // an HTTP error body containing the same words. + || retry_error_indicates_peer_unreachable(&event.last_error) + || now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) + >= SITE_REPLICATION_RETRY_DRAIN_BASE_BACKOFF_SECS }) }) .cloned() @@ -1036,11 +1306,31 @@ pub(crate) async fn probe_site_replication_peer_reachable(runtime: &SiteReplicat /// every peer that answers. A probe failure advances nothing: retry counts /// only move on real delivery attempts, so the per-event backoff is intact /// when the peer is genuinely down. +pub(crate) fn mark_reachable_deferred_retry_events( + state: &mut SiteReplicationState, + recovered: &[SiteReplicationRetryEvent], +) -> usize { + let mut promoted = 0; + for recovered in recovered { + if let Some(current) = state.retry_queue.iter_mut().find(|current| { + current.id == recovered.id + && current.peer_deployment_id == recovered.peer_deployment_id + && current.path == recovered.path + && current.updated_at == recovered.updated_at + }) { + current.updated_at = None; + current.peer_unreachable = false; + promoted += 1; + } + } + promoted +} + pub(crate) async fn promote_reachable_deferred_retry_events( runtime: &SiteReplicationRuntime, - actionable: &mut Vec, + actionable: &[SiteReplicationRetryEvent], deferred: Vec, -) { +) -> S3Result<()> { let due_peers: HashSet = actionable.iter().map(|event| event.peer_deployment_id.clone()).collect(); let mut deferred_by_peer: BTreeMap> = BTreeMap::new(); for event in deferred { @@ -1054,29 +1344,50 @@ pub(crate) async fn promote_reachable_deferred_retry_events( .or_default() .push(event); } - for (deployment_id, events) in deferred_by_peer { - let Some(peer) = runtime.state.peers.get(&deployment_id) else { - continue; - }; + let probes = deferred_by_peer.into_iter().filter_map(|(deployment_id, events)| { + let peer = runtime.state.peers.get(&deployment_id)?; if deployment_id == runtime.local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &runtime.local_peer.endpoint) { - continue; + return None; } - if probe_site_replication_peer_reachable(runtime, peer).await { + Some(async move { + let reachable = probe_site_replication_peer_reachable(runtime, peer).await; + (deployment_id, peer.endpoint.clone(), events, reachable) + }) + }); + let mut recovered = Vec::new(); + let probe_results = stream::iter(probes) + .buffer_unordered(SITE_REPLICATION_RETRY_DRAIN_PEER_CONCURRENCY) + .collect::>() + .await; + for (deployment_id, peer_endpoint, events, reachable) in probe_results { + if reachable { info!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - peer = %peer.endpoint, + result = "retry_backoff_probe_promoted", + peer = %peer_endpoint, deployment_id = %deployment_id, promoted = events.len(), - result = "retry_backoff_probe_promoted", - "peer reachable again; replaying its backed-off retry events this tick" + "peer reachable again; promoting backed-off retry events for replay" ); - actionable.extend(events); + recovered.extend(events); } } + if recovered.is_empty() { + return Ok(()); + } + update_site_replication_state_when_changed(move |state| { + let promoted = mark_reachable_deferred_retry_events(state, &recovered); + Ok(if promoted == 0 { + StateCommit::Unchanged(()) + } else { + StateCommit::Changed(()) + }) + }) + .await } /// Operator-visible per-tick alert for retry entries that no longer converge @@ -1142,6 +1453,78 @@ pub(crate) async fn drain_site_replication_retry_queue() { } } +/// Fast outage-recovery pass used by the 30-second scheduler. It limits work +/// to one bounded bucket-op chain per peer, builds no site-wide snapshot, and +/// runs reachability probes concurrently with replay for other peers. +pub(crate) async fn drain_site_replication_retry_queue_lightweight() { + if let Err(err) = drain_site_replication_retry_queue_lightweight_inner().await { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "retry_drain_failed", + error = ?err, + "admin site replication state" + ); + } +} + +async fn drain_site_replication_retry_queue_lightweight_inner() -> S3Result<()> { + let Some(runtime) = runtime_site_replication_targets().await? else { + return Ok(()); + }; + log_site_replication_retry_liabilities(&runtime.state); + if runtime.state.pending_endpoint_refresh.is_some() + || runtime.state.pending_remove.is_some() + || runtime.state.pending_rotation.is_some() + { + return Ok(()); + } + let now = OffsetDateTime::now_utc(); + let mut actionable = actionable_site_replication_retry_events(&runtime.state, now); + let mut deferred = deferred_site_replication_retry_events(&runtime.state, now); + actionable.retain(|event| { + classify_site_replication_retry_event(event).is_some_and(|action| is_lightweight_retry_drain_action(&action)) + }); + deferred.retain(|event| { + classify_site_replication_retry_event(event).is_some_and(|action| is_lightweight_retry_drain_action(&action)) + }); + if actionable.is_empty() && deferred.is_empty() { + return Ok(()); + } + let Some(store) = current_object_store_handle() else { + return Ok(()); + }; + + // Probes are read-only and can consume the full request timeout. Keep + // them outside repair coordination; promotion is fenced by event id and + // timestamp, and the locked reload below decides what may actually send. + promote_reachable_deferred_retry_events(&runtime, &actionable, deferred).await?; + + with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move { + let Some(runtime) = runtime_site_replication_targets().await? else { + return Ok(()); + }; + if runtime.state.pending_endpoint_refresh.is_some() + || runtime.state.pending_remove.is_some() + || runtime.state.pending_rotation.is_some() + { + return Ok(()); + } + let now = OffsetDateTime::now_utc(); + let mut actionable = actionable_site_replication_retry_events(&runtime.state, now); + actionable.retain(|event| { + classify_site_replication_retry_event(event).is_some_and(|action| is_lightweight_retry_drain_action(&action)) + }); + if actionable.is_empty() { + return Ok(()); + } + drain_site_replication_retry_queue_lightweight_locked(Arc::new(runtime), actionable, now).await + }) + .await + .map_err(ApiError::from)? +} + pub(crate) async fn drain_site_replication_retry_queue_inner() -> S3Result<()> { let Some(runtime) = runtime_site_replication_targets().await? else { return Ok(()); @@ -1150,7 +1533,7 @@ pub(crate) async fn drain_site_replication_retry_queue_inner() -> S3Result<()> { // escalated markers are exactly the entries the drain skips. log_site_replication_retry_liabilities(&runtime.state); let now = OffsetDateTime::now_utc(); - let mut actionable = actionable_site_replication_retry_events(&runtime.state, now); + let actionable = actionable_site_replication_retry_events(&runtime.state, now); let deferred = deferred_site_replication_retry_events(&runtime.state, now); if actionable.is_empty() && deferred.is_empty() { return Ok(()); @@ -1167,27 +1550,146 @@ pub(crate) async fn drain_site_replication_retry_queue_inner() -> S3Result<()> { // guard) may have started since. Re-check on the fresh state. return Ok(()); } - // Probe before taking the repair lock: probes are read-only peer traffic - // and a dead peer's connect timeout must not hold the lock. - promote_reachable_deferred_retry_events(&runtime, &mut actionable, deferred).await; - if actionable.is_empty() { - return Ok(()); - } - // Serialize against operator repair execution. This does NOT close the + + // Persist successful recovery probes without monopolizing repair + // coordination. The locked reload below observes those promotions and + // replays them in this same round. + promote_reachable_deferred_retry_events(&runtime, &actionable, deferred).await?; + + // Serialize against operator repair execution. Peer membership is + // re-checked from a distributed state snapshot immediately before each + // network request, so the caller need not hold the lifecycle guard while + // a large snapshot is replayed. This does NOT close the // dry-run -> execute window (dry-run takes no lock): a drain settling a // replayable bucket-op entry in that window changes the preflight token // and execute fails safe with "preflight is stale" — the operator - // re-runs the dry-run. Lock order matches repair: lifecycle guard (held - // by the reconcile tick) -> repair execution lock -> state object lock - // inside the send bookkeeping. An operator repair holding the lock makes - // this tick skip after the lock-acquire timeout. + // re-runs the dry-run. The lock elects one server to replay the queue; + // after acquiring it, reload state so a settled event or deleted bucket + // cannot be replayed from this admission snapshot. with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move { + // Runtime and queue snapshots captured before this distributed lock + // are only admission hints. Another node may have settled the event, + // or a local bucket may have been deleted, while this node waited. + let Some(runtime) = runtime_site_replication_targets().await? else { + return Ok(()); + }; + if runtime.state.pending_endpoint_refresh.is_some() + || runtime.state.pending_remove.is_some() + || runtime.state.pending_rotation.is_some() + { + return Ok(()); + } + let now = OffsetDateTime::now_utc(); + let actionable = actionable_site_replication_retry_events(&runtime.state, now); + if actionable.is_empty() { + return Ok(()); + } drain_site_replication_retry_queue_locked(runtime, actionable).await }) .await .map_err(ApiError::from)? } +async fn drain_site_replication_retry_queue_lightweight_locked( + runtime: Arc, + events: Vec, + now: OffsetDateTime, +) -> S3Result<()> { + let mut events_by_peer: BTreeMap> = BTreeMap::new(); + for event in events { + events_by_peer + .entry(event.peer_deployment_id.clone()) + .or_default() + .push(event); + } + + let mut peer_groups = events_by_peer + .into_iter() + .filter_map(|(deployment_id, peer_events)| { + let peer = runtime.state.peers.get(&deployment_id)?.clone(); + if deployment_id == runtime.local_peer.deployment_id + || same_identity_endpoint(&peer.endpoint, &runtime.local_peer.endpoint) + { + return None; + } + Some((peer, peer_events)) + }) + .collect::>(); + let interval_secs = crate::site_replication_reconcile::RETRY_DRAIN_INTERVAL.as_secs() as i64; + let round = now.unix_timestamp().div_euclid(interval_secs); + let rotation = lightweight_retry_peer_rotation(peer_groups.len(), round); + peer_groups.rotate_left(rotation); + + let peer_replays = peer_groups + .into_iter() + .map(|(peer, peer_events)| { + let runtime = Arc::clone(&runtime); + async move { + let Some((event, action, bucket)) = peer_events.into_iter().find_map(|event| { + let action = classify_site_replication_retry_event(&event)?; + let bucket = match &action { + RetryDrainAction::BucketOpReplay { bucket, .. } => bucket.clone(), + _ => return None, + }; + Some((event, action, bucket)) + }) else { + return (0, 0); + }; + let plan = match site_replication_bucket_retry_plan( + &bucket, + site_replication_state_replicates_ilm_expiry(&runtime.state), + ) + .await + { + Ok(plan) => plan, + Err(err) => { + enqueue_site_replication_retry_event(&peer, &event.path, &err).await; + return (0, 1); + } + }; + if retry_drain_request_count(&action, Some(&plan)) > SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER { + return (0, 0); + } + let transport = match PeerTransport::for_runtime_peer(&peer).await { + Ok(transport) => transport, + Err(err) => { + enqueue_site_replication_retry_event(&peer, &event.path, &err).await; + return (0, 1); + } + }; + match drain_one_site_replication_retry_event(&runtime, &peer, &transport, &event, action, Some(&plan)).await { + Ok(true) => (1, 0), + Ok(false) => (0, 0), + Err(_) => (0, 1), + } + } + }) + .take(SITE_REPLICATION_RETRY_DRAIN_PEER_CONCURRENCY); + + let mut settled = 0usize; + let mut failures = 0usize; + let replay_results = stream::iter(peer_replays) + .buffer_unordered(SITE_REPLICATION_RETRY_DRAIN_PEER_CONCURRENCY) + .collect::>() + .await; + for (peer_settled, peer_failures) in replay_results { + settled += peer_settled; + failures += peer_failures; + } + if settled > 0 || failures > 0 { + info!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "retry_drain_settled", + settled, + failures, + "admin site replication state" + ); + } + Ok(()) +} + pub(crate) async fn drain_site_replication_retry_queue_locked( runtime: SiteReplicationRuntime, events: Vec, @@ -1212,39 +1714,54 @@ pub(crate) async fn drain_site_replication_retry_queue_locked( .push(event); } - let mut settled = 0usize; - let mut failures = 0usize; - for (deployment_id, peer_events) in events_by_peer { - let Some(peer) = runtime.state.peers.get(&deployment_id) else { - continue; - }; + let runtime = Arc::new(runtime); + let plan = plan.map(Arc::new); + let peer_replays = events_by_peer.into_iter().filter_map(|(deployment_id, peer_events)| { + let peer = runtime.state.peers.get(&deployment_id)?.clone(); if deployment_id == runtime.local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &runtime.local_peer.endpoint) { - continue; + return None; } - let transport = match PeerTransport::for_runtime_peer(peer).await { - Ok(transport) => transport, - Err(err) => { - // Record the attempt so backoff advances for an unreachable - // peer instead of re-dialing it every tick. - for event in &peer_events { - enqueue_site_replication_retry_event(peer, &event.path, &err).await; + let runtime = Arc::clone(&runtime); + let plan = plan.as_ref().map(Arc::clone); + Some(async move { + let mut settled = 0usize; + let mut failures = 0usize; + let transport = match PeerTransport::for_runtime_peer(&peer).await { + Ok(transport) => transport, + Err(err) => { + // Record the attempt so backoff advances for an unreachable + // peer instead of re-dialing it every tick. + for event in &peer_events { + enqueue_site_replication_retry_event(&peer, &event.path, &err).await; + } + return (0, peer_events.len()); } - failures += peer_events.len(); - continue; - } - }; - for event in peer_events { - let Some(action) = classify_site_replication_retry_event(&event) else { - continue; }; - match drain_one_site_replication_retry_event(&runtime, peer, &transport, &event, action, plan.as_ref()).await { - Ok(true) => settled += 1, - Ok(false) => {} - Err(_) => failures += 1, + for event in peer_events { + let Some(action) = classify_site_replication_retry_event(&event) else { + continue; + }; + match drain_one_site_replication_retry_event(&runtime, &peer, &transport, &event, action, plan.as_deref()).await { + Ok(true) => settled += 1, + Ok(false) => {} + Err(_) => failures += 1, + } } - } + (settled, failures) + }) + }); + + let mut settled = 0usize; + let mut failures = 0usize; + let replay_results = stream::iter(peer_replays) + .buffer_unordered(SITE_REPLICATION_RETRY_DRAIN_PEER_CONCURRENCY) + .collect::>() + .await; + for (peer_settled, peer_failures) in replay_results { + settled += peer_settled; + failures += peer_failures; } if settled > 0 || failures > 0 { @@ -1292,12 +1809,21 @@ pub(crate) async fn drain_one_site_replication_retry_event( drop_corrupt_iam_deletion_replay(peer, &record.id).await; continue; }; - if let Err(err) = SiteReplicationRepairTask::Iam(&item) - .send(transport, access_key, secret_key) - .await + match send_retry_task_if_peer_current( + peer, + &SiteReplicationRepairTask::Iam(&item), + transport, + access_key, + secret_key, + ) + .await { - enqueue_site_replication_retry_event(peer, &event.path, &err).await; - return Err(err); + Ok(true) => {} + Ok(false) => return Ok(false), + Err(err) => { + enqueue_site_replication_retry_event(peer, &event.path, &err).await; + return Err(err); + } } replayed_record_ids.push(record.id.clone()); } @@ -1306,22 +1832,20 @@ pub(crate) async fn drain_one_site_replication_retry_event( let mut replay = current_snapshot.clone(); for _ in 0..SITE_REPLICATION_RETRY_SNAPSHOT_STABILITY_ATTEMPTS { let current_fingerprint = current_snapshot.fingerprint()?; - if let Err(err) = replay.send(transport, access_key, secret_key).await { - enqueue_site_replication_retry_event(peer, &event.path, &err).await; - return Err(err); + match replay.send(peer, transport, access_key, secret_key).await { + Ok(true) => {} + Ok(false) => return Ok(false), + Err(err) => { + enqueue_site_replication_retry_event(peer, &event.path, &err).await; + return Err(err); + } } let fresh_info = build_sr_info(&runtime.state, &runtime.local_peer).await?; let fresh_plan = site_replication_bootstrap_plan(&fresh_info)?; let fresh_snapshot = RetrySnapshot::from_plan(&action, &fresh_plan).expect("snapshot action has a snapshot"); if fresh_snapshot.fingerprint()? == current_fingerprint { if is_iam { - settle_replayed_site_replication_iam_retry_event( - peer, - &event.path, - event.updated_at, - replayed_record_ids, - ) - .await; + settle_replayed_site_replication_iam_retry_event(peer, event, replayed_record_ids).await; } else { escalate_site_replication_retry_event_up_to(peer, &event.path, event.updated_at).await; } @@ -1339,37 +1863,29 @@ pub(crate) async fn drain_one_site_replication_retry_event( // Replay from the CURRENT plan, never the recorded path: the // recorded query can carry an expired one-shot bootstrap token or // a stale createdAt. - let make_op = operation == SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING; - let paths = if make_op { - &plan.bucket_make_ops - } else { - &plan.bucket_configure_ops - }; - let tasks: Vec> = paths - .iter() - .filter(|path| retry_bucket_name(path).as_deref() == Some(bucket.as_str())) - .map(|path| { - if make_op { - SiteReplicationRepairTask::BucketMake(path) - } else { - SiteReplicationRepairTask::Replication(path) - } - }) - .collect(); - if tasks.is_empty() { - // The bucket left the plan (deleted, or replication no longer - // configured): the recorded intent is stale, settle it. - dequeue_site_replication_retry_event(peer, &event.path).await; - return Ok(true); - } - for task in &tasks { - if let Err(err) = task.send(transport, access_key, secret_key).await { + let tasks = match bucket_op_retry_replay_tasks(plan, &operation, &bucket) { + Ok(tasks) => tasks, + Err(err) => { enqueue_site_replication_retry_event(peer, &event.path, &err).await; return Err(err); } + }; + if tasks.is_empty() { + // The bucket left the plan (deleted, or replication no longer + // configured): the recorded intent is stale, settle it. + return Ok(dequeue_observed_site_replication_retry_event(peer, event).await); } - dequeue_site_replication_retry_event(peer, &event.path).await; - Ok(true) + for task in &tasks { + match send_retry_task_if_peer_current(peer, task, transport, access_key, secret_key).await { + Ok(true) => {} + Ok(false) => return Ok(false), + Err(err) => { + enqueue_site_replication_retry_event(peer, &event.path, &err).await; + return Err(err); + } + } + } + Ok(dequeue_observed_site_replication_retry_event(peer, event).await) } RetryDrainAction::PeerEdit => { // The recorded generation is stale by definition — the receiver @@ -1394,19 +1910,22 @@ pub(crate) async fn drain_one_site_replication_retry_event( let edit_path = peer_edit_path_with_fence(local_deployment_id, generation); let delivery_fence = local_deployment_id.is_some().then_some(generation); for body in &bodies { - if let Err(err) = PeerAdminRequest::put(&transport.connection, &edit_path, access_key) - .with_client(&transport.client) - .send(secret_key, body) - .await - { - enqueue_site_replication_retry_event_for_generation( - peer, - SITE_REPLICATION_PEER_EDIT_PATH, - &err, - delivery_fence, - ) - .await; - return Err(err); + let body = serde_json::to_value(body).map_err(|err| { + S3Error::with_message(S3ErrorCode::InternalError, format!("serialize retry peer edit failed: {err}")) + })?; + match send_peer_edit_retry_if_peer_current(peer, transport, &edit_path, access_key, secret_key, body).await { + Ok(true) => {} + Ok(false) => return Ok(false), + Err(err) => { + enqueue_site_replication_retry_event_for_generation( + peer, + SITE_REPLICATION_PEER_EDIT_PATH, + &err, + delivery_fence, + ) + .await; + return Err(err); + } } } dequeue_site_replication_retry_event_for_generation(peer, SITE_REPLICATION_PEER_EDIT_PATH, delivery_fence).await; @@ -1422,6 +1941,40 @@ pub(crate) async fn dequeue_site_replication_retry_event(peer: &PeerInfo, path: dequeue_site_replication_retry_event_for_generation(peer, path, None).await } +pub(crate) async fn dequeue_observed_site_replication_retry_event(peer: &PeerInfo, observed: &SiteReplicationRetryEvent) -> bool { + let result = async { + let mut probe = load_site_replication_state().await?; + if settle_observed_site_replication_retry_event(&mut probe.retry_queue, peer, observed) == 0 { + return Ok(false); + } + let peer_owned = peer.clone(); + let observed_owned = observed.clone(); + update_site_replication_state(move |state| { + Ok(settle_observed_site_replication_retry_event(&mut state.retry_queue, &peer_owned, &observed_owned) > 0) + }) + .await + } + .await; + + match result { + Ok(settled) => settled, + Err(err) => { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "retry_event_dequeue_failed", + peer = %peer.endpoint, + deployment_id = %peer.deployment_id, + path = %observed.path, + error = ?err, + "failed to dequeue observed site replication retry event" + ); + false + } + } +} + pub(crate) async fn dequeue_site_replication_retry_event_for_generation(peer: &PeerInfo, path: &str, generation: Option) { let result = async { // Fast path: this sits on every successful hook broadcast, so probe diff --git a/rustfs/src/site_replication/state_lock.rs b/rustfs/src/site_replication/state_lock.rs index 6f9cc7bfd..9eb228882 100644 --- a/rustfs/src/site_replication/state_lock.rs +++ b/rustfs/src/site_replication/state_lock.rs @@ -28,14 +28,18 @@ //! process-local lock must never be reintroduced in front of it as if it //! added protection. All IO inside the closure must use the `*_no_lock` //! config helpers — the locked variants would self-deadlock on the same -//! object lock. Do not perform peer network calls or take other config locks -//! inside the closure. +//! object lock. Write-lock closures must not perform peer network calls or +//! take other config locks. A read-lock closure may carry bounded peer +//! delivery only when the receiver cannot write this state. Peer-edit +//! delivery must run after the read lock is released. //! -//! Lock order: lifecycle -> bucket operation -> repair admission -//! -> state object lock -> per-bucket metadata. +//! Lock order: lifecycle -> bucket-mutation admission -> per-bucket mutation +//! -> bucket operation -> repair admission -> state object lock -> +//! per-bucket metadata. A path may skip levels, but must not acquire an +//! earlier level while holding a later one. -use super::{S3Error, S3ErrorCode, S3Result}; -use crate::storage_api::site_replication::{ECStore, with_config_object_write_lock}; +use super::{S3Error, S3ErrorCode, S3Result, SiteReplicationState, load_site_replication_state_no_lock}; +use crate::storage_api::site_replication::{ECStore, with_config_object_read_lock, with_config_object_write_lock}; use std::sync::Arc; use crate::runtime_sources::current_object_store_handle; @@ -57,6 +61,27 @@ where with_site_replication_state_lock_on(store, operation).await } +/// Hold the distributed state-object read lock while `operation` validates a +/// topology snapshot. The closure may carry a bounded peer delivery only when +/// its receiver cannot write site replication state; peer-edit delivery must +/// run after this lock is released. Topology writers use the matching write +/// lock through [`with_site_replication_state_lock`]. +pub(crate) async fn with_site_replication_state_read_lock(operation: F) -> S3Result +where + T: Send + 'static, + F: FnOnce(SiteReplicationState) -> Fut + Send + 'static, + Fut: std::future::Future> + Send + 'static, +{ + let store = current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?; + let read_store = store.clone(); + with_config_object_read_lock(store, SITE_REPLICATION_STATE_PATH.to_string(), move || async move { + let state = load_site_replication_state_no_lock(read_store).await?; + operation(state).await + }) + .await + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("lock site replication state failed: {e}")))? +} + /// Context-store variant for callers that resolve their store from an /// explicit [`AppContext`] (the service-side reload driven over node RPC). /// diff --git a/rustfs/src/site_replication/tests.rs b/rustfs/src/site_replication/tests.rs index e37869a43..e2ab27bb7 100644 --- a/rustfs/src/site_replication/tests.rs +++ b/rustfs/src/site_replication/tests.rs @@ -33,6 +33,18 @@ use temp_env::with_var; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; +#[test] +fn test_bucket_mutation_lock_path_is_bucket_scoped() { + assert_eq!( + site_replication_bucket_mutation_lock_path("photos"), + "config/site-replication/bucket-mutation/photos.lock" + ); + assert_ne!( + site_replication_bucket_mutation_lock_path("photos"), + site_replication_bucket_mutation_lock_path("videos") + ); +} + fn valid_test_ca_pem(name: &str) -> String { rcgen::generate_simple_self_signed(vec![name.to_string()]) .expect("generate test CA") @@ -381,6 +393,30 @@ async fn peer_clients_do_not_follow_redirects() { assert!(tls_server.await.expect("custom redirect TLS server task")); } +#[tokio::test] +async fn peer_http_error_body_cannot_spoof_an_unreachable_peer() { + let (endpoint, ca_pem, server) = spawn_test_tls_server_with_response( + b"HTTP/1.1 500 Internal Server Error\r\ncontent-length: 27\r\nconnection: close\r\n\r\ndownstream failed (connect)", + ) + .await; + let connection = validate_peer_connection_inner(&endpoint, false, &ca_pem, true).expect("custom CA peer connection"); + let client = + build_custom_site_replication_peer_client(&empty_outbound_tls_state(), &connection).expect("custom CA peer client"); + let err = PeerAdminRequest::post(&connection, SITE_REPLICATION_PEER_DEVNULL_PATH, "access-key") + .with_client(&client) + .send("secret-key", &serde_json::json!({})) + .await + .expect_err("HTTP 500 must fail"); + let detail = err.to_string(); + + assert!(detail.contains("downstream failed (connect)")); + assert!( + !retry_error_indicates_peer_unreachable(&detail), + "an untrusted response body must not enable the fast reachability probe" + ); + assert!(server.await.expect("HTTP error TLS server task")); +} + fn peer(name: &str, endpoint: &str) -> PeerInfo { PeerInfo { name: name.to_string(), @@ -419,6 +455,7 @@ fn drain_event(peer: &str, path: &str, retry_count: u32, updated_at: Option = state.iam_deletion_replays.iter().map(|record| record.id.clone()).collect(); - assert!(settle_replayed_iam_retry_events( - &mut state, - &target, - SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH, - Some(snapshot_at), - &replayed, - )); + assert!(settle_replayed_iam_retry_events(&mut state, &target, &observed, &replayed)); assert!(state.retry_queue.is_empty()); assert!(state.iam_deletion_replays.is_empty()); // Not fully recorded: replayed records are still removed, but the entry // escalates instead of settling. let mut state = deletion_replay_state(&target); - record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline"); + record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline").expect("record failure"); state.retry_queue[0].updated_at = Some(snapshot_at); state.retry_queue[0].deletions_recorded = false; + let observed = state.retry_queue[0].clone(); let replayed: Vec = state.iam_deletion_replays.iter().map(|record| record.id.clone()).collect(); - assert!(!settle_replayed_iam_retry_events( - &mut state, - &target, - SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH, - Some(snapshot_at), - &replayed, - )); + assert!(!settle_replayed_iam_retry_events(&mut state, &target, &observed, &replayed)); assert!(state.iam_deletion_replays.is_empty()); assert_eq!(state.retry_queue.len(), 1); assert_eq!(state.retry_queue[0].last_error, SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER); @@ -654,17 +683,13 @@ fn test_settle_replayed_iam_retry_events_settles_or_escalates() { // Newer failure since the snapshot: entry untouched and drain-eligible, // residual (unreplayed) record kept for the next pass. let mut state = deletion_replay_state(&target); - record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline"); + record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline").expect("record failure"); + state.retry_queue[0].updated_at = Some(snapshot_at); + let observed = state.retry_queue[0].clone(); let replayed: Vec = state.iam_deletion_replays.iter().map(|record| record.id.clone()).collect(); - record_failed_iam_delivery(&mut state, &target, &user_delete_item("bob"), "peer offline"); + record_failed_iam_delivery(&mut state, &target, &user_delete_item("bob"), "peer offline").expect("record failure"); state.retry_queue[0].updated_at = Some(snapshot_at + time::Duration::seconds(5)); - assert!(!settle_replayed_iam_retry_events( - &mut state, - &target, - SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH, - Some(snapshot_at), - &replayed, - )); + assert!(!settle_replayed_iam_retry_events(&mut state, &target, &observed, &replayed)); assert_eq!(state.retry_queue.len(), 1); assert_ne!(state.retry_queue[0].last_error, SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER); assert!( @@ -673,6 +698,24 @@ fn test_settle_replayed_iam_retry_events_settles_or_escalates() { ); assert_eq!(state.iam_deletion_replays.len(), 1); assert_eq!(state.iam_deletion_replays[0].entity, "iam-user:bob"); + + // A newer deletion of the same entity gets a fresh replay-record id. An + // older settlement therefore removes neither its body nor its queue + // revision, even if the persisted timestamps happen to be equal. + let mut state = deletion_replay_state(&target); + record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "first failure").expect("record failure"); + state.retry_queue[0].updated_at = Some(snapshot_at); + let observed = state.retry_queue[0].clone(); + let replayed = vec![state.iam_deletion_replays[0].id.clone()]; + record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "newer failure").expect("record newer failure"); + state.retry_queue[0].updated_at = Some(snapshot_at); + assert_ne!(state.iam_deletion_replays[0].id, replayed[0]); + + assert!(!settle_replayed_iam_retry_events(&mut state, &target, &observed, &replayed)); + assert_eq!(state.retry_queue.len(), 1); + assert_ne!(state.retry_queue[0].id, observed.id); + assert_eq!(state.iam_deletion_replays.len(), 1); + assert_eq!(state.iam_deletion_replays[0].entity, "iam-user:alice"); } /// Merging legacy wire-path rows into the collapsed entry must not launder an @@ -748,6 +791,281 @@ fn test_classify_site_replication_retry_event_actions() { assert_eq!(classify("/rustfs/admin/v3/site-replication/peer/unknown"), None); } +#[test] +fn test_bucket_make_retry_replays_matching_configure_before_settlement() { + let make_photos = + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning".to_string(); + let configure_photos = + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=configure-replication".to_string(); + let plan = SiteReplicationBootstrapPlan { + bucket_make_ops: vec![ + make_photos.clone(), + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=videos&operation=make-with-versioning".to_string(), + ], + bucket_configure_ops: vec![ + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=videos&operation=configure-replication".to_string(), + configure_photos.clone(), + ], + bucket_items: vec![ + SRBucketMeta { + bucket: "videos".to_string(), + r#type: "tags".to_string(), + ..Default::default() + }, + SRBucketMeta { + bucket: "photos".to_string(), + r#type: "policy".to_string(), + ..Default::default() + }, + ], + ..Default::default() + }; + + let tasks = bucket_op_retry_replay_tasks(&plan, SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING, "photos") + .expect("make retry plan should include its configure follow-up"); + assert_eq!( + tasks.iter().map(SiteReplicationRepairTask::path).collect::>(), + vec![ + make_photos.as_str(), + "/rustfs/admin/v3/site-replication/peer/bucket-meta", + configure_photos.as_str() + ] + ); + assert!(matches!(tasks[0], SiteReplicationRepairTask::BucketMake(_))); + assert!(matches!(&tasks[1], SiteReplicationRepairTask::BucketMetadata(item) if item.bucket == "photos")); + assert!(matches!(tasks[2], SiteReplicationRepairTask::Replication(_))); +} + +#[test] +fn test_bucket_make_retry_without_matching_configure_fails_closed() { + let plan = SiteReplicationBootstrapPlan { + bucket_make_ops: vec![ + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning".to_string(), + ], + ..Default::default() + }; + + let err = match bucket_op_retry_replay_tasks(&plan, SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING, "photos") { + Ok(_) => panic!("make retry must not settle without a matching configure operation"), + Err(err) => err, + }; + assert_eq!(err.code(), &S3ErrorCode::InternalError); +} + +#[test] +fn test_retry_drain_bounds_each_peer_round_to_one_small_request_chain() { + let plan = SiteReplicationBootstrapPlan { + iam_items: vec![SRIAMItem::default(); 3], + bucket_make_ops: vec![ + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning".to_string(), + ], + bucket_configure_ops: vec![ + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=configure-replication".to_string(), + ], + bucket_items: vec![SRBucketMeta { + bucket: "photos".to_string(), + r#type: "tags".to_string(), + ..Default::default() + }], + ..Default::default() + }; + let make = RetryDrainAction::BucketOpReplay { + operation: SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING.to_string(), + bucket: "photos".to_string(), + }; + + assert!(is_lightweight_retry_drain_action(&make)); + assert!(!is_lightweight_retry_drain_action(&RetryDrainAction::IamSnapshot)); + assert!(!is_lightweight_retry_drain_action(&RetryDrainAction::PeerEdit)); + assert_eq!(retry_drain_request_count(&make, Some(&plan)), 3); + assert!(retry_drain_request_count(&make, Some(&plan)) <= SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER); + assert!( + retry_drain_request_count(&RetryDrainAction::IamSnapshot, Some(&plan)) + > SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER + ); + assert!( + retry_drain_request_count(&RetryDrainAction::PeerEdit, Some(&plan)) > SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER + ); +} + +#[test] +fn test_lightweight_retry_peer_rotation_covers_all_queued_peers() { + let limit = SITE_REPLICATION_RETRY_DRAIN_PEER_CONCURRENCY; + for peer_count in 1..=(limit * 3 + 1) { + let rounds = peer_count.div_ceil(limit); + let mut seen = HashSet::new(); + for round in 7..(7 + rounds as i64) { + let start = lightweight_retry_peer_rotation(peer_count, round); + for offset in 0..limit.min(peer_count) { + seen.insert((start + offset) % peer_count); + } + } + assert_eq!( + seen.len(), + peer_count, + "every peer must enter the bounded lightweight window within {rounds} rounds" + ); + } +} + +#[test] +fn test_lightweight_bucket_retry_plan_is_targeted_and_preserves_make_options() { + let created_at = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let bucket = SRBucketInfo { + bucket: "photos".to_string(), + created_at: Some(created_at), + object_lock_config: Some(String::new()), + tags: Some("dGFncy14bWw=".to_string()), + tag_config_updated_at: Some(created_at), + ..Default::default() + }; + let plan = site_replication_bucket_retry_plan_for(&bucket, false).expect("targeted retry plan"); + + assert!(plan.iam_items.is_empty()); + assert_eq!(plan.bucket_make_ops.len(), 1); + assert!(plan.bucket_make_ops[0].contains("bucket=photos")); + assert!(plan.bucket_make_ops[0].contains("lockEnabled=true")); + assert!(plan.bucket_make_ops[0].contains("createdAt=")); + assert_eq!(plan.bucket_items.len(), 2); + assert_eq!(plan.bucket_items[0].r#type, "tags"); + assert_eq!(plan.bucket_items[1].r#type, "object-lock-config"); + assert_eq!(plan.bucket_configure_ops.len(), 1); + assert!(plan.bucket_configure_ops[0].contains("operation=configure-replication")); + + let tasks = + bucket_op_retry_replay_tasks(&plan, SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING, "photos").expect("retry task chain"); + assert!(matches!(tasks[0], SiteReplicationRepairTask::BucketMake(_))); + assert!(matches!(tasks[1], SiteReplicationRepairTask::BucketMetadata(_))); + assert!(matches!(tasks[2], SiteReplicationRepairTask::BucketMetadata(_))); + assert!(matches!(tasks[3], SiteReplicationRepairTask::Replication(_))); +} + +#[test] +fn test_lightweight_bucket_retry_plan_orders_real_metadata_and_counts_it() { + let versioning = bucket_versioning_xml().expect("canonical versioning config"); + let replication = serialize(&site_repl_config("remote-dep")).expect("derived replication config"); + let bucket = SRBucketInfo { + bucket: "photos".to_string(), + policy: Some(serde_json::json!({"Version":"2012-10-17","Statement":[]})), + tags: Some(BASE64_STANDARD.encode_to_string("")), + versioning: Some(BASE64_STANDARD.encode_to_string(&versioning)), + replication_config: Some(BASE64_STANDARD.encode_to_string(&replication)), + ..Default::default() + }; + let plan = site_replication_bucket_retry_plan_from_info(&bucket, false).expect("targeted retry plan"); + let tasks = bucket_op_retry_replay_tasks(&plan, SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING, "photos") + .expect("bucket replay tasks"); + + assert!(matches!(tasks.first(), Some(SiteReplicationRepairTask::BucketMake(_)))); + assert!(matches!(tasks.last(), Some(SiteReplicationRepairTask::Replication(_)))); + assert!( + tasks[1..tasks.len() - 1] + .iter() + .all(|task| matches!(task, SiteReplicationRepairTask::BucketMetadata(_))) + ); + assert_eq!(tasks.len(), 4, "make + policy + tags + configure must all count against the budget"); + assert!( + tasks.len() <= SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER, + "the complete metadata chain must fit the bounded lightweight replay" + ); + + let mut operator_replication = site_repl_config("remote-dep"); + operator_replication.rules.push(operator_rule("operator-backup")); + let mut bucket_with_operator_rule = bucket; + bucket_with_operator_rule.replication_config = + Some(BASE64_STANDARD.encode_to_string(&serialize(&operator_replication).expect("operator replication config"))); + let plan = site_replication_bucket_retry_plan_from_info(&bucket_with_operator_rule, false).expect("targeted retry plan"); + assert!( + plan.bucket_items.iter().any(|item| item.r#type == "replication-config"), + "operator-authored replication rules cannot be replaced by configure-replication" + ); +} + +#[test] +fn test_delete_bucket_broadcast_fences_target_membership_through_delivery() { + let hooks = include_str!("hooks.rs"); + let delete_broadcast = hooks + .split("async fn broadcast_site_replication_delete_bucket") + .nth(1) + .and_then(|rest| rest.split("pub(crate) async fn commit_site_replication_delete_bucket").next()) + .expect("delete-bucket broadcast should exist"); + assert!( + delete_broadcast.contains("with_site_replication_state_read_lock(move |state| async move {") + && delete_broadcast.contains("state.peers.get(&fallback_peer.deployment_id)") + && delete_broadcast.contains("site_replicator_service_account_secret(&state.service_account_access_key)") + && delete_broadcast + .contains("PeerAdminRequest::put(&transport.connection, &delivery_path, &state.service_account_access_key)"), + "a destructive bucket delivery must resolve current topology and credentials under the distributed state read lock" + ); + assert!( + delete_broadcast.contains("enqueue_site_replication_retry_event(¤t_peer, &request_path, &err).await"), + "a failed destructive delivery must remain visible for operator repair" + ); + + let usecase = include_str!("../app/bucket_usecase.rs"); + let delete = usecase + .split("async fn execute_delete_bucket_inner") + .nth(1) + .and_then(|rest| rest.split("pub async fn execute_head_bucket").next()) + .expect("delete bucket usecase"); + assert!( + delete + .find("prepare_site_replication_delete_bucket") + .expect("durable reservation") + < delete.find(".delete_bucket(").expect("local delete"), + "destructive peer liabilities must be persisted before the local bucket is deleted" + ); +} + +#[test] +fn test_bucket_retry_settlement_preserves_a_newer_same_path_failure() { + let peer = peer("remote", "https://remote.example.com"); + let path = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=configure-replication"; + let observed_at = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let observed = drain_event("remote", path, 1, Some(observed_at)); + let mut queue = vec![observed.clone()]; + + queue[0].id = "evt-remote-new-revision".to_string(); + queue[0].retry_count += 1; + assert_eq!(settle_observed_site_replication_retry_event(&mut queue, &peer, &observed), 0); + assert_eq!(queue.len(), 1, "a newer same-timestamp failure must survive stale settlement"); + + let current = queue[0].clone(); + assert_eq!(settle_observed_site_replication_retry_event(&mut queue, &peer, ¤t), 1); + assert!(queue.is_empty()); +} + +#[test] +fn test_reachable_probe_promotion_is_fenced_by_the_observed_event() { + let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let path = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning"; + let mut event = drain_event("remote", path, 3, Some(now)); + event.peer_unreachable = true; + let recovered = event.clone(); + let mut state = SiteReplicationState { + retry_queue: vec![event], + ..Default::default() + }; + state + .peers + .insert("remote".to_string(), peer("remote", "https://remote.example.com")); + + assert_eq!(mark_reachable_deferred_retry_events(&mut state, &[recovered.clone()]), 1); + assert_eq!(state.retry_queue[0].updated_at, None); + assert!(!state.retry_queue[0].peer_unreachable); + assert_eq!( + actionable_site_replication_retry_events(&state, now).len(), + 1, + "a successful probe must make the event replayable in the same drain tick" + ); + + state.retry_queue[0].updated_at = Some(now + time::Duration::seconds(1)); + state.retry_queue[0].peer_unreachable = true; + assert_eq!(mark_reachable_deferred_retry_events(&mut state, &[recovered]), 0); + assert_eq!(state.retry_queue[0].updated_at, Some(now + time::Duration::seconds(1))); + assert!(state.retry_queue[0].peer_unreachable); +} + #[test] fn test_retry_snapshot_fingerprint_detects_concurrent_iam_change() { let old = SRIAMItem { @@ -828,6 +1146,100 @@ fn test_site_replication_retry_backoff_schedule() { assert!(elapsed(30, 86_401)); } +#[test] +fn test_retry_error_marks_peer_unreachable_only_for_connection_failures() { + let mut queue = Vec::new(); + let peer = peer("remote", "https://remote.example.com"); + let bucket_make = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning"; + + upsert_site_replication_retry_event( + &mut queue, + &peer, + bucket_make, + "peer request to https://remote.example.com failed (connect): connection refused", + None, + ) + .expect("upsert retry event"); + assert!(queue[0].peer_unreachable); + + upsert_site_replication_retry_event( + &mut queue, + &peer, + bucket_make, + "peer request to https://remote.example.com failed (timeout): request exceeded 10 seconds", + None, + ) + .expect("upsert retry event"); + assert!( + !queue[0].peer_unreachable, + "a whole-request timeout does not prove the peer is unreachable" + ); + + upsert_site_replication_retry_event( + &mut queue, + &peer, + bucket_make, + "peer request to https://remote.example.com failed with 500 Internal Server Error: downstream failed (connect)", + None, + ) + .expect("upsert retry event"); + assert!( + !queue[0].peer_unreachable, + "application failures and their untrusted bodies must keep the normal replay backoff" + ); + + upsert_site_replication_retry_event( + &mut queue, + &peer, + bucket_make, + "peer request to https://remote.example.com failed with 500 Internal Server Error: backend failed (connect): spoofed", + None, + ) + .expect("upsert retry event"); + assert!(!queue[0].peer_unreachable, "peer response bodies must not spoof transport failures"); +} + +#[test] +fn test_connect_timeout_is_classified_as_a_connection_failure() { + assert_eq!(classify_peer_transport_error(true, true, "tcp connect timed out"), "connect"); + assert_eq!(classify_peer_transport_error(false, true, "request timed out"), "timeout"); + assert_eq!( + classify_peer_transport_error(false, true, "request timed out for https://tls-gateway.example"), + "timeout" + ); + assert_eq!(classify_peer_transport_error(true, false, "tls handshake failed"), "tls handshake"); +} + +#[test] +fn test_retry_event_peer_unreachable_is_legacy_serde_default() { + let json = r#"{ + "id":"evt-legacy", + "peer_deployment_id":"remote", + "peer_endpoint":"https://remote.example.com", + "path":"/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning", + "retry_count":1, + "failed":false, + "last_error":"peer request to https://remote.example.com failed (connect): connection refused" + }"#; + + let mut event: SiteReplicationRetryEvent = serde_json::from_str(json).expect("legacy retry event decodes"); + assert!(!event.peer_unreachable); + + let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + event.updated_at = Some(now - time::Duration::seconds(30)); + let mut state = SiteReplicationState::default(); + state + .peers + .insert("remote".to_string(), peer("remote", "https://remote.example.com")); + state.retry_queue.push(event); + + assert_eq!( + deferred_site_replication_retry_events(&state, now).len(), + 1, + "rolling-upgrade records must retain fast recovery from their trusted outer error shape" + ); +} + /// The actionable subset respects classification, peer membership and /// backoff; everything else stays untouched in the queue. #[test] @@ -915,6 +1327,51 @@ fn test_deferred_site_replication_retry_events_partition() { assert_eq!(actionable[0].path, "/rustfs/admin/v3/site-replication/peer/bucket-meta"); } +#[test] +fn test_deferred_retry_events_probe_fresh_peer_transport_failures() { + let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let mut state = SiteReplicationState::default(); + state + .peers + .insert("remote".to_string(), peer("remote", "https://remote.example.com")); + + let bucket_make = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning"; + let mut fresh_transport_failure = drain_event("remote", bucket_make, 1, Some(now - time::Duration::seconds(30))); + fresh_transport_failure.peer_unreachable = true; + state.retry_queue.push(fresh_transport_failure); + + let deferred = deferred_site_replication_retry_events(&state, now); + assert_eq!( + deferred.len(), + 1, + "fresh transport failures must be eligible for a cheap reachability probe" + ); + assert_eq!(deferred[0].path, bucket_make); + + let actionable = actionable_site_replication_retry_events(&state, now); + assert!(actionable.is_empty(), "the event is still protected from direct replay by normal backoff"); +} + +#[test] +fn test_deferred_retry_events_do_not_probe_fresh_application_failures() { + let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let mut state = SiteReplicationState::default(); + state + .peers + .insert("remote".to_string(), peer("remote", "https://remote.example.com")); + + let bucket_make = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning"; + state + .retry_queue + .push(drain_event("remote", bucket_make, 1, Some(now - time::Duration::seconds(30)))); + + assert!( + deferred_site_replication_retry_events(&state, now).is_empty(), + "reachable peers that reject an operation must keep the base replay backoff" + ); + assert!(actionable_site_replication_retry_events(&state, now).is_empty()); +} + /// The drain settles a peer-edit success under a freshly allocated /// generation; legacy queue entries carry `edit_generation: None` and /// must be cleared by that generation-scoped settlement (`(Some, None)` @@ -996,7 +1453,7 @@ fn test_escalate_up_to_marks_snapshot_replayed_and_keeps_newer_failures() { // successful Bob update on the shared wire path cannot erase it even // before the drain runs. let mut queue = Vec::new(); - upsert_site_replication_retry_event(&mut queue, &target, path, "alice delete failed", None); + upsert_site_replication_retry_event(&mut queue, &target, path, "alice delete failed", None).expect("upsert retry event"); assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); assert_eq!(dequeue_site_replication_retry_events(&mut queue, &target, path), 0); assert_eq!(queue.len(), 1); @@ -1005,7 +1462,7 @@ fn test_escalate_up_to_marks_snapshot_replayed_and_keeps_newer_failures() { // A later hook failure overwrites the marker and re-arms the drain. let mut queue = vec![drain_event("remote", path, 2, Some(snapshot_at))]; escalate_site_replication_retry_events_up_to(&mut queue, &target, path, Some(snapshot_at)); - upsert_site_replication_retry_event(&mut queue, &target, path, "peer offline", None); + upsert_site_replication_retry_event(&mut queue, &target, path, "peer offline", None).expect("upsert retry event"); assert!(classify_site_replication_retry_event(&queue[0]).is_some()); // Legacy entry without a timestamp: escalated. @@ -1781,17 +2238,85 @@ fn test_retry_event_upsert_marks_repeated_failures() { }; let mut queue = Vec::new(); - upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "first", None); - upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "second", None); - upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "third", None); + upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "first", None) + .expect("upsert retry event"); + let first_revision = queue[0].id.clone(); + upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "second", None) + .expect("upsert retry event"); + let second_revision = queue[0].id.clone(); + upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "third", None) + .expect("upsert retry event"); assert_eq!(queue.len(), 1); + assert_ne!(first_revision, second_revision); + assert_ne!(second_revision, queue[0].id, "each failure must advance the settlement revision"); assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); assert_eq!(queue[0].retry_count, SITE_REPLICATION_RETRY_FAILED_AFTER); assert!(queue[0].failed); assert_eq!(queue[0].last_error, "third"); } +#[test] +fn retry_queue_capacity_never_evicts_destructive_bucket_liabilities() { + let target = PeerInfo { + deployment_id: "remote-dep".to_string(), + ..peer("remote", "https://remote.example.com") + }; + let destructive = |index: usize| SiteReplicationRetryEvent { + id: format!("delete-{index}"), + peer_deployment_id: target.deployment_id.clone(), + peer_endpoint: target.endpoint.clone(), + path: format!("{SITE_REPLICATION_PEER_BUCKET_OPS_PATH}?bucket=bucket-{index}&operation=delete-bucket"), + ..Default::default() + }; + let mut queue = (0..SITE_REPLICATION_RETRY_QUEUE_LIMIT).map(destructive).collect::>(); + let original_ids = queue.iter().map(|event| event.id.clone()).collect::>(); + let new_path = format!("{SITE_REPLICATION_PEER_BUCKET_OPS_PATH}?bucket=overflow&operation=force-delete-bucket"); + + let err = upsert_site_replication_retry_event(&mut queue, &target, &new_path, "reserve delete", None) + .expect_err("an all-destructive full queue must fail closed"); + assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable); + assert_eq!(queue.len(), SITE_REPLICATION_RETRY_QUEUE_LIMIT); + assert_eq!(queue.iter().map(|event| event.id.clone()).collect::>(), original_ids); + + queue[0] = SiteReplicationRetryEvent { + id: "iam-snapshot".to_string(), + peer_deployment_id: target.deployment_id.clone(), + peer_endpoint: target.endpoint.clone(), + path: SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH.to_string(), + deletions_recorded: true, + ..Default::default() + }; + upsert_site_replication_retry_event(&mut queue, &target, &new_path, "reserve delete", None) + .expect_err("a collapsed IAM liability may contain a deletion and must not be evicted"); + assert!(queue.iter().any(|event| event.id == "iam-snapshot")); + + queue[0] = SiteReplicationRetryEvent { + id: "rebuildable".to_string(), + peer_deployment_id: target.deployment_id.clone(), + peer_endpoint: target.endpoint.clone(), + path: SITE_REPLICATION_PEER_EDIT_PATH.to_string(), + ..Default::default() + }; + let preserved_delete_ids = queue + .iter() + .filter(|event| is_destructive_bucket_retry_path(&event.path)) + .map(|event| event.id.clone()) + .collect::>(); + let evicted = upsert_site_replication_retry_event(&mut queue, &target, &new_path, "reserve delete", None) + .expect("a rebuildable row may make room for a destructive liability"); + + assert_eq!(evicted.len(), 1); + assert_eq!(evicted[0].id, "rebuildable"); + assert_eq!(queue.len(), SITE_REPLICATION_RETRY_QUEUE_LIMIT); + assert!( + preserved_delete_ids + .iter() + .all(|id| queue.iter().any(|event| &event.id == id)) + ); + assert!(queue.iter().any(|event| event.path == new_path)); +} + /// P1-15 review follow-up: a successful peer-edit delivery only proves the /// peer reached the state THAT delivery carried. Settling it must not /// erase a retry event a newer edit left behind, or the local site sits on @@ -1807,7 +2332,8 @@ fn retry_settlement_must_not_erase_a_newer_generation_failure() { // Edit A (generation 5) delivered successfully and is stalled before // settling. Edit B (generation 6) commits meanwhile, fails delivery to // the same peer, and enqueues. - upsert_site_replication_retry_event(&mut queue, &peer, SITE_REPLICATION_PEER_EDIT_PATH, "peer offline", Some(6)); + upsert_site_replication_retry_event(&mut queue, &peer, SITE_REPLICATION_PEER_EDIT_PATH, "peer offline", Some(6)) + .expect("upsert retry event"); // A resumes: its own settlement must leave B's retry alone. assert_eq!( @@ -1818,7 +2344,8 @@ fn retry_settlement_must_not_erase_a_newer_generation_failure() { assert_eq!(queue[0].edit_generation, Some(6)); // An even older delivery failing afterwards must not lower the fence. - upsert_site_replication_retry_event(&mut queue, &peer, SITE_REPLICATION_PEER_EDIT_PATH, "still offline", Some(4)); + upsert_site_replication_retry_event(&mut queue, &peer, SITE_REPLICATION_PEER_EDIT_PATH, "still offline", Some(4)) + .expect("upsert retry event"); assert_eq!(queue[0].edit_generation, Some(6)); // B's own delivery succeeding is what clears it. @@ -1831,7 +2358,7 @@ fn retry_settlement_must_not_erase_a_newer_generation_failure() { // Collapsed broadcast failures live under an internal snapshot path; // an unrelated success on their shared wire path cannot settle them. let iam_path = "/rustfs/admin/v3/site-replication/peer/iam-item"; - upsert_site_replication_retry_event(&mut queue, &peer, iam_path, "peer offline", None); + upsert_site_replication_retry_event(&mut queue, &peer, iam_path, "peer offline", None).expect("upsert retry event"); assert_eq!(dequeue_site_replication_retry_events(&mut queue, &peer, iam_path), 0); assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); } diff --git a/rustfs/src/site_replication/transport.rs b/rustfs/src/site_replication/transport.rs index 30ad28f95..bc941c9f0 100644 --- a/rustfs/src/site_replication/transport.rs +++ b/rustfs/src/site_replication/transport.rs @@ -331,6 +331,7 @@ pub(crate) fn runtime_peer_connection(peer: &PeerInfo) -> S3Result PeerAdminRequest<'a> { } let response = req.send().await.map_err(|e| { - let classify = if e.is_timeout() { - "timeout" - } else if e.is_connect() && e.to_string().to_ascii_lowercase().contains("dns") { - "dns resolution" - } else if e.to_string().to_ascii_lowercase().contains("certificate") - || e.to_string().to_ascii_lowercase().contains("tls") - { - "tls handshake" - } else if e.is_connect() { - "connect" - } else { - "request" - }; + let classify = classify_peer_transport_error(e.is_connect(), e.is_timeout(), &e.to_string()); S3Error::with_message(S3ErrorCode::InternalError, format!("peer request to {url} failed ({classify}): {e}")) })?; @@ -825,6 +814,21 @@ impl<'a> PeerAdminRequest<'a> { } } +pub(crate) fn classify_peer_transport_error(is_connect: bool, is_timeout: bool, detail: &str) -> &'static str { + let detail = detail.to_ascii_lowercase(); + if is_connect && detail.contains("dns") { + "dns resolution" + } else if is_connect && (detail.contains("certificate") || detail.contains("tls")) { + "tls handshake" + } else if is_connect { + "connect" + } else if is_timeout { + "timeout" + } else { + "request" + } +} + pub(crate) fn peer_error_may_be_secret_mismatch(detail: &str) -> bool { let detail = detail.to_ascii_lowercase(); detail.contains("signaturedoesnotmatch") diff --git a/rustfs/src/site_replication_reconcile.rs b/rustfs/src/site_replication_reconcile.rs index e327e3669..b4ea52c93 100644 --- a/rustfs/src/site_replication_reconcile.rs +++ b/rustfs/src/site_replication_reconcile.rs @@ -28,16 +28,19 @@ use std::pin::Pin; use std::sync::OnceLock; use std::time::Duration; +use tokio::time::Instant; use tokio_util::sync::CancellationToken; use tracing::warn; const RECONCILE_INTERVAL: Duration = Duration::from_secs(600); +pub(crate) const RETRY_DRAIN_INTERVAL: Duration = Duration::from_secs(30); /// A reconciler reports its own failures; the outcome carries no value because neither /// caller can act on one — a site that cannot repair its replication wiring still serves S3. type ReconcileHook = fn() -> Pin + Send>>; static RECONCILER: OnceLock = OnceLock::new(); +static RETRY_DRAINER: OnceLock = OnceLock::new(); /// Install the admin layer's reconciler. Idempotent: a second call is ignored, which keeps /// repeated router construction (tests, the embedded server) from panicking. @@ -45,6 +48,12 @@ pub(crate) fn register_site_replication_reconciler(reconcile: ReconcileHook) { let _ = RECONCILER.set(reconcile); } +/// Install the admin layer's lightweight retry drain. Idempotent for the same +/// reason as [`register_site_replication_reconciler`]. +pub(crate) fn register_site_replication_retry_drainer(drain: ReconcileHook) { + let _ = RETRY_DRAINER.set(drain); +} + /// Repair drifted site-replication wiring, immediately and then on a timer. /// /// The first pass runs inside the spawned task rather than on the caller's path: it walks @@ -62,16 +71,38 @@ pub(crate) fn spawn_site_replication_reconcile_task(ctx: CancellationToken) { return; } + spawn_reconcile_loop(ctx.clone(), RECONCILE_INTERVAL, &RECONCILER, true); + + if RETRY_DRAINER.get().is_none() { + warn!("site replication retry drainer is not registered; periodic retry drain disabled"); + return; + } + spawn_reconcile_loop(ctx, RETRY_DRAIN_INTERVAL, &RETRY_DRAINER, false); +} + +fn spawn_reconcile_loop( + ctx: CancellationToken, + interval: Duration, + hook: &'static OnceLock, + run_immediately: bool, +) { tokio::spawn(async move { - let mut ticker = tokio::time::interval(RECONCILE_INTERVAL); + let first_tick = if run_immediately { + Instant::now() + } else { + Instant::now() + interval + }; + let mut ticker = tokio::time::interval_at(first_tick, interval); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { tokio::select! { _ = ctx.cancelled() => break, - // The first tick fires immediately, which is the startup repair pass. + // The heavy reconciler owns the startup repair pass. The lightweight + // retry drain starts on its normal cadence so it cannot steal that + // first lifecycle lock and defer bucket/IAM repair for a full interval. _ = ticker.tick() => { - if let Some(reconcile) = RECONCILER.get() { + if let Some(reconcile) = hook.get() { reconcile().await; } } @@ -79,3 +110,14 @@ pub(crate) fn spawn_site_replication_reconcile_task(ctx: CancellationToken) { } }); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn retry_drain_runs_faster_than_heavy_reconcile() { + assert!(RETRY_DRAIN_INTERVAL < RECONCILE_INTERVAL); + assert!(RETRY_DRAIN_INTERVAL <= Duration::from_secs(60)); + } +} diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index e74e88d02..db459b72d 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -244,8 +244,8 @@ pub(crate) mod site_replication { pub(crate) use crate::storage::storage_api::{Endpoint, Endpoints, PoolEndpoints}; pub(crate) use crate::storage::storage_api::{ - ECStore, EndpointServerPools, StorageError, delete_config_no_lock, lock_bucket_targets_metadata, read_config, - read_config_no_lock, save_config_no_lock, with_config_object_read_lock, with_config_object_write_lock, + ECStore, EndpointServerPools, StorageError, delete_config_no_lock, is_err_bucket_not_found, lock_bucket_targets_metadata, + read_config, read_config_no_lock, save_config_no_lock, with_config_object_read_lock, with_config_object_write_lock, }; pub(crate) mod metadata_sys { From 8ae8fb7eea00d92acc771f9efb25129925217055 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 16:16:53 +0800 Subject: [PATCH 04/12] fix(ecstore): drain control writes and preserve uncertain rollback (#7163) * fix(ecstore): drain durable control-plane write tails * fix(ecstore): retain PUT staging after incomplete rollback * fix(ecstore): drain backfill checkpoint before confirmation * fix(ecstore): retain per-disk rename rollback outcomes * fix(ecstore): retain indeterminate rename recovery evidence * test(ecstore): mark rollback fixtures as inline data * test(ecstore): match sealed context fixture map type * fix(ecstore): preserve known preflight rename rejections * test(ecstore): cover observed rename outer failures * test(ecstore): count decommission faults across retry restarts --- crates/ecstore/src/disk/local.rs | 997 +++++++++++++++++++++++++++++++ 1 file changed, 997 insertions(+) diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 9cd683578..bf2239a9f 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -9862,7 +9862,1004 @@ fn should_read_legacy_inline_part(fi: &FileInfo, storage_class_config: &crate::c storage_class_config.should_inline(shard_size, fi.erasure.data_blocks, versioned) } +/// Proof produced only when the local rename returns at an existing access +/// preflight, before metadata, backups, or object data can be published. +#[derive(Debug)] +pub(in crate::disk) struct LocalRenamePreflightRejection(()); + impl LocalDisk { + #[tracing::instrument(name = "rename_data", level = "trace", skip_all)] + async fn rename_data_inner( + &self, + src_volume: &str, + src_path: &str, + fi: FileInfo, + dst_volume: &str, + dst_path: &str, + preflight_rejection: &mut Option, + ) -> Result { + crate::hp_guard!("LocalDisk::rename_data"); + let mut fi = fi; + // A non-force DeleteBucket must not remove a directory while a local + // object commit is publishing into it. The peer's empty scan remains + // optimistic; this lease establishes the local commit/delete order and + // remains owned by any blocking syscall that outlives async cancellation. + let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?; + let quota_fence_token = + match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) { + Some(value) => { + let token = Uuid::parse_str(value).map_err(|_| DiskError::FileCorrupt)?; + Some(SnapshotLeaseToken::from_slice(token.as_bytes())?) + } + None if rustfs_utils::http::metadata_compat::contains_key_str( + &fi.metadata, + QUOTA_MUTATION_FENCE_METADATA_SUFFIX, + ) => + { + return Err(DiskError::FileCorrupt); + } + None => None, + }; + rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX); + let quota_fence_claim = match quota_fence_token { + Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?), + None => None, + }; + let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await; + if let Some(claim) = quota_fence_claim { + mutation_lease.attach_external_guard(claim); + } + if fi.is_legacy_indexed_delete_marker() { + fi.erasure.index = 0; + } + fi.validate_for_metadata_read()?; + // Snapshot the destination part paths before `fi` is consumed below. These + // are the descriptors a reader may hold for the version this call is about + // to replace (backlog#1145); readers build the identical string in + // `io_primitives`. An inline-data version has no parts and yields none. + let invalidate_part_paths: Vec = { + let data_dir = fi.data_dir.unwrap_or_default(); + fi.parts + .iter() + .map(|part| format!("{dst_path}/{data_dir}/part.{}", part.number)) + .collect() + }; + let src_volume_dir = self.io_get_bucket_path(src_volume)?; + if !skip_access_checks(src_volume) + && let Err(e) = super::fs::access_std(&src_volume_dir) + { + info!( + event = EVENT_DISK_LOCAL_ACCESS_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + path = ?src_volume_dir, + operation = "rename_data_src_access", + error = %e, + "Disk local access check failed" + ); + *preflight_rejection = Some(LocalRenamePreflightRejection(())); + return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); + } + + let dst_volume_dir = self.io_get_bucket_path(dst_volume)?; + if !skip_access_checks(dst_volume) + && let Err(e) = super::fs::access_std(&dst_volume_dir) + { + info!( + event = EVENT_DISK_LOCAL_ACCESS_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + path = ?dst_volume_dir, + operation = "rename_data_dst_access", + error = %e, + "Disk local access check failed" + ); + *preflight_rejection = Some(LocalRenamePreflightRejection(())); + return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); + } + + // xl.meta path + let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?; + let dst_file_path = self.io_get_object_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?; + + // data_dir path + let has_data_dir_path = { + let has_data_dir = { + if !fi.is_remote() { + fi.data_dir + .map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str())) + } else { + None + } + }; + + if let Some(data_dir) = has_data_dir { + let src_data_path = self.io_get_object_path( + src_volume, + rustfs_utils::path::retain_slash(format!("{}/{}", src_path, data_dir).as_str()).as_str(), + )?; + let dst_data_path = self.io_get_object_path( + dst_volume, + rustfs_utils::path::retain_slash(format!("{}/{}", dst_path, data_dir).as_str()).as_str(), + )?; + + Some((src_data_path, dst_data_path)) + } else { + None + } + }; + + check_path_length(src_file_path.to_string_lossy().to_string().as_str())?; + check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?; + + let no_inline = fi.data.is_none() && fi.size > 0; + // Captured before `fi` is consumed by add_version; gates the stale + // destination purge below. + let fi_healing = fi.is_healing(); + + // Resolved once for the whole commit so a concurrent configuration + // change can never leave a single rename_data half-synced. The tier is + // keyed on the destination volume: user data staged in scratch + // namespaces follows the configured tier, while commits into + // system-critical namespaces (IAM, config, bucket metadata) stay + // pinned to strict. + let durability = effective_durability(dst_volume); + + let src_file_parent = src_file_path + .parent() + .ok_or_else(|| DiskError::other("missing staged metadata parent"))?; + let dst_file_parent = dst_file_path + .parent() + .ok_or_else(|| DiskError::other("missing object metadata parent"))?; + if !no_inline { + fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?; + } + // Acquire the common trees before reading destination metadata. On + // Windows this pins the object directory identity across metadata + // preparation, data publication, rollback backup, and final commit. + let rename_commit_guard = lock_rename_commit_directories( + src_file_parent, + dst_file_parent, + &dst_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + let has_dst_buf = read_rename_destination_metadata(&dst_file_path, &rename_commit_guard, mutation_lease.clone()).await?; + + if no_inline { + // Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta + let mut xlmeta = FileMeta::new(); + // An existing dst xl.meta that fails to parse leaves `xlmeta` empty + // and gets overwritten by the commit below (pre-existing behavior); + // track that so the old-size observation reports unknown instead of + // a false `Absent` (rustfs/backlog#1009). + let mut dst_meta_unparsable = false; + if let Some(dst_buf) = has_dst_buf.as_ref() { + if FileMeta::is_xl2_v1_format(dst_buf) + && let Ok(nmeta) = FileMeta::load(dst_buf) + { + xlmeta = nmeta + } else { + dst_meta_unparsable = true; + } + } + + let old_current_size = if dst_meta_unparsable { + None + } else { + observe_old_current_size(has_dst_buf.is_some(), &xlmeta) + }; + + let mut skip_parent = dst_volume_dir.clone(); + if has_dst_buf.as_ref().is_some() + && let Some(parent) = dst_file_path.parent() + { + skip_parent = parent.to_path_buf(); + } + + let version_id = fi.version_id.unwrap_or_default(); + let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); + let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); + let rollback_data_dir = has_old_data_dir.or_else(|| { + if old_version_exists && has_dst_buf.is_some() { + Some(inline_metadata_rollback_dir(version_id, &xlmeta)) + } else { + None + } + }); + if let Some(old_data_dir) = has_old_data_dir.as_ref() { + let _ = xlmeta.data.remove_two(version_id, *old_data_dir); + } + xlmeta.add_version(fi)?; + let version_signature = rename_data_versions_signature(&xlmeta); + let new_dst_buf = xlmeta.marshal_msg()?; + + // This tmp xl.meta is renamed onto dst_file_path at the commit + // point below, so only its contents must be durable before the + // rename (SyncMode::FileOnly); the dst parent directory is fsynced + // after the commit rename, and a crash before the rename means the + // PUT was never acknowledged. A metadata commit: relaxed tiers + // leave it to the page cache. + let tmp_meta_sync = if durability.syncs_commit_metadata() { + SyncMode::FileOnly + } else { + SyncMode::None + }; + // The tmp xl.meta write and the shard-file fdatasync are independent + // (disjoint paths) and both only need to be durable before the commit + // renames below, so run them concurrently to drop a blocking + // round-trip from the PUT commit critical path (rustfs/backlog#922 + // step 2). The "contents durable -> rename -> dst dir fsync" ordering + // is unchanged — both futures complete before any rename — which the + // rename_data crash-consistency harness (backlog#935) exercises. + // + // Shard durability: once rename_data succeeds the write is + // acknowledged, so data must not live only in the page cache. + // Multipart parts were already synced during rename_part, so their + // fdatasync here is a cheap no-op. A missing source dir is left for the + // rename below to report through the existing rollback path. Payload + // durability is kept by both strict and relaxed. + let tmp_meta_write = { + let src_file_path = src_file_path.clone(); + let dst_file_path = dst_file_path.clone(); + let rename_commit_guard = rename_commit_guard.clone(); + let mutation_lease = mutation_lease.clone(); + async move { + os::run_blocking_namespace_operation(mutation_lease, move || { + #[cfg(test)] + run_owned_file_write_before_open(&src_file_path); + let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard( + &src_file_path, + &dst_file_path, + &rename_commit_guard, + )?; + prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?; + Ok(prepared_metadata_source) + }) + .await + .map_err(to_file_error) + .map_err(DiskError::from) + } + }; + let shard_sync = async { + if durability.syncs_data_shards() + && let Some((src_data_path, _)) = has_data_dir_path.as_ref() + && let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await + && err.kind() != ErrorKind::NotFound + { + return Err::<(), DiskError>(to_file_error(err).into()); + } + Ok(()) + }; + let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync); + // Surface a tmp-meta failure first (its prior serial position), then a + // shard-sync failure; either aborts before any rename, exactly as the + // sequential version did. + let prepared_metadata_source = tmp_meta_res?; + shard_sync_res?; + let rename_commit_guard = remove_dst_base_before_commit( + dst_path, + rename_commit_guard, + src_file_parent, + dst_file_parent, + &dst_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + if should_remove_staged_meta_before_commit(dst_path) { + drop(prepared_metadata_source); + std::fs::remove_file(&src_file_path).map_err(to_file_error)?; + return Err(DiskError::FileNotFound); + } + + // Heal reuses the version's data_dir, so for in-place corruption + // the destination dir still exists — and rename(2) cannot replace + // a non-empty directory (EEXIST on XFS, ENOTEMPTY on ext4). Purge + // it first, healing commits only; fresh PUTs mint a new data_dir + // and never collide. Best effort: a real failure surfaces in the + // rename below. + if fi_healing + && let Some((_, dst_data_path)) = has_data_dir_path.as_ref() + && let Err(err) = self.move_to_trash(dst_data_path, true, false).await + { + warn!( + event = EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + dst_path = ?dst_data_path, + error = ?err, + "Healing commit could not purge the stale destination data dir" + ); + } + if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() + && let Err(err) = os::rename_all_with_commit_guard( + src_data_path, + dst_data_path, + &skip_parent, + &self.publication_root, + &rename_commit_guard, + mutation_lease.clone(), + ) + .await + { + info!( + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "rename_all_data_path_failed", + src_path = ?src_data_path, + dst_path = ?dst_data_path, + error = ?err, + "Disk local rename flow failed" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + #[cfg(test)] + if has_data_dir_path.is_some() { + run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); + } + + // Crash-consistency injection: hard power loss after the data dir + // is in place but before xl.meta commits. No cleanup — the harness + // reopens the disk and asserts the object still reads as the old + // version (the staged data dir is a harmless orphan for GC). + if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, dst_path) { + return Err(DiskError::Unexpected); + } + + if should_fail_before_old_metadata_backup(dst_path) { + info!( + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "test_fail_before_old_metadata_backup", + "Disk local rename flow failed before metadata commit" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(DiskError::Unexpected); + } + + // The rollback backup stays where it is written (no rename) and is + // the sole restore source for a later undo_write, so under strict + // it keeps SyncMode::FileAndDir: contents and directory entry both + // durable. It is part of the metadata commit machinery, so relaxed + // tiers leave it to the page cache like the xl.meta it mirrors. + let backup_sync = if durability.syncs_commit_metadata() { + SyncMode::FileAndDir + } else { + SyncMode::None + }; + if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) { + let backup_parent = dst_file_parent.join(old_data_dir.to_string()); + #[cfg(not(windows))] + if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await { + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + let backup_path_guard = match rename_commit_guard.create_destination_directory_for_path_access(&backup_parent) { + Ok(guard) => guard, + Err(err) => { + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(DiskError::from(to_file_error(err))); + } + }; + let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP); + if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) { + #[cfg(windows)] + drop(backup_path_guard); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + let backup_bytes = dst_buf.clone(); + // Keep the volume, commit-tree, and exact destination-path + // guards in this task until the backup write and durability + // sync finish. A detached spawn_blocking writer could survive + // cancellation and later truncate a newer transaction's + // deterministic rollback backup. + let write_result = os::run_blocking_namespace_operation(mutation_lease.clone(), move || { + #[cfg(test)] + run_owned_file_write_before_open(&backup_path); + backup_path_guard.write_file_for_path_access( + &backup_path, + backup_bytes.as_ref(), + backup_sync != SyncMode::None, + backup_sync == SyncMode::FileAndDir, + ) + }) + .await + .map_err(to_file_error) + .map_err(DiskError::from); + if let Err(err) = write_result { + info!( + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "write_old_metadata_backup_failed", + error = ?err, + "Disk local rename flow failed" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + } + + // Crash-consistency injection: hard power loss after the rollback + // backup is durable but before the xl.meta commit rename. No + // cleanup — the harness asserts the object still reads as the old + // version, since the destination xl.meta is untouched here. + if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path) { + return Err(DiskError::Unexpected); + } + + if let Err(err) = os::rename_all_with_prepared_source( + prepared_metadata_source, + &src_file_path, + &dst_file_path, + &skip_parent, + &self.publication_root, + &rename_commit_guard, + mutation_lease.clone(), + ) + .await + { + info!( + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "rename_all_metadata_failed", + src_path = ?src_file_path, + dst_path = ?dst_file_path, + error = ?err, + "Disk local rename flow failed" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + + let committed_new_data_path = has_data_dir_path.as_ref().map(|(_, dst_data_path)| dst_data_path.as_path()); + if should_fail_after_metadata_commit(dst_path) { + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + return Err(DiskError::Unexpected); + } + + // Crash-consistency injection: hard power loss immediately after the + // xl.meta commit rename but before the durability fsync. Unlike the + // graceful failpoint above, no rollback runs — the commit rename is + // already on disk, so the harness asserts the object reads back as + // the new version. + if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, dst_path) { + return Err(DiskError::Unexpected); + } + + // Persist the directory entries for both the data dir and xl.meta renames; + // without this the commit itself can vanish on power loss. Relaxed tiers + // accept that window (documented in docs/operations/durability-modes.md). + if durability.syncs_commit_metadata() + && let Some(parent) = dst_file_path.parent() + { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = os::fsync_dst_dir_group_commit(parent).await { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + // The commit rename changed the dst part inodes before this fsync + // failed and rolled them back; drop any fd cached during that + // window so readers re-open the restored inode (rustfs/backlog#1177). + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(to_file_error(err).into()); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + } + + // First PUT of an object creates its directory (and any missing prefix + // dirs) via reliable_mkdir_all, which never fsyncs the parent chain. The + // commit fsync above persists the object dir's *contents*, not its own + // entry in the bucket/prefix dir, so on power loss after ack the whole + // object dir could vanish (rustfs/backlog#922 step 4). For a new object + // (no prior xl.meta) fsync the ancestor chain from the object dir's + // parent up to and including the bucket so those new directory entries + // are durable. Overwrites already have a durable object dir. The + // starts_with guard bounds the walk to the bucket subtree. Relaxed/none + // accept the wider window, like the commit fsync above. + if has_dst_buf.is_none() && durability.syncs_commit_metadata() { + let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); + while let Some(dir) = ancestor { + if !dir.starts_with(&dst_volume_dir) { + break; + } + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = os::fsync_dir(dir).await { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + // Same post-commit rollback window as above — drop cached + // dst part fds so readers re-open the restored inode + // (rustfs/backlog#1177). + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(to_file_error(err).into()); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + if dir == dst_volume_dir.as_path() { + break; + } + ancestor = dir.parent(); + } + } + + // Publication and every rollback-capable durability step are now + // complete. Do not retain the Windows object identity guard while + // cleaning staging paths or invalidating cached descriptors. + #[cfg(windows)] + drop(rename_commit_guard); + + if let Some(src_file_path_parent) = src_file_path.parent() { + if src_volume != super::RUSTFS_META_MULTIPART_BUCKET { + let _ = std::fs::remove_dir(src_file_path_parent); + } else { + let _ = self + .delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false) + .await; + } + } + + // Heal reuses a version's `data_dir` and lands the rebuilt shard on + // the SAME `//part.N` path. Without this, a cached + // descriptor would keep serving the pre-heal inode, defeating the heal + // and eroding read quorum (backlog#1145). + // + // The exact keys are derivable here, and this runs on every write, so + // use them rather than registering a predicate the read path would then + // have to evaluate. Readers build the same string + // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every + // part of the version now at `dst_path` — any part path absent from it + // no longer exists for readers to ask for. + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + + Ok(RenameDataResp { + old_data_dir: has_old_data_dir, + rollback_data_dir, + cleanup_data_dir: has_old_data_dir, + sign: version_signature, + old_current_size, + }) + } else { + // Inline metadata preparation is blocking. The transaction lease is + // moved into that work so a timeout can release the async waiter without + // allowing a retry to reuse the deterministic staging path too early. + let src = src_file_path.clone(); + let dst = dst_file_path.clone(); + let cleanup_path = if src_volume == super::RUSTFS_META_MULTIPART_BUCKET { + src_file_path.parent().map(|p| p.to_path_buf()) + } else { + None + }; + let dst_path_for_failpoint = dst_path.to_string(); + #[cfg(windows)] + let source_parent = src_file_parent.to_path_buf(); + let rename_commit_guard_for_preparation = rename_commit_guard.clone(); + let sync = durability.syncs_commit_metadata(); + #[cfg(test)] + run_inline_before_file_sync_admission(dst_path); + let mut file_sync_admission = if sync { + Some( + os::acquire_file_sync_admission(self.file_sync_permits.clone()) + .await + .map_err(to_file_error) + .map_err(DiskError::from)?, + ) + } else { + None + }; + let prepare_inline_metadata = move || { + let mut prepared_metadata_source = + os::create_prepared_rename_source_with_commit_guard(&src, &dst, &rename_commit_guard_for_preparation)?; + #[cfg(windows)] + let source_metadata_guard = + rename_commit_guard_for_preparation.lock_source_directory_for_path_access(&source_parent)?; + let mut xlmeta = FileMeta::new(); + // Same as the non-inline branch: an unparsable existing dst + // xl.meta must surface as unknown, not `Absent` + // (rustfs/backlog#1009). + let mut dst_meta_unparsable = false; + if let Some(ref buf) = has_dst_buf { + if FileMeta::is_xl2_v1_format(buf) + && let Ok(nmeta) = FileMeta::load(buf) + { + xlmeta = nmeta + } else { + dst_meta_unparsable = true; + } + } + + let old_current_size = if dst_meta_unparsable { + None + } else { + observe_old_current_size(has_dst_buf.is_some(), &xlmeta) + }; + + let version_id = fi.version_id.unwrap_or_default(); + let old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); + let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); + let rollback_data_dir = old_data_dir.or_else(|| { + if old_version_exists && has_dst_buf.is_some() { + Some(inline_metadata_rollback_dir(version_id, &xlmeta)) + } else { + None + } + }); + let mut staged_rollback_path = None; + if let Some(d) = old_data_dir.as_ref() { + let _ = xlmeta.data.remove_two(version_id, *d); + } + xlmeta.add_version(fi)?; + let version_signature = rename_data_versions_signature(&xlmeta); + let new_buf = xlmeta.marshal_msg()?; + // Write the staged xl.meta. Inline objects carry their data inside + // xl.meta, so this is the durable preparation for the metadata commit: + // relaxed tiers do no per-object fsync here at all (aligned + // with MinIO's default), trading a documented power-loss + // window for latency. + prepared_metadata_source.write_all(&new_buf, sync)?; + run_inline_preparation_before_backup(&dst_path_for_failpoint); + if let Some(ref old_metadata) = has_dst_buf + && (rollback_data_dir.is_some() || sync || cfg!(test)) + { + #[cfg(windows)] + let backup_path = { + let backup_path = src + .parent() + .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"))? + .join(STORAGE_FORMAT_FILE_BACKUP); + source_metadata_guard.write_file_for_path_access(&backup_path, old_metadata, sync, false)?; + backup_path + }; + #[cfg(not(windows))] + let backup_path = create_local_inline_rollback_backup(&dst, &src, old_metadata)?; + #[cfg(not(windows))] + if sync { + std::fs::File::open(&backup_path)?.sync_data()?; + } + staged_rollback_path = Some(backup_path); + } + + Ok::<_, std::io::Error>(( + rollback_data_dir, + old_data_dir, + version_signature, + old_current_size, + staged_rollback_path, + has_dst_buf.is_none(), + prepared_metadata_source, + )) + }; + let inline_preparation = if let Some(admission) = file_sync_admission.as_ref() { + os::run_blocking_namespace_file_sync_operation(mutation_lease.clone(), admission, prepare_inline_metadata).await + } else { + os::run_blocking_namespace_operation(mutation_lease.clone(), prepare_inline_metadata).await + } + .map_err(to_file_error) + .map_err(DiskError::from); + + let ( + rollback_data_dir, + cleanup_data_dir, + version_signature, + old_current_size, + mut local_rollback_path, + destination_was_absent, + prepared_metadata_source, + ) = match inline_preparation { + Ok(prepared) => prepared, + Err(err) => { + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(err); + } + }; + + let rename_commit_guard = remove_dst_base_before_commit( + dst_path, + rename_commit_guard, + src_file_parent, + dst_file_parent, + &dst_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + + if should_remove_staged_meta_before_commit(dst_path) { + drop(prepared_metadata_source); + let remove_result = std::fs::remove_file(&src_file_path); + if let Some(backup_path) = local_rollback_path.as_deref() { + let _ = remove_file_if_exists(backup_path); + } + remove_result.map_err(to_file_error)?; + return Err(DiskError::FileNotFound); + } + + if let (Some(rollback_data_dir), Some(staged_backup)) = (rollback_data_dir, local_rollback_path.as_deref()) { + let Some(dst_parent) = dst_file_path.parent() else { + return Err(DiskError::other("missing object metadata parent")); + }; + let backup_path = dst_parent + .join(rollback_data_dir.to_string()) + .join(STORAGE_FORMAT_FILE_BACKUP); + // rename_all acquires the backup path's namespace lease. Do not + // hold a disk admission while acquiring another namespace lock. + drop(file_sync_admission.take()); + if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await { + let _ = remove_file_if_exists(staged_backup); + return Err(err); + } + #[cfg(test)] + run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); + if sync { + file_sync_admission = Some( + os::acquire_file_sync_admission(self.file_sync_permits.clone()) + .await + .map_err(to_file_error) + .map_err(DiskError::from)?, + ); + } + if let Some(admission) = file_sync_admission.as_ref() + && let Some(backup_parent) = backup_path.parent() + { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dir_with_namespace_file_sync_limit(backup_parent, mutation_lease.clone(), admission).await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, + fsync_started, + ); + return Err(DiskError::from(to_file_error(err))); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, + fsync_started, + ); + } + local_rollback_path = None; + } + + let commit_result = if should_fail_commit_rename(dst_path) { + Err(DiskError::other("test fail during metadata commit rename")) + } else { + os::rename_all_with_prepared_source( + prepared_metadata_source, + &src_file_path, + &dst_file_path, + &dst_volume_dir, + &self.publication_root, + &rename_commit_guard, + mutation_lease.clone(), + ) + .await + }; + if let Err(err) = commit_result { + if let Some(backup_path) = local_rollback_path.as_deref() { + let _ = remove_file_if_exists(backup_path); + } + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(err); + } + + let post_commit = async { + if should_fail_after_metadata_commit(dst_path) { + rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; + return Err(std::io::Error::other("test fail after metadata commit")); + } + + // Persist the commit rename's directory entry across power loss. + if let Some(admission) = file_sync_admission.as_ref() + && let Some(dst_parent) = dst_file_path.parent() + { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dst_dir_group_commit_or_namespace_file_sync_limit(dst_parent, mutation_lease.clone(), admission) + .await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; + return Err(err); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + } + + // Same power-loss gap as the non-inline path (rustfs/backlog#922 + // step 4): a first PUT creates the object dir (and any missing + // prefix dirs) whose entry in the bucket/prefix dir reliable_mkdir_all + // never fsynced. The fsync above persists the object dir's contents, + // not its own entry, so for a new inline object fsync the ancestor + // chain up to and including the bucket. Overwrites already have a + // durable object dir; the starts_with guard bounds the walk. + if let Some(admission) = file_sync_admission.as_ref() + && destination_was_absent + { + let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); + while let Some(ancestor_dir) = ancestor { + if !ancestor_dir.starts_with(&dst_volume_dir) { + break; + } + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, mutation_lease.clone(), admission).await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + rollback_inline_metadata_commit_std( + &dst_file_path, + rollback_data_dir, + local_rollback_path.as_deref(), + )?; + return Err(err); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + if ancestor_dir == dst_volume_dir.as_path() { + break; + } + ancestor = ancestor_dir.parent(); + } + } + + Ok::<(), std::io::Error>(()) + } + .await; + + // The disk admission protects the durability chain, not staging + // cleanup or cache invalidation after that chain has completed. + drop(file_sync_admission.take()); + + // A post-commit rollback (for example, a commit-metadata fsync + // failure under strict durability) restores the old metadata; drop any + // descriptors cached during the committed window before propagating the + // error (rustfs/backlog#1177). Inline objects carry data in xl.meta, so + // this is mostly defensive and keeps both commit branches consistent. + if let Err(err) = post_commit { + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(DiskError::from(err)); + } + + // The commit no longer has a rollback path. Release the Windows + // object identity guard before best-effort staging cleanup. + #[cfg(windows)] + drop(rename_commit_guard); + + if let Some(backup_path) = local_rollback_path.as_deref() { + let _ = remove_file_if_exists(backup_path); + } + + // Cleanup + if let Some(ref cleanup) = cleanup_path { + let _ = self.delete_file(&dst_volume_dir, cleanup, true, false).await; + } else if let Some(parent) = src_file_path.parent() { + let _ = std::fs::remove_dir(parent); + } + + // Heal reuses a version's `data_dir` and lands the rebuilt shard on + // the SAME `//part.N` path. Without this, a cached + // descriptor would keep serving the pre-heal inode, defeating the heal + // and eroding read quorum (backlog#1145). + // + // The exact keys are derivable here, and this runs on every write, so + // use them rather than registering a predicate the read path would then + // have to evaluate. Readers build the same string + // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every + // part of the version now at `dst_path` — any part path absent from it + // no longer exists for readers to ask for. + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + + Ok(RenameDataResp { + old_data_dir: cleanup_data_dir, + rollback_data_dir, + cleanup_data_dir, + sign: version_signature, + old_current_size, + }) + } + } + + pub(in crate::disk) async fn rename_data_observed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> super::RenameDataObservation { + let mut preflight_rejection = None; + let result = self + .rename_data_inner(src_volume, src_path, fi.clone(), dst_volume, dst_path, &mut preflight_rejection) + .await; + super::RenameDataObservation { + result, + preflight_rejection, + } + } + pub(crate) async fn rename_data_borrowed( &self, src_volume: &str, From 2f02d1d2d8bd1026cc43dfd155586c894e7b266d Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 16:26:13 +0800 Subject: [PATCH 05/12] fix(ci): preserve security suite failures and isolate reports (#7188) --- .config/make/tests.mak | 1 + .github/workflows/ci-docs-only.yml | 1 + .github/workflows/ci.yml | 1 + .github/workflows/rustfs-security-test.yml | 83 ++++++--- scripts/test_security_workflow.py | 197 +++++++++++++++++++++ 5 files changed, 255 insertions(+), 28 deletions(-) create mode 100644 scripts/test_security_workflow.py diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 3ee3337fa..d1297e9ad 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -40,6 +40,7 @@ script-tests: ## Run shell script tests $(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test + $(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py $(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py bash -n ./scripts/validate_object_data_cache_cold_stampede.sh $(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test diff --git a/.github/workflows/ci-docs-only.yml b/.github/workflows/ci-docs-only.yml index de78e7f7f..7c6ad22ec 100644 --- a/.github/workflows/ci-docs-only.yml +++ b/.github/workflows/ci-docs-only.yml @@ -129,6 +129,7 @@ jobs: run: | python3 ./scripts/check_test_wiring.py --self-test python3 ./scripts/check_scheduled_validation_freshness.py --self-test + python3 ./scripts/test_security_workflow.py python3 ./scripts/check_test_wiring.py - name: Check no planning docs committed diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98fc6cc30..171f52380 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,6 +167,7 @@ jobs: run: | python3 ./scripts/check_test_wiring.py --self-test python3 ./scripts/check_scheduled_validation_freshness.py --self-test + python3 ./scripts/test_security_workflow.py python3 ./scripts/check_test_wiring.py - name: Check no planning docs committed diff --git a/.github/workflows/rustfs-security-test.yml b/.github/workflows/rustfs-security-test.yml index 2b2a5d68b..16e10a80c 100644 --- a/.github/workflows/rustfs-security-test.yml +++ b/.github/workflows/rustfs-security-test.yml @@ -74,10 +74,23 @@ env: jobs: security-test: runs-on: smoke-testing - continue-on-error: true timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + - name: Checkout repository (for the OIDC live gate script) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Initialize security evidence + id: evidence + run: | + set -euo pipefail + umask 077 + SECURITY_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -- "${SECURITY_ARTIFACTS_DIR}" + printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}" + # auto-testing is private: clone it with the dedicated PF token (not # GITHUB_TOKEN) and retry transient GitHub/network failures. - name: Checkout auto-testing scripts (with retry) @@ -98,11 +111,6 @@ jobs: echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2 exit 1 - - name: Checkout repository (for the OIDC live gate script) - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - persist-credentials: false - - name: Show environment run: | uname -a @@ -135,7 +143,8 @@ jobs: id: test continue-on-error: true env: - REPORT_FILE: /tmp/rustfs-security-report.md + REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md + TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }} RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh run: | set -euo pipefail @@ -159,29 +168,48 @@ jobs: else ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}") fi - ./auto-testing/rustfs-security-test.sh "${ARGS[@]}" + GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}" - name: Generate report - if: always() + id: report + if: ${{ always() && steps.evidence.outcome == 'success' }} + env: + TEST_OUTCOME: ${{ steps.test.outcome }} run: | set -euo pipefail - if [ ! -f /tmp/rustfs-security-report.md ]; then - { - echo "# RustFS security test report" - echo "" - echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - echo "- Trigger: ${{ github.event_name }}" - echo "- Test Step Outcome: failure (suite did not produce a report)" - } > /tmp/rustfs-security-report.md + RESULT=failure + if [ "${TEST_OUTCOME}" = "success" ] && [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then + RESULT=success fi - cat /tmp/rustfs-security-report.md >> "${GITHUB_STEP_SUMMARY}" + { + echo "# RustFS security test report" + echo "" + echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" + echo "- Trigger: ${GITHUB_EVENT_NAME}" + echo "- Test Step Outcome: ${RESULT}" + echo "- Suite Step Outcome: ${TEST_OUTCOME}" + echo "" + # The dashboard prioritizes case rows over the step outcome. + # Keep partial case results in the artifact when the suite fails. + if [ "${RESULT}" = "success" ]; then + cat "${SECURITY_ARTIFACTS_DIR}/suite-report.md" + elif [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then + echo "The suite did not complete successfully. See suite-report.md in this run's artifact for diagnostics." + else + echo "The suite did not produce a non-empty report." + fi + } > "${SECURITY_ARTIFACTS_DIR}/report.md" + cat "${SECURITY_ARTIFACTS_DIR}/report.md" >> "${GITHUB_STEP_SUMMARY}" + [ "${RESULT}" = "success" ] - name: Upload functional report to dashboard - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} - REPORT_FILE: /tmp/rustfs-security-report.md + REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md SUITE: security run: | set -euo pipefail @@ -210,8 +238,9 @@ jobs: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} SUITE: 'security' SUITE_LABEL: 'Security' + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPORT_FILE: '/tmp/rustfs-security-report.md' + REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md LOG_FILE: '' run: | set -euo pipefail @@ -245,7 +274,7 @@ jobs: echo "" echo "## Report (errors and symptoms)" echo "" - if [ -s "${REPORT_FILE}" ]; then + if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then redact < "${REPORT_FILE}" elif [ -s "${LOG_FILE:-}" ]; then echo "(report file missing; log tail below)" @@ -263,14 +292,12 @@ jobs: echo "filed backlog issue for suite ${SUITE}" - name: Upload report and logs - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: - name: rustfs-security-test-${{ github.run_id }} - path: | - /tmp/rustfs-security-report.md - /tmp/rustfs-security.*/* - if-no-files-found: ignore + name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.SECURITY_ARTIFACTS_DIR }}/ + if-no-files-found: error retention-days: 3 - name: Cleanup environment (after) diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py new file mode 100644 index 000000000..ae82d3fb1 --- /dev/null +++ b/scripts/test_security_workflow.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""Run the security workflow's evidence and result steps without remote VMs.""" + +from __future__ import annotations + +import os +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + +from check_test_wiring import yaml_block + + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".github/workflows/rustfs-security-test.yml" +CASE_ROW = "| IAM-101 | user CRUD lifecycle | PASS |" + + +class SecurityWorkflowTests(unittest.TestCase): + def setUp(self) -> None: + self.source = WORKFLOW.read_text() + self.job = yaml_block(self.source.splitlines(), "security-test", 2) + self.assertIsNotNone(self.job) + starts = [i for i, line in enumerate(self.job) if line.startswith(" - name: ")] + self.steps = { + self.job[start].split(": ", 1)[1].strip('"'): self.job[start:end] + for start, end in zip(starts, starts[1:] + [len(self.job)]) + } + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.directory = Path(self.temp.name) + self.context = { + "runner.temp": self.temp.name, + "github.server_url": "https://github.com", + "github.repository": "rustfs/rustfs", + "github.run_id": "314159", + "github.run_attempt": "2", + "github.sha": "0123456789abcdef0123456789abcdef01234567", + "github.event_name": "workflow_dispatch", + "github.workspace": self.temp.name, + "inputs.package_url": "", + "inputs.rustfs_version": "test-version", + "inputs.topology": "all", + "inputs.oidc_live": "false", + "steps.evidence.outcome": "skipped", + "steps.test.outcome": "skipped", + "steps.report.outcome": "skipped", + } + self.env = { + **os.environ, "GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"), + "GITHUB_ENV": str(self.directory / "github-env"), "RUNNER_TEMP": self.temp.name, "TMPDIR": self.temp.name, + } + for key in ("server_url", "repository", "run_id", "run_attempt", "sha", "event_name"): + self.env[f"GITHUB_{key.upper()}"] = self.context[f"github.{key}"] + self.context["env.SECURITY_ARTIFACTS_DIR"] = "" + self.artifacts = self.directory / "rustfs-security-314159-2" + suite = self.directory / "auto-testing/rustfs-security-test.sh" + suite.parent.mkdir() + suite.write_text( + '#!/usr/bin/env bash\nset -euo pipefail\n' + 'log_dir=$(mktemp -d "$TMPDIR/rustfs-security.XXXXXX")\n' + 'echo "CURRENT SUITE LOG" > "$log_dir/suite.log"\n' + 'case "$FAKE_REPORT" in\n' + f' present) printf "%s\\n" "CURRENT SUITE DIAGNOSTIC" "{CASE_ROW}" > "$REPORT_FILE" ;;\n' + ' empty) : > "$REPORT_FILE" ;;\n' + 'esac\n' + 'echo "UNWRAPPED SUITE SUMMARY" >> "$GITHUB_STEP_SUMMARY"\n' + 'exit "$FAKE_EXIT"\n' + ) + + def render(self, value: str) -> str: + return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value) + + def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]: + result = {} + for line in yaml_block(lines, "env", indent) or []: + if line.strip() and not line.lstrip().startswith("#"): + key, value = line.strip().split(": ", 1) + result[key] = self.render(value.strip("'\"")) + return result + + def run_step(self, name: str) -> subprocess.CompletedProcess[str]: + lines = self.steps[name] + start = lines.index(" run: |") + 1 + shell_lines = [] + for line in lines[start:]: + if line.strip() and not line.startswith(" "): + break + shell_lines.append(line[10:]) + self.assertTrue(shell_lines, f"missing literal shell body: {name}") + result = subprocess.run( + ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render("\n".join(shell_lines))], + cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True, + ) + for line in lines: + if line.startswith(" id: "): + self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success" + if Path(self.env["GITHUB_ENV"]).exists(): + for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines(): + key, value = line.split("=", 1) + self.env[key] = value + self.context[f"env.{key}"] = value + return result + + def test_workflow_wiring(self) -> None: + names = list(self.steps) + self.assertLess(names.index("Checkout repository (for the OIDC live gate script)"), names.index("Checkout auto-testing scripts (with retry)")) + self.assertNotIn(" continue-on-error: true", self.job) + self.assertIn(" continue-on-error: true", self.steps["Run security suite"]) + for name in ("Initialize security evidence", "Generate report"): + self.assertNotIn(" continue-on-error: true", self.steps[name]) + self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps["Generate report"]) + self.assertNotIn("/tmp/rustfs-security", self.source) + for name in ("Upload functional report to dashboard", "File failure issue in rustfs/backlog"): + report = next(line for line in self.steps[name] if line.strip().startswith("REPORT_FILE:")) + self.assertIn("${{ env.SECURITY_ARTIFACTS_DIR }}/report.md", report) + for name in ("Upload functional report to dashboard", "Upload report and logs"): + self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps[name]) + artifact_settings = yaml_block(self.steps["Upload report and logs"], "with", 8) + self.assertIn(" path: ${{ env.SECURITY_ARTIFACTS_DIR }}/", artifact_settings) + self.assertIn(" if-no-files-found: error", artifact_settings) + + def test_suite_report_and_result_matrix(self) -> None: + for outcome, mode, exit_code in ( + ("success", "present", 0), ("failure", "present", 7), ("failure", "missing", 7), + ("success", "missing", 0), ("success", "empty", 0), + ("skipped", "missing", 0), ("skipped", "present", 0), + ("cancelled", "missing", 0), ("cancelled", "present", 0), + ): + with self.subTest(outcome=outcome, report=mode): + self.setUp() + initialized = self.run_step("Initialize security evidence") + self.assertEqual(initialized.returncode, 0, initialized.stderr) + self.assertEqual(self.env["SECURITY_ARTIFACTS_DIR"], str(self.artifacts)) + self.env.update(FAKE_REPORT=mode, FAKE_EXIT=str(exit_code)) + if outcome != "skipped" or mode == "present": + suite = self.run_step("Run security suite") + self.assertEqual(suite.returncode, exit_code, suite.stderr) + logs = list(self.artifacts.glob("rustfs-security.*/suite.log")) + self.assertEqual(len(logs), 1) + self.assertEqual(logs[0].read_text(), "CURRENT SUITE LOG\n") + self.context["steps.test.outcome"] = outcome + report = self.run_step("Generate report") + success = outcome == "success" and mode == "present" + self.assertEqual(report.returncode == 0, success, report.stderr) + contents = (self.artifacts / "report.md").read_text() + for expected in ( + "https://github.com/rustfs/rustfs/actions/runs/314159", "Attempt: 2", + f"Workflow Commit: {self.context['github.sha']}", "Trigger: workflow_dispatch", + f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}", + ): + self.assertIn(expected, contents) + self.assertEqual(CASE_ROW in contents, success) + self.assertEqual("CURRENT SUITE DIAGNOSTIC" in contents, success) + if mode == "present": + raw = (self.artifacts / "suite-report.md").read_text() + self.assertEqual(raw, f"CURRENT SUITE DIAGNOSTIC\n{CASE_ROW}\n") + summary = Path(self.env["GITHUB_STEP_SUMMARY"]).read_text() + self.assertEqual(summary, contents) + self.assertNotIn("UNWRAPPED SUITE SUMMARY", summary) + + def test_existing_evidence_directory_is_rejected(self) -> None: + self.artifacts.mkdir() + stale = self.artifacts / "suite-report.md" + stale.write_text("OLD RUN REPORT") + self.assertNotEqual(self.run_step("Initialize security evidence").returncode, 0) + self.assertEqual(stale.read_text(), "OLD RUN REPORT") + self.assertFalse(Path(self.env["GITHUB_ENV"]).exists()) + (self.artifacts / "report.md").write_text("OLD RUN REPORT") + self.context.update({ + "env.SECURITY_ARTIFACTS_DIR": str(self.artifacts), "secrets.PF_TESTING_GH_TOKEN": "fake-local-token", + }) + fake_bin = self.directory / "bin" + fake_bin.mkdir() + gh = fake_bin / "gh" + gh.write_text( + '#!/usr/bin/env bash\nset -euo pipefail\n' + 'if [ "$1 $2" = "issue create" ]; then\n' + ' while [ "$#" -gt 0 ]; do\n' + ' if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n' + ' shift\n' + ' done\n' + 'fi\n' + ) + gh.chmod(0o755) + body = self.directory / "issue-body.md" + self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body)) + result = self.run_step("File failure issue in rustfs/backlog") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn("OLD RUN REPORT", body.read_text()) + self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text()) + + +if __name__ == "__main__": + unittest.main() From 2d159635eddb9f17ad504410be91e20557eee4fe Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Sat, 5 Sep 2026 16:32:02 +0800 Subject: [PATCH 06/12] feat(scanner): plan dirty bucket cache refreshes (#7146) * feat(scanner): plan dirty bucket cache refreshes * fix(scanner): route peer snapshot through storage boundary --------- Co-authored-by: Henry Guo Co-authored-by: cxymds Co-authored-by: Zhengchao An --- crates/scanner/src/scanner.rs | 23 ++- crates/scanner/src/scanner/activity.rs | 41 ++++ crates/scanner/src/scanner/tests.rs | 38 ++++ crates/scanner/src/scanner_io.rs | 119 ++++++++++- crates/scanner/src/scanner_io/cache.rs | 18 ++ crates/scanner/src/scanner_io/io_cycle.rs | 95 ++++++++- .../src/scanner_io/publish_gate_tests.rs | 22 ++ crates/scanner/src/scanner_io/tests.rs | 190 ++++++++++++++++++ crates/scanner/src/storage_api.rs | 4 +- 9 files changed, 539 insertions(+), 11 deletions(-) diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index fcf67eed0..0f97924a4 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -1703,14 +1703,18 @@ where let (sender, receiver) = mpsc::channel::(1); let done_cycle = Metrics::time(Metric::ScanCycle); - let scan_result = crate::scanner_io::nsscanner_with_storage_status( + let scan_result = crate::scanner_io::nsscanner_with_storage_status_scoped( storeapi.as_ref(), - cycle_budget.token(), - cycle_budget.clone(), - sender, - cycle_info.current, - leader_epoch, - scan_mode, + crate::scanner_io::ScannerCycleRequest { + ctx: cycle_budget.token(), + budget: cycle_budget.clone(), + updates: sender, + want_cycle: cycle_info.current, + leader_epoch, + scan_mode, + scan_scope: crate::scanner_io::ScannerBucketScanScope::default(), + persisted_usage_baseline: usage_persist_baseline.data.clone(), + }, ) .await; let publication_defer_reason = match &scan_result { @@ -3424,10 +3428,13 @@ use cycle_state::*; use leadership::*; use usage_store::*; +#[cfg(test)] +pub(crate) use activity::scanner_activity_snapshot_digest; pub use activity::scanner_topology_digest; pub(crate) use activity::{ ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication, - scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements, + scanner_activity_dirty_usage_state_for_host, scanner_activity_publication_lease_targets, scanner_activity_structural_digest, + scanner_dirty_usage_acknowledgements, }; pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance}; pub use backlog::{ diff --git a/crates/scanner/src/scanner/activity.rs b/crates/scanner/src/scanner/activity.rs index 2f8aed1b5..ffcbc5313 100644 --- a/crates/scanner/src/scanner/activity.rs +++ b/crates/scanner/src/scanner/activity.rs @@ -902,6 +902,7 @@ where observation } +#[cfg(test)] pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] { let mut hasher = Sha256::new(); hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes()); @@ -925,6 +926,30 @@ pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapsho hasher.finalize().into() } +/// Hash the activity inputs that make an existing scanner cache unsafe to +/// reuse. Regular namespace writes and dirty-usage generations are omitted: +/// their affected buckets are tracked separately and may be refreshed from a +/// complete authoritative cache baseline. +pub(crate) fn scanner_activity_structural_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes()); + for (host, activity) in snapshot { + let host = host.as_bytes(); + let instance_id = activity.instance_id.as_bytes(); + hasher.update(u64::try_from(host.len()).unwrap_or(u64::MAX).to_be_bytes()); + hasher.update(host); + hasher.update(u64::try_from(instance_id.len()).unwrap_or(u64::MAX).to_be_bytes()); + hasher.update(instance_id); + hasher.update(activity.maintenance_generation.to_be_bytes()); + hasher.update(activity.protocol_version.to_be_bytes()); + hasher.update(activity.topology_digest); + hasher.update([u8::from(activity.data_movement_active)]); + hasher.update(activity.movement_generation.to_be_bytes()); + hasher.update([u8::from(activity.publication_blocked)]); + } + hasher.finalize().into() +} + pub(crate) fn scanner_activity_allows_usage_publication(snapshot: &ScannerActivitySnapshot) -> bool { !snapshot.is_empty() && snapshot.values().all(|activity| { @@ -955,6 +980,22 @@ pub(crate) fn scanner_dirty_usage_acknowledgements(snapshot: &ScannerActivitySna .collect() } +pub(crate) fn scanner_activity_dirty_usage_state_for_host<'a>( + snapshot: &'a ScannerActivitySnapshot, + host: &str, +) -> Option<(&'a str, u64, bool)> { + snapshot + .get(host) + .filter(|_| host != LOCAL_SCANNER_ACTIVITY_NODE) + .map(|activity| { + ( + activity.instance_id.as_str(), + activity.dirty_usage_generation, + activity.dirty_usage_pending, + ) + }) +} + pub fn scanner_topology_digest(storeapi: &ECStore) -> [u8; 32] { let endpoint_pools = storeapi.endpoints(); let mut hasher = Sha256::new(); diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index efe29f612..244e9088f 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -8169,6 +8169,44 @@ fn scanner_activity_snapshot_digest_fences_dirty_usage_state() { assert_ne!(scanner_activity_snapshot_digest(&clean), scanner_activity_snapshot_digest(&pending)); } +#[test] +fn scanner_activity_structural_digest_ignores_regular_bucket_writes() { + let baseline = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + let mut written = baseline.clone(); + let activity = written.get_mut("node-2").expect("node should exist"); + activity.namespace_generation = 8; + activity.dirty_usage_generation = 6; + activity.dirty_usage_pending = true; + + assert_ne!(scanner_activity_snapshot_digest(&baseline), scanner_activity_snapshot_digest(&written)); + assert_eq!( + scanner_activity_structural_digest(&baseline), + scanner_activity_structural_digest(&written), + "bucket writes are refreshed through the dirty-bucket scope rather than invalidating every cache" + ); +} + +#[test] +fn scanner_activity_structural_digest_fences_restart_and_maintenance() { + let baseline = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + let mut restarted = baseline.clone(); + restarted.get_mut("node-2").expect("node should exist").instance_id = "epoch-b".to_string(); + let mut maintained = baseline.clone(); + maintained + .get_mut("node-2") + .expect("node should exist") + .maintenance_generation = 4; + + assert_ne!( + scanner_activity_structural_digest(&baseline), + scanner_activity_structural_digest(&restarted) + ); + assert_ne!( + scanner_activity_structural_digest(&baseline), + scanner_activity_structural_digest(&maintained) + ); +} + #[test] fn scanner_dirty_usage_acknowledgements_exclude_local_and_clean_nodes() { let snapshot = BTreeMap::from([ diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 4807e11b8..c8353bae9 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -21,6 +21,7 @@ use crate::{ DataUsageCacheSource, DataUsageEntry, DataUsageEntryInfo, DataUsageInfo, DataUsageScanPlanDigest, DataUsageSnapshotSetState, ScannerError, SizeSummary, TierStats, }; +use bytes::Bytes; use futures::future::join_all; use metrics::counter; use rand::seq::SliceRandom as _; @@ -54,6 +55,7 @@ use tokio_util::task::AbortOnDropHandle; use tracing::{debug, error, warn}; use crate::ScannerObjectInfo as ObjectInfo; +use crate::storage_api::EcstoreScannerPeerDirtyUsageSnapshot; use crate::storage_api::ScannerStorage; use crate::storage_api::scan::NamespaceLocking as _; use crate::storage_api::scanner_io::{BucketInfo, BucketOptions}; @@ -111,6 +113,121 @@ pub(crate) struct ScannerBucketScanScope { baseline_scan_plan_digest: Option, } +impl ScannerBucketScanScope { + fn is_default(&self) -> bool { + self.selected_buckets.is_none() && self.baseline_scan_plan_digest.is_none() + } + + fn from_dirty_buckets(selected_buckets: HashSet, baseline_scan_plan_digest: DataUsageScanPlanDigest) -> Self { + Self { + selected_buckets: Some(Arc::new(selected_buckets)), + baseline_scan_plan_digest: Some(baseline_scan_plan_digest), + } + } +} + +#[derive(Clone, Copy)] +pub(super) struct ScannerCacheBaselineProof<'a> { + pub(super) data: Option<&'a Bytes>, + pub(super) expected_sources: &'a HashSet, + pub(super) leader_epoch: u64, + pub(super) want_cycle: u64, + pub(super) scan_plan_digest: DataUsageScanPlanDigest, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct ScannerPeerDirtyUsageExpectation { + instance_id: String, + generation: u64, + pending: bool, +} + +fn verified_remote_dirty_usage_buckets( + expected_peers: &HashMap, + peer_snapshots: Vec<(String, EcstoreScannerPeerDirtyUsageSnapshot)>, +) -> Option> { + if expected_peers.is_empty() || peer_snapshots.len() != expected_peers.len() { + return None; + } + + let mut received_peers = HashSet::with_capacity(peer_snapshots.len()); + let mut dirty_buckets = HashSet::new(); + for (host, snapshot) in peer_snapshots { + let expected = expected_peers.get(&host)?; + if !received_peers.insert(host) + || snapshot.instance_id != expected.instance_id + || snapshot.generation != expected.generation + || snapshot.generation == u64::MAX + || snapshot.protocol_version != crate::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION + || !snapshot.complete + || snapshot.pending_bucket_count != u64::try_from(snapshot.buckets.len()).unwrap_or(u64::MAX) + || (expected.pending && snapshot.pending_bucket_count == 0) + { + return None; + } + dirty_buckets.extend(snapshot.buckets.into_keys()); + } + + (received_peers.len() == expected_peers.len()).then_some(dirty_buckets) +} + +fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<'_>) -> Option { + let data = proof.data?; + let baseline = serde_json::from_slice::(data).ok()?; + if !baseline.is_complete_bucket_usage_snapshot() + || baseline.usage_snapshot_partial + || baseline.usage_snapshot_converged != Some(true) + || baseline.scanner_epoch != Some(proof.leader_epoch) + || baseline.usage_snapshot_set_states.len() != proof.expected_sources.len() + { + return None; + } + + let mut states = HashSet::with_capacity(baseline.usage_snapshot_set_states.len()); + for state in &baseline.usage_snapshot_set_states { + let source = DataUsageCacheSource::new(usize::try_from(state.pool_index).ok()?, usize::try_from(state.set_index).ok()?); + if !proof.expected_sources.contains(&source) + || !states.insert(source) + || !state.complete + || state.tombstone + || state.scanner_epoch != Some(proof.leader_epoch) + || state.scanner_cycle.is_none_or(|cycle| cycle > proof.want_cycle) + || state.scan_plan_digest != Some(proof.scan_plan_digest.0) + { + return None; + } + } + + (states == *proof.expected_sources).then_some(proof.scan_plan_digest) +} + +fn scoped_scan_scope_from_dirty_buckets( + requested_scope: ScannerBucketScanScope, + dirty_buckets: HashSet, + dirty_snapshot_complete: bool, + all_buckets: &[BucketInfo], + baseline_proof: ScannerCacheBaselineProof<'_>, +) -> ScannerBucketScanScope { + if !requested_scope.is_default() || !dirty_snapshot_complete { + return requested_scope; + } + + let current_buckets = all_buckets.iter().map(|bucket| bucket.name.as_str()).collect::>(); + let selected_buckets = dirty_buckets + .into_iter() + .filter(|bucket| current_buckets.contains(bucket.as_str())) + .collect::>(); + if selected_buckets.is_empty() { + return requested_scope; + } + + let Some(baseline_scan_plan_digest) = complete_scanner_cache_baseline_plan_digest(baseline_proof) else { + return requested_scope; + }; + + ScannerBucketScanScope::from_dirty_buckets(selected_buckets, baseline_scan_plan_digest) +} + pub(crate) fn is_scanner_metadata_corrupt_error(err: &StorageError) -> bool { matches!(err, StorageError::Io(io) if io.to_string().starts_with(SCANNER_METADATA_CORRUPT_ERROR)) } @@ -749,7 +866,7 @@ mod io_cache; mod io_cycle; #[cfg(test)] use io_cache::{ScannerSetCacheGeneration, prepare_scoped_set_scan}; -pub(crate) use io_cycle::nsscanner_with_storage_status; +pub(crate) use io_cycle::{ScannerCycleRequest, nsscanner_with_storage_status_scoped}; mod io_disk; #[cfg(test)] mod publish_gate_tests; diff --git a/crates/scanner/src/scanner_io/cache.rs b/crates/scanner/src/scanner_io/cache.rs index 9522ffed2..7ff684c99 100644 --- a/crates/scanner/src/scanner_io/cache.rs +++ b/crates/scanner/src/scanner_io/cache.rs @@ -282,9 +282,26 @@ pub(super) fn completed_data_usage_info( .iter() .map(|(bucket, usage)| (bucket.clone(), usage.size)) .collect(); + let mut usage_snapshot_set_states = results + .iter() + .map(|result| { + let source = result.info.source?; + Some(DataUsageSnapshotSetState { + pool_index: u64::try_from(source.pool_index).ok()?, + set_index: u64::try_from(source.set_index).ok()?, + scanner_cycle: Some(result.info.next_cycle), + scanner_epoch: Some(result.info.leader_epoch), + scan_plan_digest: Some(result.info.scan_plan_digest?.0), + complete: true, + tombstone: false, + }) + }) + .collect::>>()?; + usage_snapshot_set_states.sort_by_key(|state| (state.pool_index, state.set_index)); let data_usage_info = DataUsageInfo { last_update: Some(merged_last_update), scanner_cycle: Some(results.first()?.info.next_cycle), + scanner_epoch: Some(results.first()?.info.leader_epoch), objects_total_count: u64::try_from(total.objects).ok()?, versions_total_count: u64::try_from(total.versions).ok()?, delete_markers_total_count: u64::try_from(total.delete_markers).ok()?, @@ -295,6 +312,7 @@ pub(super) fn completed_data_usage_info( bucket_sizes, buckets_usage, usage_snapshot_complete: true, + usage_snapshot_set_states, ..Default::default() }; Some((data_usage_info, merged_last_update)) diff --git a/crates/scanner/src/scanner_io/io_cycle.rs b/crates/scanner/src/scanner_io/io_cycle.rs index 3b14e9ff2..2947d638d 100644 --- a/crates/scanner/src/scanner_io/io_cycle.rs +++ b/crates/scanner/src/scanner_io/io_cycle.rs @@ -71,6 +71,7 @@ where leader_epoch, scan_mode, scan_scope: ScannerBucketScanScope::default(), + persisted_usage_baseline: None, }; nsscanner_with_storage_status_scoped(store, request).await } @@ -83,6 +84,79 @@ pub(crate) struct ScannerCycleRequest { pub(crate) leader_epoch: u64, pub(crate) scan_mode: HealScanMode, pub(crate) scan_scope: ScannerBucketScanScope, + pub(crate) persisted_usage_baseline: Option, +} + +struct ScannerBucketScopeResolution<'a> { + requested_scope: ScannerBucketScanScope, + baseline_proof: ScannerCacheBaselineProof<'a>, + activity_before: &'a crate::scanner::ScannerActivitySnapshot, + dirty_usage_snapshot: &'a DirtyUsageSnapshot, + all_buckets: &'a [BucketInfo], +} + +async fn resolve_scanner_bucket_scan_scope( + store: &S, + distributed: bool, + resolution: ScannerBucketScopeResolution<'_>, +) -> ScannerBucketScanScope +where + S: ScannerStorage, +{ + if !resolution.requested_scope.is_default() + || !resolution.dirty_usage_snapshot.covers_all_pending + || resolution.dirty_usage_snapshot.generation == u64::MAX + || resolution.dirty_usage_snapshot.buckets.len() > crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES + { + return resolution.requested_scope; + } + + let mut dirty_buckets = resolution + .dirty_usage_snapshot + .buckets + .keys() + .cloned() + .collect::>(); + if distributed { + let Some(notification_system) = store.scanner_notification_system() else { + return resolution.requested_scope; + }; + let Ok(peer_snapshots) = notification_system.scanner_dirty_usage_snapshots().await else { + return resolution.requested_scope; + }; + let mut expected_peers = HashMap::new(); + for (host, lease_instance_id, _) in crate::scanner::scanner_activity_publication_lease_targets(resolution.activity_before) + { + let Some((activity_instance_id, generation, pending)) = + crate::scanner::scanner_activity_dirty_usage_state_for_host(resolution.activity_before, &host) + else { + return resolution.requested_scope; + }; + if activity_instance_id != lease_instance_id || expected_peers.contains_key(&host) { + return resolution.requested_scope; + } + expected_peers.insert( + host, + ScannerPeerDirtyUsageExpectation { + instance_id: activity_instance_id.to_string(), + generation, + pending, + }, + ); + } + let Some(remote_dirty_buckets) = verified_remote_dirty_usage_buckets(&expected_peers, peer_snapshots) else { + return resolution.requested_scope; + }; + dirty_buckets.extend(remote_dirty_buckets); + } + + scoped_scan_scope_from_dirty_buckets( + resolution.requested_scope, + dirty_buckets, + true, + resolution.all_buckets, + resolution.baseline_proof, + ) } pub(crate) async fn nsscanner_with_storage_status_scoped(store: &S, request: ScannerCycleRequest) -> Result @@ -97,6 +171,7 @@ where leader_epoch, scan_mode, scan_scope, + persisted_usage_baseline, } = request; let child_token = ctx.child_token(); let _tier_cycle_guard = begin_tier_registry_cycle(want_cycle, leader_epoch); @@ -186,8 +261,26 @@ where } bucket_plan_complete &= buckets_by_source.keys().copied().collect::>() == *expected_sources; let scan_plan_digest = - scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_snapshot_digest(&activity_before)); + scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_structural_digest(&activity_before)); let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list)); + let scan_scope = resolve_scanner_bucket_scan_scope( + store, + distributed, + ScannerBucketScopeResolution { + requested_scope: scan_scope, + baseline_proof: ScannerCacheBaselineProof { + data: persisted_usage_baseline.as_ref(), + expected_sources: &expected_sources, + leader_epoch, + want_cycle, + scan_plan_digest, + }, + activity_before: &activity_before, + dirty_usage_snapshot: &dirty_usage_snapshot, + all_buckets: &all_buckets, + }, + ) + .await; let cache_cycle_floor = Arc::new(AtomicU64::new(want_cycle)); let tier_registry = runtime_tier_registry_for_cycle(want_cycle, leader_epoch).await; let tier_registry_generation = tier_registry.generation; diff --git a/crates/scanner/src/scanner_io/publish_gate_tests.rs b/crates/scanner/src/scanner_io/publish_gate_tests.rs index d740cb2eb..f131cf437 100644 --- a/crates/scanner/src/scanner_io/publish_gate_tests.rs +++ b/crates/scanner/src/scanner_io/publish_gate_tests.rs @@ -655,9 +655,31 @@ fn completed_data_usage_info_requires_every_set_before_publish() { .expect("all completed sets should produce a publishable data usage snapshot"); assert_eq!(last_update, SystemTime::UNIX_EPOCH + Duration::from_secs(20)); assert_eq!(data_usage_info.scanner_cycle, Some(0)); + assert_eq!(data_usage_info.scanner_epoch, Some(0)); assert_eq!(data_usage_info.objects_total_count, 3); assert_eq!(data_usage_info.buckets_usage.len(), 3); assert!(data_usage_info.usage_snapshot_complete); + assert_eq!( + data_usage_info + .usage_snapshot_set_states + .iter() + .map(|state| { + ( + state.pool_index, + state.set_index, + state.scanner_cycle, + state.scanner_epoch, + state.scan_plan_digest, + state.complete, + state.tombstone, + ) + }) + .collect::>(), + vec![ + (0, 0, Some(0), Some(0), Some(TEST_PLAN_DIGEST.0), true, false), + (1, 0, Some(0), Some(0), Some(TEST_PLAN_DIGEST.0), true, false), + ] + ); assert_eq!( data_usage_info .buckets_usage diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index 6fddafdfe..ec2c1ad65 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -17,6 +17,7 @@ use super::io_disk::tier_stats_template; use super::*; use crate::scanner_budget::ScannerCycleBudgetConfig; use crate::scanner_folder::ScannerItem; +use crate::storage_api::EcstoreScannerPeerDirtyUsageSnapshot; use crate::storage_api::owner::{ EcstorePoolDecommissionInfo, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats, }; @@ -796,6 +797,195 @@ fn complete_set_usage_cache(buckets: &[(&str, usize)], scan_plan_digest: DataUsa cache } +fn complete_usage_baseline( + source: DataUsageCacheSource, + scan_plan_digest: DataUsageScanPlanDigest, + scanner_cycle: u64, + scanner_epoch: u64, +) -> bytes::Bytes { + let baseline = DataUsageInfo { + last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)), + scanner_cycle: Some(scanner_cycle), + scanner_epoch: Some(scanner_epoch), + buckets_count: 1, + buckets_usage: HashMap::from([("photos".to_string(), Default::default())]), + usage_snapshot_complete: true, + usage_snapshot_converged: Some(true), + usage_snapshot_set_states: vec![DataUsageSnapshotSetState { + pool_index: u64::try_from(source.pool_index).expect("test pool index should fit"), + set_index: u64::try_from(source.set_index).expect("test set index should fit"), + scanner_cycle: Some(scanner_cycle), + scanner_epoch: Some(scanner_epoch), + scan_plan_digest: Some(scan_plan_digest.0), + complete: true, + tombstone: false, + }], + ..Default::default() + }; + bytes::Bytes::from(serde_json::to_vec(&baseline).expect("test baseline should encode")) +} + +#[test] +fn scoped_scan_requires_a_converged_complete_baseline_with_exact_set_provenance() { + let source = DataUsageCacheSource::new(1, 2); + let expected_sources = HashSet::from([source]); + let scan_plan_digest = DataUsageScanPlanDigest([9; 32]); + let baseline = complete_usage_baseline(source, scan_plan_digest, 7, 11); + + assert_eq!( + complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof { + data: Some(&baseline), + expected_sources: &expected_sources, + leader_epoch: 11, + want_cycle: 8, + scan_plan_digest, + }), + Some(scan_plan_digest) + ); + + let mut incomplete = serde_json::from_slice::(&baseline).expect("test baseline should decode"); + incomplete.usage_snapshot_converged = Some(false); + let incomplete = bytes::Bytes::from(serde_json::to_vec(&incomplete).expect("test baseline should encode")); + assert_eq!( + complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof { + data: Some(&incomplete), + expected_sources: &expected_sources, + leader_epoch: 11, + want_cycle: 8, + scan_plan_digest, + }), + None + ); + + let mut wrong_provenance = serde_json::from_slice::(&baseline).expect("test baseline should decode"); + wrong_provenance.usage_snapshot_set_states[0].scan_plan_digest = Some([8; 32]); + let wrong_provenance = bytes::Bytes::from(serde_json::to_vec(&wrong_provenance).expect("test baseline should encode")); + assert_eq!( + complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof { + data: Some(&wrong_provenance), + expected_sources: &expected_sources, + leader_epoch: 11, + want_cycle: 8, + scan_plan_digest, + }), + None + ); +} + +#[test] +fn scoped_scan_selects_only_current_dirty_buckets_after_baseline_validation() { + let source = DataUsageCacheSource::new(1, 2); + let expected_sources = HashSet::from([source]); + let baseline_scan_plan_digest = DataUsageScanPlanDigest([4; 32]); + let current_scan_plan_digest = DataUsageScanPlanDigest([5; 32]); + let baseline = complete_usage_baseline(source, current_scan_plan_digest, 7, 11); + let scope = scoped_scan_scope_from_dirty_buckets( + ScannerBucketScanScope::default(), + HashSet::from(["photos".to_string(), "deleted".to_string()]), + true, + &[bucket_info("photos")], + ScannerCacheBaselineProof { + data: Some(&baseline), + expected_sources: &expected_sources, + leader_epoch: 11, + want_cycle: 8, + scan_plan_digest: current_scan_plan_digest, + }, + ); + + assert_eq!(scope.baseline_scan_plan_digest, Some(current_scan_plan_digest)); + assert_eq!( + scope + .selected_buckets + .as_deref() + .expect("validated scope should select a bucket"), + &HashSet::from(["photos".to_string()]) + ); + assert_ne!(scope.baseline_scan_plan_digest, Some(baseline_scan_plan_digest)); +} + +fn peer_dirty_usage_snapshot( + instance_id: &str, + generation: u64, + complete: bool, + buckets: &[(&str, u64)], +) -> EcstoreScannerPeerDirtyUsageSnapshot { + EcstoreScannerPeerDirtyUsageSnapshot { + instance_id: instance_id.to_string(), + generation, + pending_bucket_count: u64::try_from(buckets.len()).expect("test bucket count should fit"), + protocol_version: crate::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, + complete, + buckets: buckets + .iter() + .map(|(bucket, generation)| ((*bucket).to_string(), *generation)) + .collect(), + } +} + +#[test] +fn verified_remote_dirty_usage_buckets_merges_only_complete_current_snapshots() { + let expected_peers = HashMap::from([ + ( + "node-a:9000".to_string(), + ScannerPeerDirtyUsageExpectation { + instance_id: "instance-a".to_string(), + generation: 7, + pending: true, + }, + ), + ( + "node-b:9000".to_string(), + ScannerPeerDirtyUsageExpectation { + instance_id: "instance-b".to_string(), + generation: 3, + pending: false, + }, + ), + ]); + + assert_eq!( + verified_remote_dirty_usage_buckets( + &expected_peers, + vec![ + ( + "node-a:9000".to_string(), + peer_dirty_usage_snapshot("instance-a", 7, true, &[("photos", 7)]), + ), + ( + "node-b:9000".to_string(), + peer_dirty_usage_snapshot("instance-b", 3, true, &[("archive", 3)]), + ), + ], + ), + Some(HashSet::from(["photos".to_string(), "archive".to_string()])) + ); +} + +#[test] +fn verified_remote_dirty_usage_buckets_rejects_incomplete_or_stale_peer_state() { + let expected_peers = HashMap::from([( + "node-a:9000".to_string(), + ScannerPeerDirtyUsageExpectation { + instance_id: "instance-a".to_string(), + generation: 7, + pending: true, + }, + )]); + + for snapshot in [ + peer_dirty_usage_snapshot("instance-a", 7, false, &[("photos", 7)]), + peer_dirty_usage_snapshot("instance-a", 6, true, &[("photos", 6)]), + peer_dirty_usage_snapshot("instance-b", 7, true, &[("photos", 7)]), + peer_dirty_usage_snapshot("instance-a", 7, true, &[]), + ] { + assert!( + verified_remote_dirty_usage_buckets(&expected_peers, vec![("node-a:9000".to_string(), snapshot)]).is_none(), + "incomplete, stale, mismatched, or empty pending peer state must fall back to a full scan" + ); + } +} + #[test] fn scoped_set_scan_preserves_unselected_usage_and_drops_deleted_buckets() { let baseline_digest = DataUsageScanPlanDigest([1; 32]); diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index a0aa874b0..f81065d59 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -103,7 +103,9 @@ pub(crate) use rustfs_ecstore::api::rebalance::{ RebalStatus as EcstoreRebalStatus, RebalanceInfo as EcstoreRebalanceInfo, RebalanceMeta as EcstoreRebalanceMeta, RebalanceStats as EcstoreRebalanceStats, }; -pub(crate) use rustfs_ecstore::api::rpc::ScannerBucketListing as EcstoreScannerBucketListing; +pub(crate) use rustfs_ecstore::api::rpc::{ + ScannerBucketListing as EcstoreScannerBucketListing, ScannerPeerDirtyUsageSnapshot as EcstoreScannerPeerDirtyUsageSnapshot, +}; #[cfg(test)] pub(crate) use rustfs_ecstore::api::runtime::InstanceContext as EcstoreInstanceContext; pub(crate) use rustfs_ecstore::api::runtime::{ From cf9688898d9fc9ae11d6b16793f355435618d122 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 5 Sep 2026 16:39:26 +0800 Subject: [PATCH 07/12] fix(heal): retain completed task progress (#7177) * chore(deps): refresh SDKs and pin clock skew regression coverage Refresh compatible dependencies for Scanner/Heal V2 batch 1 and verify the production S3 retry/signing path with a deterministic clock. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(heal): retain completed task progress Refs rustfs/backlog#2262 and rustfs/backlog#2240. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/heal/src/heal/manager.rs | 72 ++++- crates/heal/src/heal/manager/queue.rs | 129 ++++++++ crates/heal/src/heal/manager/scheduler.rs | 114 ++++--- crates/heal/src/heal/manager/tests.rs | 351 +++++++++++++++++++++- crates/heal/src/heal/task.rs | 2 +- 5 files changed, 613 insertions(+), 55 deletions(-) diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index d36c9f9a7..971590fdb 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -45,6 +45,11 @@ use tracing::{debug, error, info, warn}; use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read}; const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60); +// Each cache includes alias tokens in its count and byte budget. Eviction +// removes every token sharing a snapshot; neither cache retains repair state. +const MAX_COMPLETED_HEAL_TOKENS: usize = 1024; +const MAX_COMPLETED_HEAL_BYTES: usize = 64 * 1024 * 1024; +const MAX_COMPLETED_HEAL_RESULT_BYTES: usize = 1024 * 1024; const DISPLACED_HEAL_REASON: &str = "reason=displaced; retry_hint=submit_again"; const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_SUBSYSTEM_DISK_SCANNER: &str = "disk_scanner"; @@ -180,6 +185,8 @@ fn record_displaced_terminal( request: &HealRequest, ) -> Arc { let terminal = Arc::new(CompletedHealStatus { + progress: None, + retained_bytes: std::sync::OnceLock::new(), heal_type: request.heal_type.clone(), status: HealTaskStatus::Failed { error: format!("heal task displaced by a higher-priority request ({DISPLACED_HEAL_REASON})"), @@ -193,6 +200,7 @@ fn record_displaced_terminal( let mut terminals = lock_displaced_terminals(registry); prune_completed_heal_statuses(&mut terminals); terminals.insert(request.id.clone(), Arc::clone(&terminal)); + prune_completed_heal_statuses(&mut terminals); terminal } @@ -209,9 +217,15 @@ async fn remove_displaced_task_aliases( .collect::>(); let mut displaced_terminals = lock_displaced_terminals(terminals); prune_completed_heal_statuses(&mut displaced_terminals); - for alias_id in alias_ids { - displaced_terminals.insert(alias_id, Arc::clone(terminal)); + if displaced_terminals + .get(task_id) + .is_some_and(|current| Arc::ptr_eq(current, terminal)) + { + for alias_id in alias_ids { + displaced_terminals.insert(alias_id, Arc::clone(terminal)); + } } + prune_completed_heal_statuses(&mut displaced_terminals); aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id); } @@ -222,6 +236,36 @@ async fn remove_task_aliases_for_task(registry: &Arc +// retrying (when needed) -> aliases -> completed; queries release aliases +// before looking up active state. Publishing aliases before removing their +// mapping keeps both an already-resolved token and a new lookup valid. +async fn publish_completed_heal( + completed_heals: &Mutex>>, + task_aliases: &Mutex>, + task_id: &str, + completed: CompletedHealStatus, + terminal: bool, +) { + let completed = Arc::new(completed); + completed.retained_bytes(); + let mut aliases = task_aliases.lock().await; + let mut retained = completed_heals.lock().await; + if let Some(previous) = retained.get(task_id).cloned() { + for entry in retained.values_mut().filter(|entry| Arc::ptr_eq(entry, &previous)) { + *entry = Arc::clone(&completed); + } + } + retained.insert(task_id.to_owned(), Arc::clone(&completed)); + if terminal { + for (alias_id, _) in aliases.iter().filter(|(_, alias)| alias.task_id == task_id) { + retained.insert(alias_id.clone(), Arc::clone(&completed)); + } + aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id); + } + prune_completed_heal_statuses(&mut retained); +} + #[derive(Debug, Clone)] pub struct HealTaskReport { pub status: HealTaskStatus, @@ -268,7 +312,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option) -> let result_items = match since { None => completed.seqed_items.iter().map(|(_, item)| item.clone()).collect(), Some(cursor) => { - if cursor + 1 < completed.min_seq { + if cursor.saturating_add(1) < completed.min_seq { lagged = true; } completed @@ -283,7 +327,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option) -> status: completed.status.clone(), result_items, result_items_truncated: completed.result_items_truncated || lagged, - progress: None, + progress: completed.progress.clone(), next_seq: completed.next_seq, min_seq: completed.min_seq, } @@ -1847,14 +1891,14 @@ impl HealManager { pub async fn get_task_progress(&self, task_id: &str) -> Result { let canonical_task_id = self.canonical_task_id(task_id).await; - let active_heals = self.active_heals.lock().await; - if let Some(task) = active_heals.get(&canonical_task_id) { - Ok(task.get_progress().await) - } else { - Err(Error::TaskNotFound { - task_id: task_id.to_string(), - }) - } + let progress = match self.lookup_task_state(&canonical_task_id, None).await { + TaskStateLookup::Active(task) => Some(task.get_progress().await), + TaskStateLookup::Completed(completed) => completed.progress.clone(), + _ => None, + }; + progress.ok_or_else(|| Error::TaskNotFound { + task_id: task_id.to_string(), + }) } /// Cancel task @@ -1864,6 +1908,8 @@ impl HealManager { let mut active_heals = self.active_heals.lock().await; if let Some(task) = active_heals.get(&canonical_task_id) { task.cancel().await?; + let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await; + publish_completed_heal(&self.completed_heals, &self.task_aliases, &canonical_task_id, completed, true).await; active_heals.remove(&canonical_task_id); publish_active_heal_count(&active_heals); info!( @@ -1940,6 +1986,8 @@ impl HealManager { for task_id in &task_ids { if let Some(task) = active_heals.get(task_id) { task.cancel().await?; + let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await; + publish_completed_heal(&self.completed_heals, &self.task_aliases, task_id, completed, true).await; } active_heals.remove(task_id); cancelled += 1; diff --git a/crates/heal/src/heal/manager/queue.rs b/crates/heal/src/heal/manager/queue.rs index b4608f27b..aceabe42a 100644 --- a/crates/heal/src/heal/manager/queue.rs +++ b/crates/heal/src/heal/manager/queue.rs @@ -82,6 +82,8 @@ pub(super) enum QueuePushOutcome { pub(super) struct CompletedHealStatus { pub(super) heal_type: HealType, pub(super) status: HealTaskStatus, + pub(super) progress: Option, + pub(super) retained_bytes: std::sync::OnceLock, pub(super) result_items_truncated: bool, pub(super) completed_at: SystemTime, /// Sequence-stamped retained window, archived with the completion so @@ -92,6 +94,133 @@ pub(super) struct CompletedHealStatus { pub(super) min_seq: u64, } +impl CompletedHealStatus { + // Account for owned capacities, including nested drive arrays. Aliases + // conservatively charge the shared allocation again, keeping both token + // count and retained payload bounded without a second ownership index. + pub(super) fn retained_bytes(&self) -> usize { + *self.retained_bytes.get_or_init(|| self.measure_retained_bytes()) + } + + fn measure_retained_bytes(&self) -> usize { + let mut bytes = size_of::(); + let mut add = |amount: usize| bytes = bytes.saturating_add(amount); + match &self.heal_type { + HealType::Cluster => {} + HealType::Bucket { bucket } => add(bucket.capacity()), + HealType::Object { + bucket, + object, + version_id, + } + | HealType::ECDecode { + bucket, + object, + version_id, + } => { + add(bucket.capacity()); + add(object.capacity()); + add(version_id.as_ref().map_or(0, String::capacity)); + } + HealType::Prefix { bucket, prefix } => { + add(bucket.capacity()); + add(prefix.capacity()); + } + HealType::Metadata { bucket, object } => { + add(bucket.capacity()); + add(object.capacity()); + } + HealType::ErasureSet { buckets, set_disk_id } => { + add(buckets.capacity().saturating_mul(size_of::())); + for bucket in buckets { + add(bucket.capacity()); + } + add(set_disk_id.capacity()); + } + } + if let HealTaskStatus::Failed { error } | HealTaskStatus::Retrying { error, .. } = &self.status { + add(error.capacity()); + } + add(self + .progress + .as_ref() + .and_then(|progress| progress.current_object.as_ref()) + .map_or(0, String::capacity)); + add(self.seqed_items.capacity().saturating_mul(size_of::<(u64, HealResultItem)>())); + for (_, item) in &self.seqed_items { + add(Self::result_item_heap_bytes(item)); + } + bytes + } + + fn result_item_heap_bytes(item: &HealResultItem) -> usize { + let mut bytes = 0usize; + let mut add = |amount: usize| bytes = bytes.saturating_add(amount); + for value in [ + &item.heal_item_type, + &item.bucket, + &item.object, + &item.version_id, + &item.detail, + ] { + add(value.capacity()); + } + for infos in [&item.before, &item.after] { + add(infos + .drives + .capacity() + .saturating_mul(size_of::())); + for drive in &infos.drives { + add(drive.uuid.capacity()); + add(drive.endpoint.capacity()); + add(drive.state.capacity()); + } + } + bytes + } + + pub(super) fn bound_result_window(&mut self) { + let mut bytes = 0usize; + let retained = self + .seqed_items + .iter() + .rev() + .take_while(|(_, item)| { + bytes = bytes + .saturating_add(size_of::<(u64, HealResultItem)>()) + .saturating_add(Self::result_item_heap_bytes(item)); + bytes <= MAX_COMPLETED_HEAL_RESULT_BYTES + }) + .count(); + let truncated = retained < self.seqed_items.len(); + if truncated { + self.seqed_items.drain(..self.seqed_items.len() - retained); + self.seqed_items.shrink_to_fit(); + self.min_seq = self.seqed_items.first().map_or(self.next_seq, |(seq, _)| *seq); + self.result_items_truncated = true; + self.retained_bytes.take(); + } + } + + pub(super) async fn snapshot(task: &HealTask, status: HealTaskStatus) -> Self { + let seqed_items = task.get_seqed_result_items().await; + let (next_seq, min_seq) = task.result_seq_cursors(); + let mut snapshot = Self { + heal_type: task.heal_type.clone(), + status, + progress: Some(task.get_progress().await), + retained_bytes: std::sync::OnceLock::new(), + result_items_truncated: task.result_items_truncated(), + completed_at: SystemTime::now(), + seqed_items, + next_seq, + min_seq, + }; + snapshot.bound_result_window(); + snapshot + } +} + #[derive(Debug, Clone)] pub(super) struct HealTaskAlias { pub(super) task_id: String, diff --git a/crates/heal/src/heal/manager/scheduler.rs b/crates/heal/src/heal/manager/scheduler.rs index c0deae524..feb0c1231 100644 --- a/crates/heal/src/heal/manager/scheduler.rs +++ b/crates/heal/src/heal/manager/scheduler.rs @@ -264,7 +264,7 @@ impl HealManager { error: error.clone(), retry_attempt: request.retry_attempts, }); - let retry_request_for_queue = retry_request; + let mut retry_request_for_queue = retry_request; let retry_cancel_token = retry_request_for_queue.as_ref().map(|_| CancellationToken::new()); if retry_request_for_queue.is_none() { replacement_recovery_anchors_clone @@ -272,7 +272,35 @@ impl HealManager { .unwrap_or_else(|poisoned| poisoned.into_inner()) .remove(&task_id); } + let mut completed_status = match retry_request_for_status { + Some(status) => status, + None => task.get_status().await, + }; + let mut completed_status_entry = CompletedHealStatus::snapshot(&task, completed_status.clone()).await; + let completed_progress = task.get_progress().await; + #[cfg(test)] + tests::pause_completed_retention_before_publish(&task_id, &completed_status).await; let mut active_heals_guard = active_heals_clone.lock().await; + let owns_completion = active_heals_guard.contains_key(&task_id); + let cancelled_completion = if owns_completion { + false + } else { + // Cancellation can win while a finished worker waits + // for active ownership. It must not resurrect a retry + // or replace an acknowledged cancellation with success. + retry_request_for_queue = None; + completed_heals_clone + .lock() + .await + .get(&task_id) + .is_some_and(|completed| completed.status == HealTaskStatus::Cancelled) + }; + if cancelled_completion { + completed_status = HealTaskStatus::Cancelled; + completed_status_entry.status = HealTaskStatus::Cancelled; + } + let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. }); + let successful_completion = matches!(completed_status, HealTaskStatus::Completed); // Keep retry ownership continuous: status snapshots acquire // these locks in the same active -> retrying order. let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) = @@ -295,6 +323,16 @@ impl HealManager { } else { None }; + if owns_completion || cancelled_completion { + publish_completed_heal( + &completed_heals_clone, + &task_aliases_clone, + &task_id, + completed_status_entry, + terminal_completion, + ) + .await; + } let completed_task = active_heals_guard.remove(&task_id); if let Some(completed_task) = completed_task.as_ref() { publish_active_heal_count(&active_heals_guard); @@ -304,33 +342,10 @@ impl HealManager { drop(retrying_heals_guard.take()); drop(active_heals_guard); - if let Some(completed_task) = completed_task { - let completed_status = if let Some(status) = retry_request_for_status { - status - } else { - completed_task.get_status().await - }; - let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. }); - let successful_completion = matches!(completed_status, HealTaskStatus::Completed); - let completed_progress = completed_task.get_progress().await; - // Single snapshot of the retained window: the task is - // finished and already off the active map, so there is - // no concurrent writer to race with. - let seqed_items = completed_task.get_seqed_result_items().await; - let (next_seq, min_seq) = completed_task.result_seq_cursors(); - let completed_status_entry = CompletedHealStatus { - heal_type: completed_task.heal_type.clone(), - status: completed_status.clone(), - result_items_truncated: completed_task.result_items_truncated(), - completed_at: SystemTime::now(), - seqed_items, - next_seq, - min_seq, - }; - let mut completed_heals_guard = completed_heals_clone.lock().await; - prune_completed_heal_statuses(&mut completed_heals_guard); - completed_heals_guard.insert(task_id.clone(), Arc::new(completed_status_entry)); - drop(completed_heals_guard); + #[cfg(test)] + tests::pause_completed_retention_handoff(&task_id).await; + + if completed_task.is_some() { // update statistics let mut stats = statistics_clone.write().await; match completed_status { @@ -352,10 +367,6 @@ impl HealManager { } else { release_mrf_repair_notice_targets(notice_targets); } - task_aliases_clone - .lock() - .await - .retain(|alias_id, alias| alias_id != &task_id && alias.task_id != task_id); } } @@ -718,17 +729,42 @@ pub(super) fn heal_request_set_key_for_task(task: &HealTask) -> Option { } pub(super) fn prune_completed_heal_statuses(completed_heals: &mut HashMap>) { - let Ok(now) = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) else { - return; - }; + prune_completed_heal_statuses_at(completed_heals, SystemTime::now()); +} +pub(super) fn prune_completed_heal_statuses_at(completed_heals: &mut HashMap>, now: SystemTime) { completed_heals.retain(|_, completed| { - completed - .completed_at - .duration_since(SystemTime::UNIX_EPOCH) - .map(|completed_at| now.saturating_sub(completed_at) <= KEEP_HEAL_TASK_STATUS_DURATION) + now.duration_since(completed.completed_at) + .map(|age| age <= KEEP_HEAL_TASK_STATUS_DURATION) .unwrap_or(false) }); + let entry_bytes = |key: &String, value: &Arc| { + key.capacity() + .saturating_add(size_of::<(String, Arc)>()) + .saturating_add(value.retained_bytes()) + }; + let mut bytes = completed_heals + .iter() + .fold(0usize, |total, (key, value)| total.saturating_add(entry_bytes(key, value))); + while completed_heals.len() > MAX_COMPLETED_HEAL_TOKENS || bytes > MAX_COMPLETED_HEAL_BYTES { + let Some(oldest) = completed_heals + .iter() + .min_by(|(left_id, left), (right_id, right)| { + left.completed_at.cmp(&right.completed_at).then_with(|| left_id.cmp(right_id)) + }) + .map(|(_, value)| Arc::clone(value)) + else { + break; + }; + completed_heals.retain(|key, value| { + if Arc::ptr_eq(value, &oldest) { + bytes = bytes.saturating_sub(entry_bytes(key, value)); + false + } else { + true + } + }); + } } pub(super) fn can_schedule_request( diff --git a/crates/heal/src/heal/manager/tests.rs b/crates/heal/src/heal/manager/tests.rs index 37a100f4b..aa1c1293f 100644 --- a/crates/heal/src/heal/manager/tests.rs +++ b/crates/heal/src/heal/manager/tests.rs @@ -101,6 +101,326 @@ async fn process_manager_queue_once(manager: &HealManager) { struct MockStorage; +fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus { + CompletedHealStatus { + heal_type: HealType::Cluster, + status: HealTaskStatus::Completed, + progress: Some(HealProgress { + objects_scanned: 9, + objects_healed: 8, + objects_failed: 1, + ..Default::default() + }), + retained_bytes: std::sync::OnceLock::new(), + result_items_truncated: false, + completed_at, + seqed_items: vec![(3, HealResultItem::default()), (4, HealResultItem::default())], + next_seq: 5, + min_seq: 3, + } +} + +#[test] +fn completed_retention_cursor_boundaries_preserve_progress() { + let completed = completed_retention_fixture(SystemTime::now()); + for (cursor, count, lagged) in [ + (0, 2, true), + (1, 2, true), + (2, 2, false), + (3, 1, false), + (4, 0, false), + (5, 0, false), + (u64::MAX, 0, false), + ] { + let report = completed_task_report(&completed, Some(cursor)); + assert_eq!(report.result_items.len(), count, "cursor={cursor}"); + assert_eq!(report.result_items_truncated, lagged, "cursor={cursor}"); + assert_eq!(report.progress, completed.progress); + assert_eq!((report.next_seq, report.min_seq), (5, 3)); + } + assert_eq!(completed_task_report(&completed, None).result_items.len(), 2); +} + +#[tokio::test] +async fn completed_retention_displaced_alias_does_not_resurrect_evicted_snapshot() { + let manager = HealManager::new(Arc::new(MockStorage), None); + let request = HealRequest::bucket("bucket".to_string()); + manager.insert_task_alias("alias", &request.id).await; + let terminal = record_displaced_terminal(&manager.displaced_terminals, &request); + lock_displaced_terminals(&manager.displaced_terminals).remove(&request.id); + remove_displaced_task_aliases(&manager.task_aliases, &manager.displaced_terminals, &request.id, &terminal).await; + for token in [&request.id, &"alias".to_string()] { + assert!(matches!(manager.get_task_report(token).await, Err(Error::TaskNotFound { .. }))); + } + assert!(manager.task_aliases.lock().await.is_empty()); + assert!(lock_displaced_terminals(&manager.displaced_terminals).is_empty()); +} + +#[test] +fn completed_retention_count_ttl_and_alias_eviction_are_bounded() { + let now = SystemTime::now(); + let mut entries = HashMap::new(); + let oldest = Arc::new(completed_retention_fixture(now - KEEP_HEAL_TASK_STATUS_DURATION)); + entries.insert("oldest".to_string(), Arc::clone(&oldest)); + entries.insert("oldest-alias".to_string(), Arc::clone(&oldest)); + for index in 2..MAX_COMPLETED_HEAL_TOKENS { + entries.insert(format!("task-{index}"), Arc::new(completed_retention_fixture(now))); + } + prune_completed_heal_statuses_at(&mut entries, now); + assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS); + entries.insert("cap-plus-one".to_string(), Arc::new(completed_retention_fixture(now))); + prune_completed_heal_statuses_at(&mut entries, now); + assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS - 1); + assert!(!entries.contains_key("oldest")); + assert!(!entries.contains_key("oldest-alias")); + entries.clear(); + entries.insert("ttl-boundary".to_string(), oldest); + entries.insert( + "expired".to_string(), + Arc::new(completed_retention_fixture( + now - KEEP_HEAL_TASK_STATUS_DURATION - Duration::from_nanos(1), + )), + ); + entries.insert("future".to_string(), Arc::new(completed_retention_fixture(now + Duration::from_nanos(1)))); + prune_completed_heal_statuses_at(&mut entries, now); + assert_eq!(entries.len(), 1); + assert!(entries.contains_key("ttl-boundary")); + prune_completed_heal_statuses_at(&mut entries, now + Duration::from_nanos(1)); + assert!(entries.is_empty()); +} + +#[test] +fn completed_retention_total_byte_cap_and_cap_plus_one() { + let now = SystemTime::now(); + let key = "large".to_string(); + let mut entry = completed_retention_fixture(now); + let base_bytes = entry.retained_bytes() + key.capacity() + size_of::<(String, Arc)>(); + entry.retained_bytes.take(); + entry.status = HealTaskStatus::Failed { + error: "x".repeat(MAX_COMPLETED_HEAL_BYTES - base_bytes), + }; + assert_eq!( + entry.retained_bytes() + key.capacity() + size_of::<(String, Arc)>(), + MAX_COMPLETED_HEAL_BYTES + ); + let mut entries = HashMap::from([(key, Arc::new(entry))]); + prune_completed_heal_statuses_at(&mut entries, now); + assert_eq!(entries.len(), 1, "exact byte cap remains retained"); + let mut over = Arc::try_unwrap(entries.remove("large").expect("entry retained")).expect("entry not shared"); + over.retained_bytes.take(); + if let HealTaskStatus::Failed { error } = &mut over.status { + *error = "x".repeat(error.len() + 1); + } + entries.insert("large".to_string(), Arc::new(over)); + prune_completed_heal_statuses_at(&mut entries, now); + assert!(entries.is_empty(), "oversized metadata cannot escape total byte bound"); +} + +#[tokio::test] +async fn completed_retention_large_window_keeps_cursors_and_progress() { + let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), Arc::new(MockStorage)); + let mut snapshot = completed_retention_fixture(SystemTime::now()); + snapshot.seqed_items[0].1.detail = "x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES); + snapshot.bound_result_window(); + assert_eq!(snapshot.seqed_items.len(), 1); + assert_eq!((snapshot.min_seq, snapshot.next_seq), (4, 5)); + assert!(snapshot.result_items_truncated); + assert!(snapshot.retained_bytes() < MAX_COMPLETED_HEAL_RESULT_BYTES); + let report = completed_task_report(&snapshot, Some(0)); + assert_eq!(report.progress.expect("progress retained").objects_scanned, 9); + assert!(report.result_items_truncated); + let active_max = task.get_result_items_since(Some(u64::MAX)).await; + assert!(active_max.items.is_empty()); + assert!(!active_max.lagged); +} + +#[test] +fn completed_retention_result_byte_cap_and_cap_plus_one() { + for extra in [0, 1] { + let mut snapshot = completed_retention_fixture(SystemTime::now()); + snapshot.seqed_items = vec![( + 4, + HealResultItem { + detail: "x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES - size_of::<(u64, HealResultItem)>() + extra), + ..Default::default() + }, + )]; + snapshot.min_seq = 4; + snapshot.bound_result_window(); + assert_eq!(snapshot.seqed_items.len(), 1 - extra); + assert_eq!(snapshot.result_items_truncated, extra == 1); + assert_eq!(snapshot.min_seq, if extra == 0 { 4 } else { 5 }); + assert_eq!(snapshot.next_seq, 5); + assert_eq!(snapshot.progress.as_ref().expect("progress retained").objects_scanned, 9); + } +} + +#[derive(Default)] +struct CompletedRetentionHook { + started: Notify, + execute: Notify, + handoff: Notify, + finish: Notify, + pause_before_publish: bool, + before_publish: Notify, + publish: Notify, + prepared_status: Mutex>, +} + +static COMPLETED_RETENTION_HOOKS: LazyLock>>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +pub(super) async fn pause_completed_retention_handoff(task_id: &str) { + let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(task_id).cloned(); + if let Some(hook) = hook { + hook.handoff.notify_one(); + hook.finish.notified().await; + } +} + +pub(super) async fn pause_completed_retention_before_publish(task_id: &str, status: &HealTaskStatus) { + let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(task_id).cloned(); + if let Some(hook) = hook.filter(|hook| hook.pause_before_publish) { + *hook.prepared_status.lock().await = Some(status.clone()); + hook.before_publish.notify_one(); + hook.publish.notified().await; + } +} + +#[tokio::test] +async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() { + let bucket = "completed-retention-retry-cancel"; + let manager = HealManager::new(Arc::new(MockStorage), None); + let request = HealRequest::object(bucket.to_string(), "object".to_string(), None); + let task_id = request.id.clone(); + let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None); + let alias = duplicate.id.clone(); + let hook = Arc::new(CompletedRetentionHook { + pause_before_publish: true, + ..Default::default() + }); + { + let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await; + hooks.insert(bucket.to_string(), Arc::clone(&hook)); + hooks.insert(task_id.clone(), Arc::clone(&hook)); + } + manager.submit_heal_request(request).await.expect("admit original"); + manager.submit_heal_request(duplicate).await.expect("admit alias"); + process_manager_queue_once(&manager).await; + tokio::time::timeout(Duration::from_secs(5), hook.started.notified()) + .await + .expect("scheduler starts"); + let task = manager.active_heals.lock().await.get(&task_id).cloned().expect("active task"); + task.progress.write().await.update_object_progress(1, 1, 0, 0, 4096); + hook.execute.notify_one(); + tokio::time::timeout(Duration::from_secs(5), hook.before_publish.notified()) + .await + .expect("retry snapshot prepared"); + manager.cancel_task(&alias).await.expect("cancel wins active ownership"); + assert!(matches!(*hook.prepared_status.lock().await, Some(HealTaskStatus::Retrying { .. }))); + hook.publish.notify_one(); + tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified()) + .await + .expect("scheduler finishes handoff"); + for token in [&task_id, &alias] { + let report = manager.get_task_report(token).await.expect("cancelled token retained"); + assert_eq!(report.status, HealTaskStatus::Cancelled); + assert_eq!(report.progress.expect("frozen progress").objects_scanned, 1); + } + assert!(!manager.retrying_heals.lock().await.contains_key(&task_id)); + assert!(!manager.heal_queue.lock().await.contains_request_id(&task_id)); + hook.finish.notify_one(); + COMPLETED_RETENTION_HOOKS + .lock() + .await + .retain(|key, _| key != bucket && key != &task_id); +} + +#[tokio::test] +async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_handoff() { + for outcome in ["success", "failed", "cancelled"] { + let bucket = format!("completed-retention-{outcome}"); + let hook = Arc::new(CompletedRetentionHook::default()); + let manager = Arc::new(HealManager::new(Arc::new(MockStorage), None)); + let request = HealRequest::object(bucket.clone(), "object".to_string(), None); + let task_id = request.id.clone(); + let duplicate = HealRequest::object(bucket.clone(), "object".to_string(), None); + let alias = duplicate.id.clone(); + { + let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await; + hooks.insert(bucket.clone(), Arc::clone(&hook)); + hooks.insert(task_id.clone(), Arc::clone(&hook)); + } + manager.submit_heal_request(request).await.expect("admit original"); + manager.submit_heal_request(duplicate).await.expect("admit alias"); + process_manager_queue_once(&manager).await; + tokio::time::timeout(Duration::from_secs(5), hook.started.notified()) + .await + .expect("scheduler reaches storage"); + let task = manager + .active_heals + .lock() + .await + .get(&task_id) + .cloned() + .expect("task is active"); + task.progress.write().await.update_object_progress(1, 1, 0, 0, 4096); + let before = manager.get_task_report(&alias).await.expect("alias resolves active progress"); + assert_eq!(before.progress.as_ref().expect("active progress").objects_scanned, 1); + let poll_manager = Arc::clone(&manager); + let poll_alias = alias.clone(); + let stop = CancellationToken::new(); + let poll_stop = stop.clone(); + let polling = tokio::spawn(async move { + while !poll_stop.is_cancelled() { + let report = poll_manager + .get_task_report(&poll_alias) + .await + .expect("handoff must never return NotFound"); + assert!(report.progress.expect("progress never disappears").objects_scanned >= 1); + tokio::task::yield_now().await; + } + }); + if outcome == "cancelled" { + manager.cancel_task(&alias).await.expect("cancel active task by alias"); + } else { + hook.execute.notify_one(); + } + tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified()) + .await + .expect("scheduler archives terminal"); + assert!(!manager.active_heals.lock().await.contains_key(&task_id)); + let expected = task.get_progress().await; + for token in [&task_id, &alias] { + assert_eq!(manager.get_task_progress(token).await.expect("terminal progress query"), expected); + let report = manager + .get_task_report_for_path_since(&format!("{bucket}/object"), token, Some(u64::MAX)) + .await + .expect("terminal token remains queryable at handoff"); + assert_eq!(report.progress.as_ref(), Some(&expected)); + assert!(report.result_items.is_empty()); + match outcome { + "success" => assert_eq!(report.status, HealTaskStatus::Completed), + "failed" => assert!(matches!(report.status, HealTaskStatus::Failed { .. })), + _ => assert_eq!(report.status, HealTaskStatus::Cancelled), + } + } + let retained = manager.completed_heals.lock().await; + assert!(Arc::ptr_eq(&retained[&task_id], &retained[&alias])); + drop(retained); + stop.cancel(); + polling.await.expect("concurrent polling succeeds"); + // Archived progress must not alias a mutable live progress object. + task.progress.write().await.objects_scanned = 999; + assert_eq!(manager.get_task_report(&alias).await.expect("frozen report").progress, Some(expected)); + hook.finish.notify_one(); + COMPLETED_RETENTION_HOOKS + .lock() + .await + .retain(|key, _| key != &bucket && key != &task_id); + } +} + #[async_trait::async_trait] impl HealStorageAPI for MockStorage { async fn get_object_meta(&self, _bucket: &str, _object: &str) -> Result> { @@ -123,6 +443,12 @@ impl HealStorageAPI for MockStorage { } async fn object_exists(&self, bucket: &str, _object: &str) -> Result { + let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(bucket).cloned(); + if let Some(hook) = hook { + hook.started.notify_one(); + hook.execute.notified().await; + return Ok(true); + } Ok(bucket == "retry-transition") } @@ -133,13 +459,18 @@ impl HealStorageAPI for MockStorage { _version_id: Option<&str>, _opts: &HealOpts, ) -> Result<(HealResultItem, Option)> { + if bucket == "completed-retention-failed" { + return Err(Error::TaskExecutionFailed { + message: "retention fixture failure".to_string(), + }); + } if let Some(hook) = manager_recovery_test_hook() { *hook .heal_object_calls .lock() .expect("manager recovery object call lock should not poison") += 1; } - if bucket == "retry-transition" { + if matches!(bucket, "retry-transition" | "completed-retention-retry-cancel") { return Ok(( HealResultItem::default(), Some(Error::Storage(EcstoreError::InsufficientReadQuorum( @@ -1145,7 +1476,13 @@ async fn test_active_duplicate_token_can_query_and_cancel_original_task() { .expect("duplicate token should cancel merged active task"); assert!(manager.active_heals.lock().await.get(&active_task_id).is_none()); - assert!(matches!(manager.get_task_status(&active_task_id).await, Err(Error::TaskNotFound { .. }))); + assert_eq!( + manager + .get_task_status(&active_task_id) + .await + .expect("cancelled task remains queryable"), + HealTaskStatus::Cancelled + ); } #[tokio::test] @@ -1638,6 +1975,8 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) -> manager.completed_heals.lock().await.insert( task_id, Arc::new(CompletedHealStatus { + progress: None, + retained_bytes: std::sync::OnceLock::new(), heal_type: request.heal_type, status: HealTaskStatus::Retrying { error: "Lock acquisition timeout".to_string(), @@ -2053,7 +2392,7 @@ async fn admin_force_start_cancels_overlapping_active_task_first() { "the overlapping admin task must be cancelled (removed from the active table) before the new one starts" ); assert!( - matches!(manager.get_task_status(&old_id).await, Err(Error::TaskNotFound { .. })), + matches!(manager.get_task_status(&old_id).await, Ok(HealTaskStatus::Cancelled)), "a cancelled task must no longer resolve as an active heal" ); } @@ -2360,6 +2699,8 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() { manager.completed_heals.lock().await.insert( task_id.clone(), Arc::new(CompletedHealStatus { + progress: None, + retained_bytes: std::sync::OnceLock::new(), heal_type: request.heal_type.clone(), status: HealTaskStatus::Retrying { error: "transient disk failure".to_string(), @@ -2395,6 +2736,8 @@ async fn test_get_task_status_reads_recent_completed_status() { manager.completed_heals.lock().await.insert( "completed-token".to_string(), Arc::new(CompletedHealStatus { + progress: None, + retained_bytes: std::sync::OnceLock::new(), heal_type: HealType::Bucket { bucket: "bucket".to_string(), }, @@ -2424,6 +2767,8 @@ async fn test_get_task_report_for_path_reads_completed_items() { manager.completed_heals.lock().await.insert( "completed-token".to_string(), Arc::new(CompletedHealStatus { + progress: None, + retained_bytes: std::sync::OnceLock::new(), heal_type: HealType::Object { bucket: "bucket".to_string(), object: "object".to_string(), diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index c95770834..c4c1d902b 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -999,7 +999,7 @@ impl HealTask { let items = match since { None => result_items.iter().map(|(_, item)| item.clone()).collect::>(), Some(cursor) => { - if cursor + 1 < min_seq { + if cursor.saturating_add(1) < min_seq { lagged = true; } result_items From e6bf2a464660ac5cd605fda090f6e73f8035efd0 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 5 Sep 2026 16:45:45 +0800 Subject: [PATCH 08/12] fix(admin): report partial background heal coverage (#7178) * chore(deps): refresh SDKs and pin clock skew regression coverage Refresh compatible dependencies for Scanner/Heal V2 batch 1 and verify the production S3 retry/signing path with a deterministic clock. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(admin): report partial background heal coverage Refs rustfs/backlog#2035 and rustfs/backlog#2240. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/madmin/src/client.rs | 72 +++++++++++ rustfs/src/admin/handlers/heal.rs | 197 +++++++++++++++++++++++++++--- 2 files changed, 249 insertions(+), 20 deletions(-) diff --git a/crates/madmin/src/client.rs b/crates/madmin/src/client.rs index 83f6c33d7..b49fde280 100644 --- a/crates/madmin/src/client.rs +++ b/crates/madmin/src/client.rs @@ -182,6 +182,9 @@ pub struct BackgroundHealStatus { pub heal_active_tasks: u64, #[serde(default)] pub cluster_status_complete: bool, + /// Missing on older servers; absent coverage or counts mean unknown. + #[serde(default)] + pub coverage: Option, #[serde(default)] pub progress: Option, /// Remaining wire fields (flattened `BackgroundHealInfo` plus the @@ -190,6 +193,22 @@ pub struct BackgroundHealStatus { pub extra: serde_json::Map, } +/// Node coverage of a background heal status snapshot. Counters describe only +/// nodes with usable snapshots; unknown peers may still be running heal work. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BackgroundHealCoverage { + #[serde(default)] + pub expected: Option, + #[serde(default)] + pub responded: Option, + #[serde(default)] + pub unknown: Option, + /// Stable reason codes; unknown future codes are preserved verbatim. + #[serde(default)] + pub reasons: Vec, +} + /// `GET /v3/scanner/status` response, typed at the fields operators branch /// on; everything else passes through verbatim. #[derive(Debug, Clone, Deserialize)] @@ -630,9 +649,39 @@ mod tests { assert_eq!(status.state, "active"); assert_eq!(status.heal_queue_length, 3); assert!(status.cluster_status_complete); + assert!(status.coverage.is_none(), "legacy payloads have unknown coverage"); assert!(status.extra.contains_key("healOperations"), "unknown nested payloads must pass through"); } + #[test] + fn background_heal_status_missing_coverage_fields_remain_unknown() { + for raw in [json!({"state": "degraded"}), json!({"state": "degraded", "coverage": {}})] { + let status: BackgroundHealStatus = serde_json::from_value(raw).expect("partial legacy payload decodes"); + assert!(!status.cluster_status_complete); + if let Some(coverage) = status.coverage { + assert_eq!(coverage.expected, None); + assert_eq!(coverage.responded, None); + assert_eq!(coverage.unknown, None); + } + } + } + + #[test] + fn background_heal_status_preserves_future_fields_and_reasons() { + let raw = json!({ + "state": "degraded", "clusterStatusComplete": false, + "coverage": {"expected": 3, "responded": 1, "unknown": 2, "reasons": ["future_reason"], "futureCoverage": true}, + "futureStatus": {"value": 7} + }); + let status: BackgroundHealStatus = serde_json::from_value(raw).expect("future additive fields decode"); + assert_eq!(status.extra["futureStatus"]["value"], 7); + let coverage = status.coverage.expect("coverage supplied"); + assert_eq!(coverage.expected, Some(3)); + assert_eq!(coverage.responded, Some(1)); + assert_eq!(coverage.unknown, Some(2)); + assert_eq!(coverage.reasons, ["future_reason"]); + } + #[test] fn scanner_status_defaults_freshness_to_unknown() { let raw = json!({"enabled": true, "freshness": {"state": "stale"}, "metrics": {}}); @@ -721,6 +770,7 @@ mod tests { let status = client.background_heal_status().await.expect("status decodes"); assert_eq!(status.state, "idle"); + assert!(status.coverage.is_none(), "older HTTP responses retain unknown coverage"); let request = server.recorded(); // The server registers this route POST-only; a GET here answers 405. assert_eq!(request.method, "POST"); @@ -728,6 +778,28 @@ mod tests { assert_eq!(request.query, ""); } + #[tokio::test] + async fn background_heal_status_decodes_partial_coverage_over_http() { + let body = r#"{"state":"degraded","healQueueLength":0,"healActiveTasks":0,"clusterStatusComplete":false,"coverage":{"expected":3,"responded":1,"unknown":2,"reasons":["notification_system_unavailable"]},"futureStatus":true}"#; + let server = TestServer::spawn(body, 200).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").expect("client builds"); + let status = client + .background_heal_status() + .await + .expect("partial status is a successful response"); + assert_eq!(status.state, "degraded"); + assert!(!status.cluster_status_complete); + assert_eq!(status.extra["futureStatus"], true); + let coverage = status.coverage.expect("partial coverage supplied"); + assert_eq!(coverage.expected, Some(3)); + assert_eq!(coverage.responded, Some(1)); + assert_eq!(coverage.unknown, Some(2)); + assert_eq!(coverage.reasons, ["notification_system_unavailable"]); + let request = server.recorded(); + assert_eq!(request.method, "POST"); + assert_eq!(request.query, "", "reading status must not send heal control parameters"); + } + #[tokio::test] async fn http_error_status_maps_to_a_typed_error_with_body() { let server = TestServer::spawn(r#"{"code":"AccessDenied","message":"denied"}"#, 403).await; diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index f10ad9123..74a2da980 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -39,7 +39,7 @@ use rustfs_utils::path::path_join; use s3s::header::{CONTENT_LENGTH, CONTENT_TYPE}; use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; use serde::{Deserialize, Serialize}; -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::future::Future; use std::path::PathBuf; use std::sync::Arc; @@ -261,6 +261,7 @@ struct BackgroundHealStatus<'a> { heal_active_tasks: u64, heal_operations: rustfs_heal::HealOperationsSnapshot, cluster_status_complete: bool, + coverage: &'a BackgroundHealCoverage, #[serde(skip_serializing_if = "Option::is_none")] progress: Option, } @@ -300,6 +301,23 @@ fn background_heal_runtime_state( type BackgroundHealProgress = rustfs_heal::HealProgress; +#[derive(Debug, Serialize)] +struct BackgroundHealCoverage { + expected: usize, + responded: usize, + unknown: usize, + reasons: BTreeSet, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(rename_all = "snake_case")] +enum BackgroundHealCoverageReason { + NotificationSystemUnavailable, + PeerTopologyIncomplete, + PeerStatusUnsupported, + PeerStatusUnavailable, +} + #[derive(Debug)] struct ClusterHealStatusSnapshot { info: BackgroundHealInfo, @@ -307,6 +325,7 @@ struct ClusterHealStatusSnapshot { operations: rustfs_heal::HealOperationsSnapshot, progress: Option, complete: bool, + coverage: BackgroundHealCoverage, } fn add_priority_counts(total: &mut rustfs_heal::HealPriorityCounts, next: rustfs_heal::HealPriorityCounts) { @@ -338,6 +357,7 @@ fn add_operations(total: &mut rustfs_heal::HealOperationsSnapshot, next: rustfs_ } fn aggregate_cluster_heal_status(snapshots: Vec) -> ClusterHealStatusSnapshot { + let responded = snapshots.len(); let mut info = BackgroundHealInfo::default(); let mut operations = rustfs_heal::HealOperationsSnapshot::default(); let mut progress = Vec::new(); @@ -379,6 +399,12 @@ fn aggregate_cluster_heal_status(snapshots: Vec) -> Clus operations, progress, complete: true, + coverage: BackgroundHealCoverage { + expected: responded, + responded, + unknown: 0, + reasons: BTreeSet::new(), + }, } } @@ -413,12 +439,14 @@ fn merge_peer_heal_statuses( mut snapshots: Vec, peer_statuses: Vec, String>>, expected_nodes: usize, - topology_complete: bool, + coverage_reason: Option, ) -> S3Result { + let mut reasons: BTreeSet<_> = coverage_reason.into_iter().collect(); for peer_status in peer_statuses { match peer_status { Ok(Some(snapshot)) => snapshots.push(snapshot), Ok(None) => { + reasons.insert(BackgroundHealCoverageReason::PeerStatusUnsupported); warn!( event = EVENT_ADMIN_REQUEST_FAILED, component = LOG_COMPONENT_ADMIN_API, @@ -430,6 +458,7 @@ fn merge_peer_heal_statuses( ); } Err(err) => { + reasons.insert(BackgroundHealCoverageReason::PeerStatusUnavailable); warn!( event = EVENT_ADMIN_REQUEST_FAILED, component = LOG_COMPONENT_ADMIN_API, @@ -452,9 +481,12 @@ fn merge_peer_heal_statuses( // so during a reconfiguration the count can equal `expected_nodes` while // the topology is known-incomplete. Counting alone would report a // definitive answer precisely when the membership itself is in doubt. - let complete = topology_complete && snapshots.len() == expected_nodes; + let complete = reasons.is_empty() && snapshots.len() == expected_nodes; let mut status = aggregate_cluster_heal_status(snapshots); status.complete = complete; + status.coverage.expected = expected_nodes; + status.coverage.unknown = expected_nodes.saturating_sub(status.coverage.responded); + status.coverage.reasons = reasons; // A partial answer must never be mistakable for a definitive verdict: an // unreachable peer might be mid-heal, so reporting the reachable nodes' // "idle" (or disabled/uninitialized) as the cluster state would falsely @@ -497,7 +529,12 @@ async fn read_cluster_heal_status( return Ok(aggregate_cluster_heal_status(snapshots)); } let Some(notification_system) = notification_system else { - return Err(cluster_heal_status_unavailable("notification_system_unavailable")); + return merge_peer_heal_statuses( + snapshots, + Vec::new(), + expected_nodes, + Some(BackgroundHealCoverageReason::NotificationSystemUnavailable), + ); }; // An incomplete peer topology (a down member's client slot, a rolling // upgrade) previously failed the whole endpoint here, before any peer was @@ -540,7 +577,12 @@ async fn read_cluster_heal_status( })) .await; - merge_peer_heal_statuses(snapshots, peer_statuses, expected_nodes, topology_complete) + merge_peer_heal_statuses( + snapshots, + peer_statuses, + expected_nodes, + (!topology_complete).then_some(BackgroundHealCoverageReason::PeerTopologyIncomplete), + ) } async fn query_peer_replacement_recovery_status( @@ -1164,6 +1206,7 @@ fn encode_background_heal_status( heal_operations: rustfs_heal::HealOperationsSnapshot, progress: Option, cluster_status_complete: bool, + coverage: &BackgroundHealCoverage, ) -> S3Result> { let status = BackgroundHealStatus { info, @@ -1172,6 +1215,7 @@ fn encode_background_heal_status( heal_active_tasks: heal_operations.active_tasks, heal_operations, cluster_status_complete, + coverage, progress, }; serde_json::to_vec(&status).map_err(|e| { @@ -1461,6 +1505,7 @@ impl Operation for BackgroundHealStatusHandler { cluster_status.operations, cluster_status.progress, cluster_status.complete, + &cluster_status.coverage, )?; info!( event = EVENT_ADMIN_RESPONSE_EMITTED, @@ -1515,13 +1560,14 @@ impl Operation for ReplacementRecoveryStatusHandler { mod tests { use super::extract_heal_init_params; use super::{ - BackgroundHealProgress, HealInitParams, HealResp, HealRuntimeState, aggregate_cluster_heal_status, - aggregate_replacement_recovery_cluster_status, background_heal_runtime_state, build_heal_channel_request, - build_replacement_recovery_status_response, encode_background_heal_status, encode_heal_control_path, - encode_heal_start_success, encode_heal_task_status, execute_after_heal_control_capability, heal_channel_response_items, - heal_channel_response_progress, heal_channel_response_summary, heal_control_response_id, json_response, - map_heal_response, merge_peer_heal_statuses, peer_topology_complete, query_peer_heal_status, - query_peer_replacement_recovery_status, reject_heal_admission, validate_heal_request_mode, validate_heal_target, + BackgroundHealCoverage, BackgroundHealCoverageReason, BackgroundHealProgress, HealInitParams, HealResp, HealRuntimeState, + aggregate_cluster_heal_status, aggregate_replacement_recovery_cluster_status, background_heal_runtime_state, + build_heal_channel_request, build_replacement_recovery_status_response, encode_background_heal_status, + encode_heal_control_path, encode_heal_start_success, encode_heal_task_status, execute_after_heal_control_capability, + heal_channel_response_items, heal_channel_response_progress, heal_channel_response_summary, heal_control_response_id, + json_response, map_heal_response, merge_peer_heal_statuses, peer_topology_complete, query_peer_heal_status, + query_peer_replacement_recovery_status, read_cluster_heal_status, reject_heal_admission, validate_heal_request_mode, + validate_heal_target, }; use crate::storage::rpc::node_service::heal::{ NodeHealProgress, NodeHealStatusSnapshot, NodeReplacementRecoveryStatusSnapshot, encode_node_replacement_recovery_status, @@ -2175,7 +2221,13 @@ mod tests { ..Default::default() }; - let encoded = encode_background_heal_status(&info, HealRuntimeState::Active, operations, None, true) + let coverage = BackgroundHealCoverage { + expected: 1, + responded: 1, + unknown: 0, + reasons: Default::default(), + }; + let encoded = encode_background_heal_status(&info, HealRuntimeState::Active, operations, None, true, &coverage) .expect("background heal info should serialize"); let json: serde_json::Value = serde_json::from_slice(&encoded).expect("json should deserialize"); @@ -2225,6 +2277,12 @@ mod tests { rustfs_heal::HealOperationsSnapshot::default(), Some(progress), true, + &BackgroundHealCoverage { + expected: 1, + responded: 1, + unknown: 0, + reasons: Default::default(), + }, ) .expect("background heal info should serialize"); let json: serde_json::Value = serde_json::from_slice(&encoded).expect("json should deserialize"); @@ -2398,6 +2456,84 @@ mod tests { assert!(peer_topology_complete(1, 0, 0, 1, 0)); } + #[tokio::test] + async fn test_background_heal_status_without_notification_preserves_local_snapshot() { + let initialized = rustfs_heal::heal_runtime_initialized(); + let info = BackgroundHealInfo { + bitrot_start_cycle: 37, + current_scan_mode: HealScanMode::Deep, + ..Default::default() + }; + for expected in [1, 3] { + let status = tokio::time::timeout(Duration::from_secs(1), read_cluster_heal_status(info.clone(), None, expected)) + .await + .expect("local status must not wait for remote peers") + .expect("missing notification must retain the local snapshot"); + assert_eq!(status.info.bitrot_start_cycle, 37); + assert_eq!(status.info.current_scan_mode, HealScanMode::Deep); + assert_eq!(status.complete, expected == 1); + assert_eq!(status.coverage.expected, expected); + assert_eq!(status.coverage.responded, 1); + assert_eq!(status.coverage.unknown, expected - 1); + if expected == 1 { + assert!(status.coverage.reasons.is_empty()); + } else { + assert!(matches!(status.state, HealRuntimeState::Degraded | HealRuntimeState::Active)); + assert_eq!( + status.coverage.reasons, + [BackgroundHealCoverageReason::NotificationSystemUnavailable].into() + ); + } + let encoded = encode_background_heal_status( + &status.info, + status.state, + status.operations, + status.progress, + status.complete, + &status.coverage, + ) + .expect("fallback status must encode"); + let decoded: rustfs_madmin::client::BackgroundHealStatus = + serde_json::from_slice(&encoded).expect("the actual madmin client must decode the server response"); + assert_eq!(decoded.cluster_status_complete, expected == 1); + let coverage = decoded.coverage.expect("new server supplies coverage"); + assert_eq!(coverage.expected, Some(expected)); + assert_eq!(coverage.responded, Some(1)); + assert_eq!(coverage.unknown, Some(expected - 1)); + if expected > 1 { + assert_eq!(coverage.reasons, ["notification_system_unavailable"]); + } + } + assert_eq!( + rustfs_heal::heal_runtime_initialized(), + initialized, + "reading status must not initialize heal" + ); + } + + #[test] + fn test_background_heal_status_coverage_reasons_are_bounded() { + let local = NodeHealStatusSnapshot::for_test(true, true, BackgroundHealInfo::default(), Default::default(), None); + let peers = (0..100) + .map(|index| { + if index % 2 == 0 { + Ok(None) + } else { + Err("peer unavailable".to_owned()) + } + }) + .collect(); + let status = merge_peer_heal_statuses(vec![local], peers, 101, None).expect("local status remains available"); + assert_eq!(status.coverage.responded, 1); + assert_eq!(status.coverage.unknown, 100); + assert_eq!(status.coverage.reasons.len(), 2); + let encoded = serde_json::to_vec(&status.coverage).expect("coverage encodes"); + assert!(encoded.len() < 256, "coverage must not grow with peer failures"); + let decoded: rustfs_madmin::client::BackgroundHealCoverage = + serde_json::from_slice(&encoded).expect("client coverage decodes"); + assert_eq!(decoded.reasons, ["peer_status_unsupported", "peer_status_unavailable"]); + } + #[test] fn test_peer_status_merge_degrades_explicitly_and_never_claims_idle() { let local = || { @@ -2413,15 +2549,20 @@ mod tests { // but the safety property of the previous fail-closed behaviour is // preserved: the partial answer is labelled Degraded, never Idle, so // unknown peer work cannot be mistaken for "nothing is running". - let partial = merge_peer_heal_statuses(vec![local()], vec![Err("peer timeout".to_string())], 2, true) + let partial = merge_peer_heal_statuses(vec![local()], vec![Err("peer timeout".to_string())], 2, None) .expect("an unreachable peer degrades the answer instead of destroying it"); assert!(!partial.complete); assert_eq!(partial.state, HealRuntimeState::Degraded); + assert_eq!(partial.coverage.expected, 2); + assert_eq!(partial.coverage.responded, 1); + assert_eq!(partial.coverage.unknown, 1); + assert_eq!(partial.coverage.reasons, [BackgroundHealCoverageReason::PeerStatusUnavailable].into()); - let older_peer = merge_peer_heal_statuses(vec![local()], vec![Ok(None)], 2, true) + let older_peer = merge_peer_heal_statuses(vec![local()], vec![Ok(None)], 2, None) .expect("an older peer degrades the answer instead of destroying it"); assert!(!older_peer.complete); assert_eq!(older_peer.state, HealRuntimeState::Degraded); + assert_eq!(older_peer.coverage.reasons, [BackgroundHealCoverageReason::PeerStatusUnsupported].into()); let known_active = NodeHealStatusSnapshot::for_test( true, @@ -2433,12 +2574,12 @@ mod tests { }, None, ); - let partial_active = merge_peer_heal_statuses(vec![known_active], vec![Ok(None)], 2, true) + let partial_active = merge_peer_heal_statuses(vec![known_active], vec![Ok(None)], 2, None) .expect("known active work may be reported as an explicit partial status"); assert!(!partial_active.complete); assert_eq!(partial_active.state, HealRuntimeState::Active); - merge_peer_heal_statuses(Vec::new(), vec![Err("peer timeout".to_string())], 2, true) + merge_peer_heal_statuses(Vec::new(), vec![Err("peer timeout".to_string())], 2, None) .expect_err("no snapshot at all still fails closed"); } @@ -2459,12 +2600,22 @@ mod tests { None, ) }; - let full_count_incomplete_topology = merge_peer_heal_statuses(vec![snapshot()], vec![Ok(Some(snapshot()))], 2, false) - .expect("incomplete topology degrades the answer instead of destroying it"); + let full_count_incomplete_topology = merge_peer_heal_statuses( + vec![snapshot()], + vec![Ok(Some(snapshot()))], + 2, + Some(BackgroundHealCoverageReason::PeerTopologyIncomplete), + ) + .expect("incomplete topology degrades the answer instead of destroying it"); assert!(!full_count_incomplete_topology.complete); assert_eq!(full_count_incomplete_topology.state, HealRuntimeState::Degraded); + assert_eq!(full_count_incomplete_topology.coverage.unknown, 0); + assert_eq!( + full_count_incomplete_topology.coverage.reasons, + [BackgroundHealCoverageReason::PeerTopologyIncomplete].into() + ); - let full_count_complete_topology = merge_peer_heal_statuses(vec![snapshot()], vec![Ok(Some(snapshot()))], 2, true) + let full_count_complete_topology = merge_peer_heal_statuses(vec![snapshot()], vec![Ok(Some(snapshot()))], 2, None) .expect("complete topology and full count is a definitive answer"); assert!(full_count_complete_topology.complete); assert_eq!(full_count_complete_topology.state, HealRuntimeState::Idle); @@ -2478,6 +2629,12 @@ mod tests { rustfs_heal::HealOperationsSnapshot::default(), None, false, + &BackgroundHealCoverage { + expected: 2, + responded: 1, + unknown: 1, + reasons: [BackgroundHealCoverageReason::PeerStatusUnavailable].into(), + }, ) .expect("degraded status must serialize"); let json: serde_json::Value = serde_json::from_slice(&encoded).expect("valid json"); From 42c32381b60d09a46e38554a37664352e6c66ec8 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 5 Sep 2026 16:45:53 +0800 Subject: [PATCH 09/12] fix(scanner): make reset cleanup safely reentrant (#7180) * chore(deps): refresh SDKs and pin clock skew regression coverage Refresh compatible dependencies for Scanner/Heal V2 batch 1 and verify the production S3 retry/signing path with a deterministic clock. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(scanner): make reset cleanup safely reentrant Refs rustfs/backlog#2264 and rustfs/backlog#2240. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/scanner/src/scanner/cycle_state.rs | 331 ++++++++++++++++--- crates/scanner/src/scanner/leadership.rs | 8 +- crates/scanner/src/scanner/tests.rs | 380 +++++++++++++++++++++- 3 files changed, 664 insertions(+), 55 deletions(-) diff --git a/crates/scanner/src/scanner/cycle_state.rs b/crates/scanner/src/scanner/cycle_state.rs index dc29e3331..6913ed836 100644 --- a/crates/scanner/src/scanner/cycle_state.rs +++ b/crates/scanner/src/scanner/cycle_state.rs @@ -379,6 +379,12 @@ pub(super) fn decode_recovery_marker_for_reset( if !matches!(marker_revision, DataUsageCacheRevision::Etag(_)) { return Err(ScannerError::Other("cycle recovery marker has no object revision".to_string())); } + if let Ok(value) = serde_json::from_slice::(data) + && let Some(state) = value.get("state") + && !matches!(state.as_str(), Some("blocked" | "cleanup-pending")) + { + return Err(ScannerError::Other("cycle recovery marker state is unsupported".to_string())); + } let compat = serde_json::from_slice::(data).ok(); let _schema_version = compat.as_ref().and_then(|marker| marker.schema_version); let primary_revision = compat @@ -406,7 +412,10 @@ pub(super) fn decode_recovery_marker_for_reset( }; let state = match compat.as_ref().and_then(|marker| marker.state.as_deref()) { Some("cleanup-pending") => "cleanup-pending", - _ => "blocked", + Some("blocked") | None => "blocked", + Some(_) => { + return Err(ScannerError::Other("cycle recovery marker state is unsupported".to_string())); + } }; let now = unix_now_secs(); Ok(ScannerCycleRecoveryMarker { @@ -721,17 +730,19 @@ async fn mark_cycle_recovery_cleanup_pending( mut marker: ScannerCycleRecoveryMarker, marker_revision: &DataUsageCacheRevision, expected_epoch: u64, + owns_reset: &(impl Fn() -> bool + Sync), ) -> Result<(ScannerCycleRecoveryMarker, DataUsageCacheRevision), ScannerError> { marker.state = "cleanup-pending".to_string(); marker.last_attempt_at_unix_secs = unix_now_secs(); let bytes = serde_json::to_vec(&marker) .map_err(|err| ScannerError::Other(format!("failed to encode cycle recovery marker: {err}")))?; - let info = save_config_with_publication_admission_for_epoch( + let info = save_reset_config( storeapi.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), bytes, marker_revision.preconditions(), expected_epoch, + owns_reset, ) .await .map_err(|err| ScannerError::Other(format!("failed to mark cycle recovery cleanup pending: {err}")))?; @@ -933,6 +944,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< .get_write_lock_quiet(Duration::from_secs(5)) .await .map_err(|err| ScannerError::Other(format!("scanner leader lock is busy: {err}")))?; + let owns_reset = || !guard.is_lock_lost() && !ctx.is_cancelled(); if guard.is_lock_lost() { return Err(ScannerError::Other("scanner leader lock was lost before recovery reset".to_string())); @@ -952,7 +964,27 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< } Err(err) => return Err(ScannerError::Other(format!("failed to read cycle recovery marker: {err}"))), }; - let marker_data = marker_data.ok_or_else(|| ScannerError::Other("scanner cycle recovery marker is absent".to_string()))?; + let Some(marker_data) = marker_data else { + // A delete may commit before its reply is lost. Confirm both durable + // fences before treating a retry without its marker as completed. + let (cycle, epoch, revision) = read_cycle_state_for_usage_reset(storeapi.clone()).await?; + let floor = persisted_usage_floor(storeapi.clone()).await?; + if !matches!(revision, DataUsageCacheRevision::Etag(_)) + || epoch < floor.leader_epoch + || cycle.next < floor.next_cycle + || !owns_reset() + || scanner_publication_admission_for_epoch(storeapi.clone(), reset_epoch) + .await + .is_none() + { + return Err(ScannerError::Other( + "scanner cycle recovery marker is absent without a completed reset fence".to_string(), + )); + } + set_scanner_cycle_recovery_status(recovery_status("healthy", None, false)); + super::notify_scanner_cycle_recovery_wake(); + return Ok(()); + }; let (marker, force_full_rescan) = match serde_json::from_slice::(&marker_data) { Ok(marker) if validate_recovery_marker(&marker).is_ok() => (marker, false), _ => (decode_recovery_marker_for_reset(&marker_data, &marker_revision)?, true), @@ -1026,8 +1058,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< } }; if let Some((primary_cycle, primary_epoch)) = primary_state { + verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?; let (cleanup_marker, cleanup_marker_revision) = - mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch).await?; + mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch, &owns_reset) + .await?; set_scanner_cycle_recovery_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending")); let usage_floor = persisted_usage_floor(storeapi.clone()).await?; let fence_epoch = primary_epoch @@ -1047,12 +1081,14 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< "preserved scanner cycle state exceeds the bounded object size".to_string(), )); } - let preserved_info = save_config_with_publication_admission_for_epoch( + verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?; + let preserved_info = save_reset_config( storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), preserved_data, primary_revision.preconditions(), reset_epoch, + &owns_reset, ) .await .map_err(|err| { @@ -1072,9 +1108,17 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< "scanner leader lock was lost after fencing newer cycle state".to_string(), )); } - fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), fence_epoch, Some(reset_epoch), false) - .await - .map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?; + verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?; + fence_scanner_usage_epoch_with_expected_epoch( + &ctx, + storeapi.clone(), + fence_epoch, + Some(reset_epoch), + false, + &owns_reset, + ) + .await + .map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?; if guard.is_lock_lost() { return Err(ScannerError::Other( "scanner leader lock was lost after fencing newer cycle state".to_string(), @@ -1088,7 +1132,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< "scanner cycle state changed before recovery marker cleanup".to_string(), )); } - delete_config_with_publication_admission_for_epoch( + verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?; + delete_reset_config( storeapi.clone(), RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), @@ -1100,6 +1145,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< ..Default::default() }, reset_epoch, + &owns_reset, ) .await .map_err(|err| { @@ -1149,17 +1195,20 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< // Persist the cleanup-pending phase before rewriting the primary. If the // process dies after the rewrite, startup still sees a durable fence and // cannot mistake the partially completed reset for a healthy state. + verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?; let (marker, marker_revision) = if marker.state == "cleanup-pending" { (marker, marker_revision) } else { - mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch).await? + mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch, &owns_reset).await? }; - let rebuilt_info = save_config_with_publication_admission_for_epoch( + verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?; + let rebuilt_info = save_reset_config( storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), data, primary_revision.preconditions(), reset_epoch, + &owns_reset, ) .await .map_err(|err| { @@ -1178,8 +1227,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< "scanner leader lock was lost after rebuilding cycle state".to_string(), )); } + verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?; if let Err(err) = - fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch), false).await + fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch), false, &owns_reset) + .await { set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { path: DATA_USAGE_BLOOM_NAME_PATH.clone(), @@ -1249,7 +1300,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< )); } - if let Err(err) = delete_config_with_publication_admission_for_epoch( + verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?; + if let Err(err) = delete_reset_config( storeapi.clone(), RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), @@ -1261,6 +1313,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< ..Default::default() }, reset_epoch, + &owns_reset, ) .await { @@ -1310,6 +1363,57 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< Ok(()) } +async fn verify_cycle_reset_intent( + storeapi: Arc, + expected_revision: &DataUsageCacheRevision, + owns_reset: &(impl Fn() -> bool + Sync), +) -> Result<(), ScannerError> { + let revision = read_config_revision(storeapi, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to verify scanner cycle reset intent: {err}")))?; + if &revision != expected_revision { + return Err(ScannerError::Other("scanner cycle reset intent changed".to_string())); + } + if !owns_reset() { + return Err(ScannerError::Other("scanner cycle reset ownership was lost".to_string())); + } + Ok(()) +} + +async fn save_reset_config( + storeapi: Arc, + path: &str, + data: Vec, + preconditions: crate::HTTPPreconditions, + expected_epoch: u64, + owns_reset: &(impl Fn() -> bool + Sync), +) -> Result { + let Some(_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else { + return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED)); + }; + if !owns_reset() { + return Err(EcstoreError::other("scanner reset ownership was lost before write")); + } + save_config_with_preconditions(storeapi, path, data, preconditions).await +} + +async fn delete_reset_config( + storeapi: Arc, + bucket: &str, + path: &str, + options: ScannerObjectOptions, + expected_epoch: u64, + owns_reset: &(impl Fn() -> bool + Sync), +) -> Result { + let Some(_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else { + return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED)); + }; + if !owns_reset() { + return Err(EcstoreError::other("scanner reset ownership was lost before delete")); + } + storeapi.delete_config_object(bucket, path, options).await +} + fn scanner_usage_state_reset_paths() -> Vec { vec![ DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(), @@ -1333,8 +1437,14 @@ pub(super) async fn read_usage_state_reset_slots( Ok(slots) } -fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result { - let mut floor = PersistedUsageFloor::default(); +enum ScannerUsageResetFloor { + Missing, + Trusted(PersistedUsageFloor), + Corrupt, +} + +fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result { + let mut floor = None; for slot in slots { let Some(data) = slot.data.as_deref() else { continue; @@ -1342,9 +1452,21 @@ fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result(data) else { continue; }; - update_persisted_usage_floor(&mut floor, &usage, &slot.path)?; + if !data_usage_info_has_persisted_baseline_identity(&usage) + && !(slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str() && data_usage_info_is_bootstrap_pending(&usage)) + && legacy_incomplete_usage_fence(data, &usage) + .and_then(|fence| fence.claimable_epoch()) + .is_none() + { + continue; + } + update_persisted_usage_floor(floor.get_or_insert_with(PersistedUsageFloor::default), &usage, &slot.path)?; } - Ok(floor) + Ok(match floor { + Some(floor) => ScannerUsageResetFloor::Trusted(floor), + None if slots.iter().any(|slot| slot.data.is_some()) => ScannerUsageResetFloor::Corrupt, + None => ScannerUsageResetFloor::Missing, + }) } async fn read_cycle_state_for_usage_reset( @@ -1401,11 +1523,12 @@ async fn delete_usage_state_reset_slot( storeapi: Arc, slot: &ScannerUsageStateResetSlot, expected_epoch: u64, + owns_reset: &(impl Fn() -> bool + Sync), ) -> Result { if matches!(slot.revision, DataUsageCacheRevision::Missing) { return Ok(false); } - let delete_result = delete_config_with_publication_admission_for_epoch( + let delete_result = delete_reset_config( storeapi.clone(), RUSTFS_META_BUCKET, &slot.path, @@ -1415,6 +1538,7 @@ async fn delete_usage_state_reset_slot( ..Default::default() }, expected_epoch, + owns_reset, ) .await; match delete_result { @@ -1486,21 +1610,24 @@ pub(super) async fn publish_scanner_usage_bootstrap_primary( expected_publication_epoch: u64, leader_epoch: Option, context: ScannerUsageBootstrapPublishContext, + owns_publication: impl Fn() -> bool + Sync, ) -> Result<(), ScannerError> { async fn inner( storeapi: Arc, expected_revision: &DataUsageCacheRevision, expected_publication_epoch: u64, leader_epoch: Option, + owns_publication: &(impl Fn() -> bool + Sync), ) -> Result<(), ScannerUsageBootstrapPublishError> { let marker = scanner_usage_bootstrap_marker(std::time::SystemTime::now(), leader_epoch); let data = serde_json::to_vec(&marker).map_err(ScannerUsageBootstrapPublishError::Encode)?; - let save_result = save_config_with_publication_admission_for_epoch( + let save_result = save_reset_config( storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data.clone(), expected_revision.preconditions(), expected_publication_epoch, + owns_publication, ) .await; if save_result @@ -1524,7 +1651,7 @@ pub(super) async fn publish_scanner_usage_bootstrap_primary( }) } - inner(storeapi, expected_revision, expected_publication_epoch, leader_epoch) + inner(storeapi, expected_revision, expected_publication_epoch, leader_epoch, &owns_publication) .await .map_err(|err| err.into_scanner_error(context)) } @@ -1534,32 +1661,108 @@ pub(super) async fn reset_scanner_usage_state_slots_for_full_rebuild( slots: &[ScannerUsageStateResetSlot], expected_epoch: u64, leader_epoch: u64, + owns_reset: impl Fn() -> bool + Sync, ) -> Result, ScannerError> { let mut reset_paths = Vec::new(); let primary = slots .iter() .find(|slot| slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str()) .ok_or_else(|| ScannerError::Other("scanner usage reset primary slot was not inspected".to_string()))?; - publish_scanner_usage_bootstrap_primary( - storeapi.clone(), - &primary.revision, - expected_epoch, - Some(leader_epoch), - ScannerUsageBootstrapPublishContext::Reset, - ) - .await?; + if !owns_reset() { + return Err(ScannerError::Other("scanner usage reset ownership was lost".to_string())); + } + let resume_epoch = usage_state_reset_resume_epoch(slots)?; + match resume_epoch { + Some(epoch) if epoch == leader_epoch => {} + Some(_) => return Err(ScannerError::Other("scanner usage reset bootstrap epoch changed".to_string())), + None => { + publish_scanner_usage_bootstrap_primary( + storeapi.clone(), + &primary.revision, + expected_epoch, + Some(leader_epoch), + ScannerUsageBootstrapPublishContext::Reset, + &owns_reset, + ) + .await?; + } + } + let (data, intent_revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to inspect scanner usage reset intent: {err}")))?; + data.as_deref() + .and_then(|data| serde_json::from_slice::(data).ok()) + .filter(|usage| data_usage_info_is_bootstrap_pending(usage) && usage.scanner_epoch == Some(leader_epoch)) + .ok_or_else(|| ScannerError::Other("scanner usage reset intent changed before cleanup".to_string()))?; + if !matches!(intent_revision, DataUsageCacheRevision::Etag(_)) + || (resume_epoch.is_some() && intent_revision != primary.revision) + { + return Err(ScannerError::Other("scanner usage reset intent revision changed".to_string())); + } reset_paths.push(DATA_USAGE_OBJ_NAME_PATH.as_str().to_string()); for slot in slots.iter().filter(|slot| slot.path != DATA_USAGE_OBJ_NAME_PATH.as_str()) { - if delete_usage_state_reset_slot(storeapi.clone(), slot, expected_epoch).await? { + if let Some(usage) = slot + .data + .as_deref() + .and_then(|data| serde_json::from_slice::(data).ok()) + && usage_epoch(&usage) >= leader_epoch + { + return Err(ScannerError::Other(format!( + "scanner usage reset slot is not older than its intent: {}", + slot.path + ))); + } + let revision = read_config_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to verify scanner usage reset intent: {err}")))?; + if revision != intent_revision { + return Err(ScannerError::Other("scanner usage reset intent changed during cleanup".to_string())); + } + if !owns_reset() { + return Err(ScannerError::Other("scanner usage reset ownership was lost".to_string())); + } + if delete_usage_state_reset_slot(storeapi.clone(), slot, expected_epoch, &owns_reset).await? { reset_paths.push(slot.path.clone()); } } + let revision = read_config_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to confirm scanner usage reset intent: {err}")))?; + if revision != intent_revision || !owns_reset() { + return Err(ScannerError::Other( + "scanner usage reset intent or ownership changed before completion".to_string(), + )); + } invalidate_admin_data_usage_snapshot_cache().await; invalidate_data_usage_snapshot_cache().await; Ok(reset_paths) } +fn usage_state_reset_resume_epoch(slots: &[ScannerUsageStateResetSlot]) -> Result, ScannerError> { + let primary = slots.iter().find(|slot| slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str()); + let usage = primary + .and_then(|slot| slot.data.as_deref()) + .and_then(|data| serde_json::from_slice::(data).ok()); + match usage { + Some(usage) if usage.usage_snapshot_bootstrap_pending => { + if !data_usage_info_is_bootstrap_pending(&usage) { + return Err(ScannerError::Other("scanner usage reset bootstrap is invalid".to_string())); + } + if usage.scanner_epoch.is_none() { + // Initial bootstrap has no reset owner yet. + return Ok(None); + } + usage + .scanner_epoch + .filter(|epoch| *epoch > 0 && *epoch < u64::MAX) + .map(Some) + .ok_or_else(|| ScannerError::Other("scanner usage reset bootstrap has no valid epoch".to_string())) + } + _ => Ok(None), + } +} + pub async fn reset_scanner_usage_state_for_full_rebuild( ctx: CancellationToken, storeapi: Arc, @@ -1584,12 +1787,31 @@ pub async fn reset_scanner_usage_state_for_full_rebuild( }; let (cycle, cycle_epoch, cycle_revision) = read_cycle_state_for_usage_reset(storeapi.clone()).await?; let slots = read_usage_state_reset_slots(storeapi.clone()).await?; - let usage_floor = usage_state_reset_floor(&slots)?; - let leader_epoch = cycle_epoch - .max(usage_floor.leader_epoch) - .checked_add(1) - .filter(|epoch| *epoch < u64::MAX) - .ok_or_else(|| ScannerError::Other("scanner leader epoch is exhausted".to_string()))?; + let usage_floor = match usage_state_reset_floor(&slots)? { + ScannerUsageResetFloor::Trusted(floor) => floor, + ScannerUsageResetFloor::Corrupt if matches!(cycle_revision, DataUsageCacheRevision::Missing) => { + return Err(ScannerError::Other("scanner usage reset has no trusted cycle or usage floor".to_string())); + } + ScannerUsageResetFloor::Missing | ScannerUsageResetFloor::Corrupt => PersistedUsageFloor { + next_cycle: cycle.next, + leader_epoch: cycle_epoch, + }, + }; + let resume_epoch = usage_state_reset_resume_epoch(&slots)?; + let leader_epoch = if let Some(epoch) = resume_epoch { + if epoch != cycle_epoch || usage_floor.leader_epoch > epoch || usage_floor.next_cycle > cycle.next { + return Err(ScannerError::Other( + "scanner usage reset bootstrap conflicts with the persisted cycle fence".to_string(), + )); + } + epoch + } else { + cycle_epoch + .max(usage_floor.leader_epoch) + .checked_add(1) + .filter(|epoch| *epoch < u64::MAX) + .ok_or_else(|| ScannerError::Other("scanner leader epoch is exhausted".to_string()))? + }; let rebuilt_cycle = CurrentCycle { next: cycle.next.max(usage_floor.next_cycle), ..Default::default() @@ -1602,21 +1824,24 @@ pub async fn reset_scanner_usage_state_for_full_rebuild( "scanner leader lock was lost before fencing usage reset cycle state".to_string(), )); } - save_config_with_publication_admission_for_epoch( - storeapi.clone(), - DATA_USAGE_BLOOM_NAME_PATH.as_str(), - cycle_data, - cycle_revision.preconditions(), - reset_epoch, - ) - .await - .map_err(|err| { - if scanner_publication_epoch_changed(&err) { - ScannerError::Other("scanner usage reset deferred by a movement epoch change".to_string()) - } else { - ScannerError::Other(format!("failed to fence scanner cycle state for usage reset: {err}")) - } - })?; + if resume_epoch.is_none() { + save_reset_config( + storeapi.clone(), + DATA_USAGE_BLOOM_NAME_PATH.as_str(), + cycle_data, + cycle_revision.preconditions(), + reset_epoch, + &|| !guard.is_lock_lost() && !ctx.is_cancelled(), + ) + .await + .map_err(|err| { + if scanner_publication_epoch_changed(&err) { + ScannerError::Other("scanner usage reset deferred by a movement epoch change".to_string()) + } else { + ScannerError::Other(format!("failed to fence scanner cycle state for usage reset: {err}")) + } + })?; + } if guard.is_lock_lost() { return Err(ScannerError::Other( @@ -1624,7 +1849,10 @@ pub async fn reset_scanner_usage_state_for_full_rebuild( )); } let reset_paths = - reset_scanner_usage_state_slots_for_full_rebuild(storeapi.clone(), &slots, reset_epoch, leader_epoch).await?; + reset_scanner_usage_state_slots_for_full_rebuild(storeapi.clone(), &slots, reset_epoch, leader_epoch, || { + !guard.is_lock_lost() && !ctx.is_cancelled() + }) + .await?; if guard.is_lock_lost() { return Err(ScannerError::Other( "scanner leader lock was lost after publishing usage reset marker".to_string(), @@ -2135,6 +2363,7 @@ async fn recover_legacy_incomplete_usage_floor( expected_publication_epoch, Some(primary.epoch), ScannerUsageBootstrapPublishContext::Recovery, + || true, ) .await?; warn!( diff --git a/crates/scanner/src/scanner/leadership.rs b/crates/scanner/src/scanner/leadership.rs index 8283a987a..260476e7a 100644 --- a/crates/scanner/src/scanner/leadership.rs +++ b/crates/scanner/src/scanner/leadership.rs @@ -191,6 +191,7 @@ pub(super) async fn initialize_usage_baseline_bootstrap( expected_epoch, None, ScannerUsageBootstrapPublishContext::Initial, + || true, ) .await } @@ -201,9 +202,10 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch( claimed_epoch: u64, expected_publication_epoch: Option, allow_bootstrap_pending: bool, + owns_fence: impl Fn() -> bool, ) -> Result<(), ScannerError> { for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { - if ctx.is_cancelled() { + if ctx.is_cancelled() || !owns_fence() { return Err(ScannerError::Other("scanner leadership was cancelled before usage fencing".to_string())); } @@ -264,6 +266,9 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch( "scanner usage epoch fence changed while preparing its conditional write".to_string(), )); }; + if ctx.is_cancelled() || !owns_fence() { + return Err(ScannerError::Other("scanner leadership was lost before usage fencing".to_string())); + } save_config_with_preconditions(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data, revision.preconditions()) .await }; @@ -319,6 +324,7 @@ pub(super) async fn complete_scanner_leadership_claim( claimed_epoch, expected_publication_epoch, allow_bootstrap_pending, + || true, ) .await { diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index 244e9088f..b9b4cd299 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -634,6 +634,8 @@ struct MemoryConfigStore { cancel_after_successful_puts: Mutex>, replace_after_successful_puts: Mutex)>>, error_after_commit_deletes: Mutex>, + cancel_after_deletes: Mutex>, + pause_next_publication_admission: Mutex, Arc)>>, put_counts: Mutex>, publication_admission_blocked: AtomicBool, block_publication_after_admissions: AtomicUsize, @@ -4081,6 +4083,9 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore { revisions.remove(&key); drop(revisions); drop(objects); + if let Some(token) = self.cancel_after_deletes.lock().await.remove(&key) { + token.cancel(); + } if self.error_after_commit_deletes.lock().await.remove(&key) { return Err(EcstoreError::other("injected delete error after commit")); } @@ -4088,6 +4093,11 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore { } async fn scanner_data_usage_publication_admission(&self) -> Option { + let pause = self.pause_next_publication_admission.lock().await.take(); + if let Some((entered, resume)) = pause { + entered.notify_one(); + resume.notified().await; + } if self.publication_admission_blocked.load(Ordering::Acquire) { return None; } @@ -4589,7 +4599,7 @@ async fn scanner_legacy_usage_backup_survives_fencing_and_restart_after_real_met .expect("publication must also read the intact backup"); assert_eq!(baseline.data.as_deref(), Some(data.as_slice())); assert_eq!(baseline.revision, DataUsageCacheRevision::Missing); - fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 7, None, false) + fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 7, None, false, || true) .await .expect("legacy backup must be fenced into v2"); let fenced = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) @@ -4818,6 +4828,28 @@ async fn scanner_usage_state_reset_publishes_fenced_bootstrap_marker() { ); } + let cycle_before_retry = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("cycle should remain before retry"); + let marker_before_retry = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("bootstrap should remain before retry"); + let retry = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone()) + .await + .expect("completed cleanup should be reentrant"); + assert_eq!(retry.leader_epoch, result.leader_epoch); + assert_eq!( + read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("cycle should remain"), + cycle_before_retry + ); + assert_eq!( + read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("bootstrap should remain"), + marker_before_retry + ); let (floor, state) = persisted_usage_floor_for_startup(store, false) .await .expect("reset marker should be resumable"); @@ -4973,7 +5005,7 @@ async fn scanner_usage_state_reset_slots_reject_primary_aba() { store.objects.lock().await.insert(key.clone(), b"newer-json".to_vec()); store.revisions.lock().await.insert(key, 2); - let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3) + let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3, || true) .await .expect_err("stale primary revision must not be overwritten"); assert!( @@ -4983,6 +5015,348 @@ async fn scanner_usage_state_reset_slots_reject_primary_aba() { ); } +#[tokio::test] +async fn scanner_usage_state_reset_resumes_every_cleanup_boundary_without_rewriting_intent() { + for completed in 0..=4 { + let store = Arc::new(MemoryConfigStore::default()); + let primary_path = DATA_USAGE_OBJ_NAME_PATH.as_str(); + let cleanup_paths = [ + format!("{primary_path}.bkp"), + LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(), + format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), + DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str().to_string(), + ]; + for path in std::iter::once(primary_path).chain(cleanup_paths.iter().map(String::as_str)) { + let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + usage.scanner_epoch = Some(1); + save_config(store.clone(), path, serde_json::to_vec(&usage).expect("fixture should encode")) + .await + .expect("fixture should persist"); + } + // These objects belong to other owners, even when reset cleanup resumes. + for path in ["buckets/quota-reservations/ledger", "buckets/example/incarnation"] { + save_config(store.clone(), path, b"retain".to_vec()) + .await + .expect("unrelated state should persist"); + } + let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load"); + let cancelled = CancellationToken::new(); + if completed == 0 { + store + .cancel_after_successful_puts + .lock() + .await + .insert(memory_config_key(RUSTFS_META_BUCKET, primary_path), (2, cancelled.clone())); + } else { + store + .cancel_after_deletes + .lock() + .await + .insert(memory_config_key(RUSTFS_META_BUCKET, &cleanup_paths[completed - 1]), cancelled.clone()); + } + let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || !cancelled.is_cancelled()) + .await + .expect_err("interruption should stop cleanup"); + assert!(err.to_string().contains("ownership"), "boundary {completed}: {err}"); + for (index, path) in cleanup_paths.iter().enumerate() { + assert_eq!( + store + .objects + .lock() + .await + .contains_key(&memory_config_key(RUSTFS_META_BUCKET, path)), + index >= completed, + "boundary {completed}, slot {index}" + ); + } + let intent = read_config_with_revision(store.clone(), primary_path) + .await + .expect("intent should persist"); + let slots = read_usage_state_reset_slots(store.clone()) + .await + .expect("restart should reload slots"); + reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true) + .await + .expect("restart should complete the same intent"); + assert_eq!( + read_config_with_revision(store.clone(), primary_path) + .await + .expect("intent should remain"), + intent + ); + assert_eq!(store.put_counts.lock().await[&memory_config_key(RUSTFS_META_BUCKET, primary_path)], 2); + for path in cleanup_paths { + assert!( + !store + .objects + .lock() + .await + .contains_key(&memory_config_key(RUSTFS_META_BUCKET, &path)) + ); + } + for path in ["buckets/quota-reservations/ledger", "buckets/example/incarnation"] { + assert_eq!(read_config(store.clone(), path).await.expect("unrelated state should remain"), b"retain"); + } + } +} + +#[tokio::test] +async fn scanner_usage_state_reset_stops_usage_fence_after_owner_loss() { + let store = Arc::new(MemoryConfigStore::default()); + let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + usage.scanner_epoch = Some(1); + let bytes = serde_json::to_vec(&usage).expect("baseline should encode"); + save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes.clone()) + .await + .expect("baseline should persist"); + let checks = AtomicUsize::new(0); + let err = fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 3, Some(0), false, || { + checks.fetch_add(1, Ordering::SeqCst) == 0 + }) + .await + .expect_err("ownership lost during reads must prevent the write"); + assert!(err.to_string().contains("leadership was lost"), "{err}"); + assert_eq!( + read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("baseline should remain"), + bytes + ); +} + +#[tokio::test] +async fn scanner_usage_state_reset_cancels_during_publication_admission() { + for resuming in [false, true] { + let store = Arc::new(MemoryConfigStore::default()); + let usage = if resuming { + scanner_usage_bootstrap_marker(std::time::SystemTime::UNIX_EPOCH, Some(3)) + } else { + complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0) + }; + save_config( + store.clone(), + DATA_USAGE_OBJ_NAME_PATH.as_str(), + serde_json::to_vec(&usage).expect("primary should encode"), + ) + .await + .expect("primary should persist"); + save_config(store.clone(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), b"corrupt".to_vec()) + .await + .expect("cleanup target should persist"); + let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load"); + let before = store.objects.lock().await.clone(); + let revisions_before = store.revisions.lock().await.clone(); + let entered = Arc::new(tokio::sync::Notify::new()); + let resume = Arc::new(tokio::sync::Notify::new()); + *store.pause_next_publication_admission.lock().await = Some((entered.clone(), resume.clone())); + let cancelled = CancellationToken::new(); + let (result, ()) = tokio::join!( + reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || !cancelled.is_cancelled()), + async { + entered.notified().await; + cancelled.cancel(); + resume.notify_one(); + } + ); + let err = result.expect_err("losing ownership during admission must prevent mutation"); + assert!(err.to_string().contains("ownership was lost"), "resuming={resuming}: {err}"); + assert_eq!(*store.objects.lock().await, before); + assert_eq!(*store.revisions.lock().await, revisions_before); + } +} + +#[tokio::test] +#[serial] +async fn scanner_usage_state_reset_rejects_corruption_without_a_trusted_floor() { + let (_temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await; + save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), b"{corrupt".to_vec()) + .await + .expect("corrupt primary should persist"); + let before = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("evidence should load"); + let err = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone()) + .await + .expect_err("corruption must not become a zero floor"); + assert!(err.to_string().contains("no trusted cycle or usage floor"), "{err}"); + assert_eq!( + read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("evidence should remain"), + before + ); + assert!(matches!( + read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); + let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + backup.scanner_epoch = Some(7); + backup.scanner_cycle = Some(40); + save_config( + store.clone(), + &format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()), + serde_json::to_vec(&backup).expect("backup should encode"), + ) + .await + .expect("valid backup should persist"); + let result = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store) + .await + .expect("valid backup should supply the recovery floor"); + assert_eq!(result.leader_epoch, 8); + assert_eq!(result.next_cycle, 41); +} + +#[tokio::test] +async fn scanner_usage_state_reset_rejects_replaced_intent_and_newer_cleanup_slot() { + let store = Arc::new(MemoryConfigStore::default()); + let marker = scanner_usage_bootstrap_marker(std::time::SystemTime::UNIX_EPOCH, Some(3)); + let bytes = serde_json::to_vec(&marker).expect("marker should encode"); + save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes.clone()) + .await + .expect("intent should persist"); + let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load"); + save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes) + .await + .expect("another intent should persist"); + let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true) + .await + .expect_err("same epoch cannot replace an intent revision"); + assert!(err.to_string().contains("intent revision changed"), "{err}"); + + let mut newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + newer.scanner_epoch = Some(3); + let path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + let bytes = serde_json::to_vec(&newer).expect("newer snapshot should encode"); + save_config(store.clone(), &path, bytes.clone()) + .await + .expect("newer snapshot should persist"); + let slots = read_usage_state_reset_slots(store.clone()) + .await + .expect("slots should reload"); + let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true) + .await + .expect_err("cleanup cannot delete same-epoch progress"); + assert!(err.to_string().contains("not older than its intent"), "{err}"); + assert_eq!(read_config(store, &path).await.expect("newer snapshot should remain"), bytes); +} + +#[tokio::test] +#[serial] +async fn scanner_usage_state_reset_rejects_decodable_untrusted_floor() { + let (_temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await; + let invalid_identity = DataUsageInfo { + usage_snapshot_complete: true, + buckets_count: 1, + last_update: Some(std::time::SystemTime::UNIX_EPOCH), + ..Default::default() + }; + for usage in [DataUsageInfo::default(), invalid_identity] { + save_config( + store.clone(), + DATA_USAGE_OBJ_NAME_PATH.as_str(), + serde_json::to_vec(&usage).expect("fixture should encode"), + ) + .await + .expect("untrusted primary should persist"); + let before = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("primary should load"); + let err = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone()) + .await + .expect_err("valid JSON alone cannot prove a usage floor"); + assert!(err.to_string().contains("no trusted cycle or usage floor"), "{err}"); + assert_eq!( + read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("evidence should remain"), + before + ); + assert!(matches!( + read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); + } +} + +#[test] +fn full_rescan_reset_rejects_unknown_marker_phase_even_with_invalid_compat_fields() { + for state in [serde_json::json!("rewrite-v2"), serde_json::json!(7), serde_json::Value::Null] { + let marker = serde_json::json!({"state": state, "retry_count": "future-type", "schema_version": 99}); + let err = super::cycle_state::decode_recovery_marker_for_reset( + &serde_json::to_vec(&marker).expect("future marker should encode"), + &DataUsageCacheRevision::Etag("intent-1".to_string()), + ) + .expect_err("unknown persistent phases must remain fenced"); + assert!(err.to_string().contains("state is unsupported"), "{err}"); + } +} + +#[tokio::test] +#[serial] +async fn full_rescan_reset_preserves_unknown_phase_and_retries_completed_cleanup() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), b"corrupt".to_vec()) + .await + .expect("corrupt primary should persist"); + save_config( + store.clone(), + DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), + br#"{"state":"future-rewrite"}"#.to_vec(), + ) + .await + .expect("future marker should persist"); + let primary_before = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("primary should load"); + let marker_before = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()) + .await + .expect("marker should load"); + let err = reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .expect_err("unknown phase must block explicit reset"); + assert!(err.to_string().contains("state is unsupported"), "{err}"); + assert_eq!( + read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("primary should remain"), + primary_before + ); + assert_eq!( + read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()) + .await + .expect("marker should remain"), + marker_before + ); + + save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{malformed".to_vec()) + .await + .expect("recoverable marker should persist"); + reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .expect("reset should complete"); + let primary = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("rebuilt primary should load"); + let usage = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("fenced usage should load"); + reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .expect("retry after marker deletion should complete"); + assert_eq!( + read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("rebuilt primary should remain"), + primary + ); + assert_eq!( + read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("fenced usage should remain"), + usage + ); +} + #[tokio::test] async fn scanner_usage_state_reset_slots_defer_when_publication_epoch_moves() { let store = Arc::new(MemoryConfigStore::default()); @@ -4993,7 +5367,7 @@ async fn scanner_usage_state_reset_slots_defer_when_publication_epoch_moves() { .expect("usage reset slots should be inspected"); store.publication_admission_blocked.store(true, Ordering::Release); - let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3) + let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3, || true) .await .expect_err("movement admission loss must defer reset"); assert!( From 0d1b312673d062e7ca236d2f444728a6afcfed5e Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 16:49:58 +0800 Subject: [PATCH 10/12] fix(ci): require fresh successful scheduled validations (#7192) --- .../scheduled-validation-freshness.yml | 1 + .../check_scheduled_validation_freshness.py | 398 +++++++++++++----- 2 files changed, 293 insertions(+), 106 deletions(-) diff --git a/.github/workflows/scheduled-validation-freshness.yml b/.github/workflows/scheduled-validation-freshness.yml index eef340869..69ddc844c 100644 --- a/.github/workflows/scheduled-validation-freshness.yml +++ b/.github/workflows/scheduled-validation-freshness.yml @@ -42,6 +42,7 @@ jobs: - name: Check latest scheduled runs env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUSTFS_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} run: | set +e python3 scripts/check_scheduled_validation_freshness.py \ diff --git a/scripts/check_scheduled_validation_freshness.py b/scripts/check_scheduled_validation_freshness.py index a5812c7a3..c694e3c89 100644 --- a/scripts/check_scheduled_validation_freshness.py +++ b/scripts/check_scheduled_validation_freshness.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 -"""Fail when a critical scheduled validation has not started recently.""" +"""Require recent scheduled attempts and completed successes on the default branch.""" from __future__ import annotations import argparse from datetime import datetime, timedelta, timezone +import io import json import os from pathlib import Path @@ -13,7 +14,7 @@ import sys import tempfile import unittest from unittest import mock -from urllib.parse import quote, urlencode +from urllib.parse import parse_qs, quote, urlencode, urlsplit from urllib.request import Request, urlopen @@ -75,15 +76,8 @@ def stale_reason( run: dict[str, object] | None, now: datetime, max_age_hours: int, - never_ran_grace_until: datetime | None = None, ) -> str | None: if run is None: - # The grace deadline only covers a workflow whose first scheduled slot - # has not arrived yet (for example a monthly cron enabled mid-month). - # A recorded-but-old run proves the schedule used to fire and stopped, - # so the grace never masks that case. - if never_ran_grace_until is not None and now <= never_ran_grace_until: - return None return "no scheduled run has been recorded" created_at = parse_timestamp(run.get("created_at")) age = now - created_at @@ -93,14 +87,23 @@ def stale_reason( def fetch_latest_scheduled_run( - repository: str, workflow: str, token: str, api_url: str + repository: str, + workflow: str, + token: str, + api_url: str, + default_branch: str, + successful: bool = False, ) -> dict[str, object] | None: owner, repo = repository.split("/", 1) workflow_name = Path(workflow).name + query = {"event": "schedule", "branch": default_branch, "per_page": 1} + if successful: + # Filter on the server: the last success may be beyond a page of failures. + query["status"] = "success" endpoint = ( f"{api_url.rstrip('/')}/repos/{quote(owner, safe='')}/{quote(repo, safe='')}" f"/actions/workflows/{quote(workflow_name, safe='')}/runs?" - + urlencode({"event": "schedule", "per_page": 1}) + + urlencode(query) ) request = Request( endpoint, @@ -110,57 +113,104 @@ def fetch_latest_scheduled_run( "X-GitHub-Api-Version": "2022-11-28", }, ) - with urlopen(request, timeout=30) as response: + # Two requests per manifest entry must fit the watchdog's ten-minute job. + with urlopen(request, timeout=15) as response: payload = json.load(response) - runs = payload.get("workflow_runs") + runs = payload.get("workflow_runs") if isinstance(payload, dict) else None if not isinstance(runs, list): raise ValueError(f"GitHub returned no workflow_runs list for {workflow}") + total_count = payload.get("total_count") + if not isinstance(total_count, int) or isinstance(total_count, bool) or total_count < len(runs): + raise ValueError(f"GitHub returned an invalid run count for {workflow}") if not runs: + if total_count: + raise ValueError(f"GitHub returned an empty first page with recorded runs for {workflow}") return None - if not isinstance(runs[0], dict): + run = runs[0] + if not isinstance(run, dict): raise ValueError(f"GitHub returned an invalid workflow run for {workflow}") - return runs[0] + if run.get("event") != "schedule" or run.get("head_branch") != default_branch: + raise ValueError(f"GitHub returned a run outside the scheduled default-branch query for {workflow}") + if not isinstance(run.get("status"), str) or not run["status"]: + raise ValueError(f"GitHub returned no run status for {workflow}") + conclusion = run.get("conclusion") + if (conclusion is not None and not isinstance(conclusion, str)) or ( + run["status"] == "completed" and not conclusion + ): + raise ValueError(f"GitHub returned an invalid run conclusion for {workflow}") + if successful and (run["status"] != "completed" or conclusion != "success"): + raise ValueError(f"GitHub returned a run without a completed success for {workflow}") + parse_timestamp(run.get("created_at")) + if not isinstance(run.get("html_url"), str) or not run["html_url"]: + raise ValueError(f"GitHub returned no run URL for {workflow}") + return run -def write_report(path: Path, failures: list[tuple[str, int, str, str]]) -> None: - lines = ["## Scheduled validation freshness"] - if not failures: - lines.append("") - lines.append("All critical scheduled validations have a recent scheduled run.") - else: - lines.extend( - [ - "", - "The following critical validations are stale or could not be inspected:", - "", - "| Workflow | Limit | Result | Last run |", - "| --- | ---: | --- | --- |", - ] - ) - for workflow, max_age_hours, reason, run_url in failures: - link = f"[open]({run_url})" if run_url else "—" - lines.append(f"| `{workflow}` | {max_age_hours}h | {reason} | {link} |") +def describe_run(run: dict[str, object] | None) -> str: + if run is None: + return "No recorded run" + outcome = run["status"] + if run.get("conclusion"): + outcome = f"{outcome}/{run['conclusion']}" + return f"[{outcome}]({run['html_url']}) — created {run['created_at']}" + + +def write_report(path: Path, rows: list[tuple[str, int, str, str, str]], default_branch: str) -> None: + lines = [ + "## Scheduled validation freshness", + "", + f"Default branch: `{default_branch}`. Ages use scheduled-run creation time; rerunning an old commit does not refresh its evidence.", + "Attempt outcomes are shown independently of successful-run freshness.", + "Success is the GitHub workflow run conclusion; suite completeness remains the responsibility of each workflow.", + "", + "| Workflow | Limit | Freshness | Last attempt | Last completed success |", + "| --- | ---: | --- | --- | --- |", + ] + for workflow, max_age_hours, result, attempt, success in rows: + cells = [f"`{workflow}`", f"{max_age_hours}h", result, attempt, success] + lines.append("| " + " | ".join(cell.replace("|", "\\|").replace("\n", " ") for cell in cells) + " |") path.write_text("\n".join(lines) + "\n") def check_freshness( - config: Path, report: Path, repository: str, token: str, api_url: str + config: Path, report: Path, repository: str, token: str, api_url: str, default_branch: str ) -> int: now = datetime.now(timezone.utc) - failures: list[tuple[str, int, str, str]] = [] + rows: list[tuple[str, int, str, str, str]] = [] + failed = False for workflow, max_age_hours, never_ran_grace_until in load_validations(config): - try: - run = fetch_latest_scheduled_run(repository, workflow, token, api_url) - reason = stale_reason(run, now, max_age_hours, never_ran_grace_until) - if reason is not None: - run_url = str(run.get("html_url", "")) if run else "" - failures.append((workflow, max_age_hours, reason, run_url)) - except Exception as error: - failures.append( - (workflow, max_age_hours, f"inspection failed: {error}", "") - ) - write_report(report, failures) - return 1 if failures else 0 + runs: dict[str, dict[str, object] | None] = {} + reasons: list[str] = [] + for label, successful in (("Last attempt", False), ("Last completed success", True)): + try: + runs[label] = fetch_latest_scheduled_run( + repository, workflow, token, api_url, default_branch, successful + ) + except Exception as error: + reasons.append(f"{label}: inspection failed: {error}") + # A failed inspection or any recorded attempt ends first-run grace. + initial_grace = ( + len(runs) == 2 + and all(run is None for run in runs.values()) + and never_ran_grace_until is not None + and now <= never_ran_grace_until + ) + if not initial_grace: + for label, run in runs.items(): + reason = stale_reason(run, now, max_age_hours) + if reason is not None: + reasons.append(f"{label}: {reason}") + failed |= bool(reasons) + result = "; ".join(reasons) if reasons else "Fresh" + if initial_grace: + result = f"Initial grace until {never_ran_grace_until.isoformat()}" + evidence = [ + describe_run(runs[label]) if label in runs else "Inspection failed" + for label in ("Last attempt", "Last completed success") + ] + rows.append((workflow, max_age_hours, result, *evidence)) + write_report(report, rows, default_branch) + return 1 if failed else 0 class SelfTests(unittest.TestCase): @@ -173,15 +223,6 @@ class SelfTests(unittest.TestCase): self.assertIsNotNone(stale_reason(past_limit, self.NOW, 36)) self.assertIsNotNone(stale_reason(None, self.NOW, 36)) - def test_never_ran_grace_only_covers_missing_runs(self) -> None: - future_grace = self.NOW + timedelta(hours=1) - past_grace = self.NOW - timedelta(seconds=1) - self.assertIsNone(stale_reason(None, self.NOW, 36, future_grace)) - self.assertIsNone(stale_reason(None, self.NOW, 36, self.NOW)) - self.assertIsNotNone(stale_reason(None, self.NOW, 36, past_grace)) - stale_run = {"created_at": "2026-08-20T23:59:59Z"} - self.assertIsNotNone(stale_reason(stale_run, self.NOW, 36, future_grace)) - def test_config_rejects_duplicate_and_invalid_entries(self) -> None: with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "validations.json" @@ -238,63 +279,205 @@ class SelfTests(unittest.TestCase): ], ) - def test_check_reports_missing_runs(self) -> None: + @staticmethod + def run_fixture(**overrides: object) -> dict[str, object]: + return { + "status": "completed", + "conclusion": "success", + "event": "schedule", + "head_branch": "release/current", + "created_at": "2026-08-22T00:00:00Z", + "html_url": "https://github.test/rustfs/rustfs/actions/runs/1", + **overrides, + } + + def check_payloads( + self, payloads: list[object], *, grace: str | None = None, workflows: int = 1 + ) -> tuple[int, str, list]: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) config = root / "validations.json" report = root / "report.md" - config.write_text( - json.dumps( - [ - {"workflow": ".github/workflows/ci.yml", "max_age_hours": 36}, - {"workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36}, - {"workflow": ".github/workflows/mint.yml", "max_age_hours": 36}, - ] - ) - ) - with mock.patch( - __name__ + ".fetch_latest_scheduled_run", - side_effect=[ - {"created_at": "2999-01-01T00:00:00Z"}, - None, - RuntimeError("API unavailable"), - ], + entries = [ + {"workflow": f".github/workflows/check-{index}.yml", "max_age_hours": 36} + for index in range(workflows) + ] + if grace is not None: + entries[0]["never_ran_grace_until"] = grace + config.write_text(json.dumps(entries)) + responses = [] + for payload in payloads: + if isinstance(payload, dict) and isinstance(payload.get("workflow_runs"), list): + payload = {"total_count": len(payload["workflow_runs"]), **payload} + responses.append(payload if isinstance(payload, Exception) else io.StringIO(json.dumps(payload))) + with ( + mock.patch(__name__ + ".urlopen", side_effect=responses) as request, + mock.patch(__name__ + ".datetime", wraps=datetime) as clock, ): - self.assertEqual( - check_freshness( - config, - report, - "rustfs/rustfs", - "token", - "https://api.github.test", - ), - 1, + clock.now.return_value = self.NOW + status = check_freshness( + config, report, "rustfs/rustfs", "test-token", + "https://api.github.test", "release/current", ) - contents = report.read_text() - self.assertIn(".github/workflows/fuzz.yml", contents) - self.assertIn("inspection failed: API unavailable", contents) - self.assertNotIn(".github/workflows/ci.yml`", contents) + return status, report.read_text(), request.call_args_list - config.write_text( - json.dumps( - [{"workflow": ".github/workflows/ci.yml", "max_age_hours": 36}] + def test_requests_filter_schedule_default_branch_and_success_on_server(self) -> None: + attempt = self.run_fixture(status="in_progress", conclusion=None) + success = self.run_fixture(html_url="https://github.test/rustfs/rustfs/actions/runs/2") + status, report, calls = self.check_payloads([ + {"workflow_runs": [attempt], "total_count": 1001}, + {"workflow_runs": [success], "total_count": 1}, + ]) + self.assertEqual(status, 0) + self.assertEqual(len(calls), 2) + for call, successful in zip(calls, (False, True)): + request = call.args[0] + url = urlsplit(request.full_url) + self.assertEqual(url.path, "/repos/rustfs/rustfs/actions/workflows/check-0.yml/runs") + expected = {"event": ["schedule"], "branch": ["release/current"], "per_page": ["1"]} + if successful: + expected["status"] = ["success"] + self.assertEqual(parse_qs(url.query), expected) + self.assertEqual(request.get_header("Authorization"), "Bearer test-token") + self.assertEqual(call.kwargs, {"timeout": 15}) + self.assertIn("[in_progress]", report) + self.assertIn(str(attempt["html_url"]), report) + self.assertIn(str(success["html_url"]), report) + + def test_cancelled_attempt_cannot_refresh_expired_success(self) -> None: + attempt = self.run_fixture(conclusion="cancelled") + success = self.run_fixture( + created_at="2026-08-20T23:59:59Z", updated_at="2026-08-22T11:59:59Z", + html_url="https://github.test/rustfs/rustfs/actions/runs/2", + ) + status, report, _ = self.check_payloads([ + {"workflow_runs": [attempt]}, {"workflow_runs": [success]}, + ]) + self.assertEqual(status, 1) + self.assertIn("Last completed success: last scheduled run is", report) + self.assertIn("[completed/cancelled]", report) + for run in (attempt, success): + self.assertIn(str(run["html_url"]), report) + self.assertIn(str(run["created_at"]), report) + + def test_attempt_outcome_does_not_replace_recent_success(self) -> None: + success = self.run_fixture(created_at="2026-08-21T00:00:00Z") + for state, conclusion in ( + ("completed", "failure"), ("completed", "cancelled"), + ("completed", "timed_out"), ("completed", "success"), + ("queued", None), ("in_progress", None), + ): + with self.subTest(state=state, conclusion=conclusion): + status, report, _ = self.check_payloads([ + {"workflow_runs": [self.run_fixture(status=state, conclusion=conclusion)]}, + {"workflow_runs": [success]}, + ]) + self.assertEqual(status, 0) + self.assertIn(f"[{state}" + (f"/{conclusion}" if conclusion else "") + "]", report) + self.assertIn("Fresh", report) + self.assertNotIn("All critical scheduled validations", report) + + def test_grace_requires_two_successful_queries_with_no_history(self) -> None: + for attempt, success, grace, expected in ( + (None, None, "2026-08-22T12:00:00Z", 0), + (None, None, "2026-08-22T11:59:59Z", 1), + (self.run_fixture(conclusion="failure"), None, "2026-08-23T00:00:00Z", 1), + (self.run_fixture(status="queued", conclusion=None), None, "2026-08-23T00:00:00Z", 1), + (None, self.run_fixture(), "2026-08-23T00:00:00Z", 1), + ): + with self.subTest(attempt=attempt, success=success, grace=grace): + status, report, _ = self.check_payloads([ + {"workflow_runs": [] if attempt is None else [attempt]}, + {"workflow_runs": [] if success is None else [success]}, + ], grace=grace) + self.assertEqual(status, expected) + self.assertEqual("Initial grace until" in report, expected == 0) + + def test_api_failures_preserve_other_evidence_and_never_enter_grace(self) -> None: + good = {"workflow_runs": [self.run_fixture()]} + for first, second in ( + (RuntimeError("API unavailable"), good), + (good, RuntimeError("API unavailable")), + (RuntimeError("API unavailable"), {"workflow_runs": []}), + ): + with self.subTest(first=first, second=second): + status, report, calls = self.check_payloads( + [first, second], grace="2026-08-23T00:00:00Z" ) - ) - with mock.patch( - __name__ + ".fetch_latest_scheduled_run", - return_value={"created_at": "2999-01-01T00:00:00Z"}, - ): - self.assertEqual( - check_freshness( - config, - report, - "rustfs/rustfs", - "token", - "https://api.github.test", - ), - 0, - ) - self.assertIn("All critical scheduled validations", report.read_text()) + self.assertEqual(status, 1) + self.assertEqual(len(calls), 2) + self.assertIn("inspection failed: API unavailable", report) + self.assertNotIn("Initial grace until", report) + if first is good or second is good: + self.assertIn(str(self.run_fixture()["html_url"]), report) + + def test_invalid_api_evidence_fails_closed(self) -> None: + malformed = [ + [], {}, {"workflow_runs": {}}, {"workflow_runs": [None]}, + {"workflow_runs": [], "total_count": 1}, + {"workflow_runs": [], "total_count": -1}, + {"workflow_runs": [], "total_count": None}, + {"workflow_runs": [], "total_count": True}, + *({"workflow_runs": [self.run_fixture(**override)]} for override in ( + {"event": "workflow_dispatch"}, {"head_branch": "other"}, + {"created_at": "invalid"}, {"created_at": "2026-08-22T00:00:00"}, + {"status": None}, {"conclusion": None}, {"conclusion": 1}, + {"html_url": ""}, + )), + ] + for payload in malformed: + for index, label in enumerate(("Last attempt", "Last completed success")): + with self.subTest(payload=payload, label=label): + payloads = [{"workflow_runs": [self.run_fixture()]} for _ in range(2)] + payloads[index] = payload + status, report, _ = self.check_payloads(payloads, grace="2026-08-23T00:00:00Z") + self.assertEqual(status, 1) + self.assertIn(f"{label}: inspection failed", report) + self.assertNotIn("Initial grace until", report) + self.assertIn(str(self.run_fixture()["html_url"]), report) + for state, conclusion in (("in_progress", "success"), ("completed", "failure"), ("completed", "skipped")): + with self.subTest(state=state, conclusion=conclusion): + status, report, _ = self.check_payloads([ + {"workflow_runs": [self.run_fixture()]}, + {"workflow_runs": [self.run_fixture(status=state, conclusion=conclusion)]}, + ]) + self.assertEqual(status, 1) + self.assertIn("without a completed success", report) + + def test_report_retains_every_workflow(self) -> None: + status, report, calls = self.check_payloads([ + {"workflow_runs": [self.run_fixture()]}, {"workflow_runs": [self.run_fixture()]}, + {"workflow_runs": []}, {"workflow_runs": []}, + RuntimeError("API unavailable"), {"workflow_runs": [self.run_fixture()]}, + ], workflows=3) + self.assertEqual(status, 1) + self.assertEqual(len(calls), 6) + for index in range(3): + self.assertEqual(report.count(f"`.github/workflows/check-{index}.yml`"), 1) + self.assertIn("No recorded run", report) + self.assertIn("Inspection failed", report) + + def test_cli_requires_the_repository_default_branch(self) -> None: + from check_test_wiring import yaml_block + + workflow = (ROOT / ".github/workflows/scheduled-validation-freshness.yml").read_text().splitlines() + job = yaml_block(workflow, "check-freshness", 2) + self.assertIsNotNone(job) + start = job.index(" - name: Check latest scheduled runs") + end = next((index for index in range(start + 1, len(job)) if job[index].startswith(" - ")), len(job)) + environment = yaml_block(job[start:end], "env", 8) + self.assertIsNotNone(environment) + self.assertIn(" RUSTFS_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}", environment) + + with ( + mock.patch.dict(os.environ, {"GITHUB_REPOSITORY": "rustfs/rustfs", "GH_TOKEN": "test-token"}, clear=True), + mock.patch.object(sys, "argv", ["checker", "--report", "unused.md"]), + mock.patch("sys.stderr", new=io.StringIO()) as stderr, + self.assertRaises(SystemExit) as error, + ): + main() + self.assertEqual(error.exception.code, 2) + self.assertIn("RUSTFS_DEFAULT_BRANCH", stderr.getvalue()) def main() -> int: @@ -318,11 +501,14 @@ def main() -> int: repository = os.environ.get("GITHUB_REPOSITORY", "") token = os.environ.get("GH_TOKEN", "") api_url = os.environ.get("GITHUB_API_URL", "https://api.github.com") + default_branch = os.environ.get("RUSTFS_DEFAULT_BRANCH", "") if not re.fullmatch(r"[^/\s]+/[^/\s]+", repository): parser.error("GITHUB_REPOSITORY must be owner/repository") if not token: parser.error("GH_TOKEN is required") - return check_freshness(args.config, args.report, repository, token, api_url) + if not default_branch or any(character.isspace() for character in default_branch): + parser.error("RUSTFS_DEFAULT_BRANCH is required and must name the repository default branch") + return check_freshness(args.config, args.report, repository, token, api_url, default_branch) if __name__ == "__main__": From acfeef55abc1a68c285d23b5dd301ed08f822337 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 5 Sep 2026 16:54:55 +0800 Subject: [PATCH 11/12] feat(scanner): add bounded incarnation-scoped ACK receiver (#7182) * chore(deps): refresh SDKs and pin clock skew regression coverage Refresh compatible dependencies for Scanner/Heal V2 batch 1 and verify the production S3 retry/signing path with a deterministic clock. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * feat(scanner): add bounded incarnation-scoped ACK receiver Refs rustfs/backlog#2265 and rustfs/backlog#2240. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/ecstore/src/api/mod.rs | 19 +- crates/ecstore/src/bucket/metadata_sys.rs | 121 +++++++- crates/ecstore/src/cluster/rpc/client.rs | 19 ++ .../src/cluster/rpc/peer_rest_client.rs | 47 +++ .../src/generated/proto_gen/node_service.rs | 286 ++++++++++++++++++ crates/protos/src/lib.rs | 2 + crates/protos/src/node.proto | 34 +++ crates/protos/src/scoped_dirty_usage.rs | 213 +++++++++++++ crates/scanner/src/lib.rs | 5 +- crates/scanner/src/scanner_io.rs | 5 +- crates/scanner/src/scanner_io/dirty_usage.rs | 106 +++++++ crates/scanner/src/storage_api.rs | 4 +- rustfs/src/server/http.rs | 47 +++ rustfs/src/storage/rpc/node_service.rs | 200 ++++++++++++ rustfs/src/storage/storage_api.rs | 10 +- rustfs/src/storage/tonic_service.rs | 1 + rustfs/src/storage_api.rs | 2 +- 17 files changed, 1097 insertions(+), 24 deletions(-) create mode 100644 crates/protos/src/scoped_dirty_usage.rs diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 34e784288..437f5c458 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -196,15 +196,16 @@ pub mod bucket { pub use crate::bucket::metadata_sys::ConfigWriteLockProbe; pub use crate::bucket::metadata_sys::{ BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock, - acquire_bucket_metadata_transaction_lock_for_incarnation, capture_bucket_metadata_incarnation, delete, - delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy, - get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config, - get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config, - get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config, get_public_access_block_config, - get_quota_config, get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, - get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, - remove_bucket_metadata, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, - update_config_with, update_if_incarnation, update_quota_if_incarnation, update_under_transaction_lock, + acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence, + capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get, + get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, + get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, + get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config, + get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config, + get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, + reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, update, + update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, update_quota_if_incarnation, + update_under_transaction_lock, }; } diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index eb40ed8c9..4a53927f9 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -655,6 +655,12 @@ pub struct BucketMetadataMutationGuard { } impl BucketMetadataMutationGuard { + /// Returns the storage-verified identity while both incarnation fences remain valid. + pub fn checked_bucket_incarnation(&self) -> Result<(&str, Uuid)> { + self.ensure_valid(&self.bucket)?; + Ok((&self.bucket, self.incarnation_id)) + } + fn ensure_valid(&self, bucket: &str) -> Result<()> { if self.bucket != bucket { return Err(Error::other("bucket metadata mutation guard does not match bucket")); @@ -674,6 +680,29 @@ async fn acquire_config_write_guard_for_incarnation( sys: Arc>, bucket: &str, expected_incarnation_id: Option, +) -> Result { + acquire_config_write_guard_with_migration(sys, bucket, expected_incarnation_id, true).await +} + +/// Scanner probes must not create an incarnation to make a capability available. +pub async fn acquire_scanner_bucket_incarnation_fence( + bucket: &str, + expected_incarnation_id: Uuid, + expected_owner_id: Uuid, +) -> Result { + super::utils::check_valid_bucket_name(bucket)?; + let sys = get_bucket_metadata_sys()?; + if expected_owner_id.is_nil() || sys.read().await.api.id != expected_owner_id || expected_incarnation_id.is_nil() { + return Err(Error::other("scanner bucket incarnation owner does not match")); + } + acquire_config_write_guard_with_migration(sys, bucket, Some(expected_incarnation_id), false).await +} + +async fn acquire_config_write_guard_with_migration( + sys: Arc>, + bucket: &str, + expected_incarnation_id: Option, + migrate: bool, ) -> Result { let metadata_sys = sys.read().await.clone(); let lifecycle_guard = metadata_sys.api.acquire_bucket_lifecycle_read_lock(bucket).await?; @@ -681,13 +710,15 @@ async fn acquire_config_write_guard_for_incarnation( // Legacy buckets are migrated while the lifecycle fence prevents a // same-name replacement. The second read under the write transaction is // the CAS source of truth for the actual rewrite. - await_bucket_namespace_operation( - Some(&lifecycle_guard), - bucket, - "bucket config incarnation migration", - metadata_sys.get_bucket_incarnation_id(bucket), - ) - .await?; + if migrate { + await_bucket_namespace_operation( + Some(&lifecycle_guard), + bucket, + "bucket config incarnation migration", + metadata_sys.get_bucket_incarnation_id(bucket), + ) + .await?; + } let transaction_guard = await_bucket_namespace_operation( Some(&lifecycle_guard), bucket, @@ -3176,6 +3207,82 @@ mod tests { ); } + #[tokio::test] + async fn scoped_dirty_usage_incarnation_probe_does_not_migrate_legacy_metadata() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let sys = Arc::new(RwLock::new(BucketMetadataSys::new(store.clone()))); + let bucket = "scoped-ack-legacy"; + for dir in &dirs { + std::fs::create_dir_all(dir.path().join(bucket)).expect("create legacy bucket"); + } + let mut metadata = BucketMetadata::new(bucket); + metadata.bucket_incarnation_id = Uuid::nil(); + sys.read() + .await + .persist_and_set(metadata) + .await + .expect("persist legacy metadata"); + assert!( + acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(Uuid::new_v4()), false) + .await + .is_err() + ); + assert!(load_bucket_incarnation(store, bucket).await.expect("read sidecar").is_none()); + assert!( + sys.read() + .await + .get_config_from_disk(bucket) + .await + .expect("read metadata") + .bucket_incarnation_id + .is_nil() + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[serial] + async fn scoped_dirty_usage_incarnation_rejects_deleted_and_recreated_bucket() { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner"); + let bucket = "scoped-ack-recreated"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let old = store.bucket_incarnation_id_from_disk(bucket).await.expect("old incarnation"); + let guard = acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false) + .await + .expect("trusted incarnation fence"); + assert_eq!(guard.checked_bucket_incarnation().expect("valid fences"), (bucket, old)); + drop(guard); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("delete bucket"); + assert!( + acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false) + .await + .is_err() + ); + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("recreate bucket"); + let new = store.bucket_incarnation_id_from_disk(bucket).await.expect("new incarnation"); + assert_ne!(old, new); + assert!( + acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false) + .await + .is_err() + ); + assert!( + acquire_config_write_guard_with_migration(sys, bucket, Some(new), false) + .await + .is_ok() + ); + } + #[tokio::test] async fn old_node_metadata_rewrite_cannot_replace_bucket_incarnation_sidecar() { let (dirs, ecstore) = isolated_store_over_temp_disks().await; diff --git a/crates/ecstore/src/cluster/rpc/client.rs b/crates/ecstore/src/cluster/rpc/client.rs index 435cbf64c..c4bf786f9 100644 --- a/crates/ecstore/src/cluster/rpc/client.rs +++ b/crates/ecstore/src/cluster/rpc/client.rs @@ -30,6 +30,7 @@ use rustfs_protos::{ ChannelClass, create_new_channel, get_channel_for_class, proto_gen::node_service::{ heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient, + scanner_control_service_client::ScannerControlServiceClient, tier_mutation_control_service_client::TierMutationControlServiceClient, }, }; @@ -60,6 +61,24 @@ pub async fn node_service_time_out_client( node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await } +pub(crate) async fn scanner_control_time_out_client( + addr: &str, + interceptor: TonicInterceptor, +) -> crate::error::Result>> { + let interceptor = interceptor.with_rpc_audience(addr)?; + let channel = match runtime_sources::cached_node_channel(addr).await { + Some(channel) => channel, + None => create_new_channel(addr) + .await + .map_err(|err| crate::error::Error::other(err.to_string()))?, + }; + let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience()); + let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize; + Ok(ScannerControlServiceClient::with_interceptor(channel, interceptor) + .max_decoding_message_size(limit) + .max_encoding_message_size(limit)) +} + pub async fn heal_control_time_out_client( addr: &str, interceptor: TonicInterceptor, diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index 5316199e3..9d57c4647 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -2050,6 +2050,53 @@ impl PeerRestClient { .await } + /// Probe only: scoped ACK production requires a durable per-bucket proof. + pub async fn scanner_scoped_dirty_usage_capability( + &self, + owner_id: String, + instance_id: String, + entries: Vec, + ) -> Result { + use rustfs_protos::scoped_dirty_usage::*; + let payload = rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest { + challenge: Uuid::new_v4().as_bytes().to_vec().into(), + protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION, + owner_id, + instance_id, + scope: SCOPED_DIRTY_USAGE_BUCKET_SCOPE, + probe_only: true, + entries, + }; + let canonical = canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?; + self.finalize_result( + async { + let mut client = super::client::scanner_control_time_out_client( + &self.grid_host, + TonicInterceptor::Signature(gen_tonic_signature_interceptor()), + ) + .await?; + let mut request = Request::new(payload.clone()); + set_tonic_canonical_body_digest(&mut request, &canonical)?; + let response = client.scanner_scoped_dirty_usage_ack(request).await?.into_inner(); + let body = canonical_scoped_dirty_usage_response(&canonical, &response) + .map_err(|_| Error::other("scoped dirty usage capability response is too large"))?; + verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?; + if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION + || response.owner_id != payload.owner_id + || response.instance_id != payload.instance_id + || response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES + || response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES + || response.cleared != 0 + { + return Err(Error::other("scoped dirty usage capability response does not match request")); + } + Ok(response.supported) + } + .await, + ) + .await + } + pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result { let result = self .scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION) diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index f015305f0..fe04d93ab 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -1283,6 +1283,54 @@ pub struct ScannerDirtyUsageSnapshotResponse { #[prost(bytes = "bytes", tag = "7")] pub response_proof: ::prost::bytes::Bytes, } +/// Receiver-only protocol. Producers must retain whole-cycle ACK until they +/// have a durable per-bucket publication proof. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ScannerScopedDirtyUsageEntry { + #[prost(string, tag = "1")] + pub bucket: ::prost::alloc::string::String, + #[prost(bytes = "bytes", tag = "2")] + pub bucket_incarnation: ::prost::bytes::Bytes, + #[prost(uint64, tag = "3")] + pub generation: u64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ScannerScopedDirtyUsageAckRequest { + #[prost(bytes = "bytes", tag = "1")] + pub challenge: ::prost::bytes::Bytes, + #[prost(uint32, tag = "2")] + pub protocol_version: u32, + #[prost(string, tag = "3")] + pub owner_id: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub instance_id: ::prost::alloc::string::String, + /// Only scope 1 (a complete bucket) is supported; zero is invalid. + #[prost(uint32, tag = "5")] + pub scope: u32, + #[prost(bool, tag = "6")] + pub probe_only: bool, + #[prost(message, repeated, tag = "7")] + pub entries: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ScannerScopedDirtyUsageAckResponse { + #[prost(uint32, tag = "1")] + pub protocol_version: u32, + #[prost(string, tag = "2")] + pub owner_id: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub instance_id: ::prost::alloc::string::String, + #[prost(bool, tag = "4")] + pub supported: bool, + #[prost(uint32, tag = "5")] + pub max_entries: u32, + #[prost(uint32, tag = "6")] + pub max_request_bytes: u32, + #[prost(uint64, tag = "7")] + pub cleared: u64, + #[prost(bytes = "bytes", tag = "8")] + pub response_proof: ::prost::bytes::Bytes, +} /// A short-lived storage-owned read admission used only around a final /// scanner metadata publication. It is intentionally separate from the /// ScannerActivity observation wire so v6/v7 rolling compatibility remains @@ -6282,6 +6330,244 @@ pub mod node_service_server { } } /// Generated client implementations. +pub mod scanner_control_service_client { + #![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)] + use tonic::codegen::http::Uri; + use tonic::codegen::*; + #[derive(Debug, Clone)] + pub struct ScannerControlServiceClient { + inner: tonic::client::Grpc, + } + impl ScannerControlServiceClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl ScannerControlServiceClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor(inner: T, interceptor: F) -> ScannerControlServiceClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response<>::ResponseBody>, + >, + >>::Error: + Into + std::marker::Send + std::marker::Sync, + { + ScannerControlServiceClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn scanner_scoped_dirty_usage_ack( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.ScannerControlService", "ScannerScopedDirtyUsageAck")); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated server implementations. +pub mod scanner_control_service_server { + #![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with ScannerControlServiceServer. + #[async_trait] + pub trait ScannerControlService: std::marker::Send + std::marker::Sync + 'static { + async fn scanner_scoped_dirty_usage_ack( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + } + #[derive(Debug)] + pub struct ScannerControlServiceServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl ScannerControlServiceServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for ScannerControlServiceServer + where + T: ScannerControlService, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/node_service.ScannerControlService/ScannerScopedDirtyUsageAck" => { + #[allow(non_camel_case_types)] + struct ScannerScopedDirtyUsageAckSvc(pub Arc); + impl tonic::server::UnaryService + for ScannerScopedDirtyUsageAckSvc + { + type Response = super::ScannerScopedDirtyUsageAckResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::scanner_scoped_dirty_usage_ack(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ScannerScopedDirtyUsageAckSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => Box::pin(async move { + let mut response = http::Response::new(tonic::body::Body::default()); + let headers = response.headers_mut(); + headers.insert(tonic::Status::GRPC_STATUS, (tonic::Code::Unimplemented as i32).into()); + headers.insert(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE); + Ok(response) + }), + } + } + } + impl Clone for ScannerControlServiceServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "node_service.ScannerControlService"; + impl tonic::server::NamedService for ScannerControlServiceServer { + const NAME: &'static str = SERVICE_NAME; + } +} +/// Generated client implementations. pub mod heal_control_service_client { #![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)] use tonic::codegen::http::Uri; diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index fd8704382..81ead7d65 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -541,6 +541,8 @@ pub fn canonical_scanner_activity_v7_response_body( Ok(body) } +pub mod scoped_dirty_usage; + pub fn canonical_scanner_dirty_usage_snapshot_request_body( request: &proto_gen::node_service::ScannerDirtyUsageSnapshotRequest, ) -> Result, std::num::TryFromIntError> { diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index d3796991c..abc030adc 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -903,6 +903,36 @@ message ScannerDirtyUsageSnapshotResponse { bytes response_proof = 7; } +// Receiver-only protocol. Producers must retain whole-cycle ACK until they +// have a durable per-bucket publication proof. +message ScannerScopedDirtyUsageEntry { + string bucket = 1; + bytes bucket_incarnation = 2; + uint64 generation = 3; +} + +message ScannerScopedDirtyUsageAckRequest { + bytes challenge = 1; + uint32 protocol_version = 2; + string owner_id = 3; + string instance_id = 4; + // Only scope 1 (a complete bucket) is supported; zero is invalid. + uint32 scope = 5; + bool probe_only = 6; + repeated ScannerScopedDirtyUsageEntry entries = 7; +} + +message ScannerScopedDirtyUsageAckResponse { + uint32 protocol_version = 1; + string owner_id = 2; + string instance_id = 3; + bool supported = 4; + uint32 max_entries = 5; + uint32 max_request_bytes = 6; + uint64 cleared = 7; + bytes response_proof = 8; +} + // A short-lived storage-owned read admission used only around a final // scanner metadata publication. It is intentionally separate from the // ScannerActivity observation wire so v6/v7 rolling compatibility remains @@ -1245,6 +1275,10 @@ service NodeService { rpc GetLiveEvents(GetLiveEventsRequest) returns (GetLiveEventsResponse) {}; // auth-policy: read-only } +service ScannerControlService { + rpc ScannerScopedDirtyUsageAck(ScannerScopedDirtyUsageAckRequest) returns (ScannerScopedDirtyUsageAckResponse) {}; // auth-policy: body-bound +} + service HealControlService { rpc HealControl(HealControlRequest) returns (HealControlResponse) {}; } diff --git a/crates/protos/src/scoped_dirty_usage.rs b/crates/protos/src/scoped_dirty_usage.rs new file mode 100644 index 000000000..ac3effaf7 --- /dev/null +++ b/crates/protos/src/scoped_dirty_usage.rs @@ -0,0 +1,213 @@ +// Copyright 2024 RustFS Team +// Licensed under the Apache License, Version 2.0. + +//! Bounded, authenticated receiver contract for per-bucket dirty acknowledgements. + +use crate::CanonicalBodyBuilder; +use crate::proto_gen::node_service::{ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse}; +use prost::Message; + +pub const SCOPED_DIRTY_USAGE_PROTOCOL_VERSION: u32 = 1; +pub const SCOPED_DIRTY_USAGE_BUCKET_SCOPE: u32 = 1; +pub const SCOPED_DIRTY_USAGE_MAX_ENTRIES: u32 = 32; +pub const SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES: u32 = 8192; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ScopedDirtyUsageRequestError { + UnsupportedProtocol, + UnsupportedScope, + InvalidIdentity, + InvalidGeneration, + InvalidEntries, + TooLarge, +} + +impl std::fmt::Display for ScopedDirtyUsageRequestError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::UnsupportedProtocol => "unsupported scoped dirty usage protocol", + Self::UnsupportedScope => "unsupported scoped dirty usage scope", + Self::InvalidIdentity => "invalid scoped dirty usage identity", + Self::InvalidGeneration => "invalid scoped dirty usage generation", + Self::InvalidEntries => "scoped dirty usage entries must be nonempty and strictly ordered", + Self::TooLarge => "scoped dirty usage request exceeds its budget", + }) + } +} + +impl std::error::Error for ScopedDirtyUsageRequestError {} + +pub fn validate_scoped_dirty_usage_request( + request: &ScannerScopedDirtyUsageAckRequest, +) -> Result<(), ScopedDirtyUsageRequestError> { + use ScopedDirtyUsageRequestError as E; + if request.entries.len() > SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize + || request.encoded_len() > SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize + { + return Err(E::TooLarge); + } + if request.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION { + return Err(E::UnsupportedProtocol); + } + if request.scope != SCOPED_DIRTY_USAGE_BUCKET_SCOPE { + return Err(E::UnsupportedScope); + } + if request.challenge.len() != 16 || request.owner_id.len() != 36 || request.instance_id.len() != 32 { + return Err(E::InvalidIdentity); + } + if request.entries.is_empty() || request.entries.windows(2).any(|pair| pair[0].bucket >= pair[1].bucket) { + return Err(E::InvalidEntries); + } + for entry in &request.entries { + if entry.bucket.is_empty() + || entry.bucket.len() > 63 + || entry.bucket_incarnation.len() != 16 + || entry.bucket_incarnation.iter().all(|byte| *byte == 0) + { + return Err(E::InvalidIdentity); + } + if entry.generation == 0 || entry.generation == u64::MAX { + return Err(E::InvalidGeneration); + } + } + Ok(()) +} + +pub fn canonical_scoped_dirty_usage_request( + request: &ScannerScopedDirtyUsageAckRequest, +) -> Result, ScopedDirtyUsageRequestError> { + validate_scoped_dirty_usage_request(request)?; + let mut body = CanonicalBodyBuilder::new(b"rustfs-scoped-dirty-usage-ack-request-v1\0"); + let encode = |_: std::num::TryFromIntError| ScopedDirtyUsageRequestError::TooLarge; + body.push_bytes(request.challenge.as_ref()).map_err(encode)?; + body.push_u32(request.protocol_version); + body.push_str(&request.owner_id).map_err(encode)?; + body.push_str(&request.instance_id).map_err(encode)?; + body.push_u32(request.scope); + body.push_bool(request.probe_only); + body.push_count(request.entries.len()).map_err(encode)?; + for entry in &request.entries { + body.push_str(&entry.bucket).map_err(encode)?; + body.push_bytes(entry.bucket_incarnation.as_ref()).map_err(encode)?; + body.push_u64(entry.generation); + } + Ok(body.finish()) +} + +pub fn canonical_scoped_dirty_usage_response( + request_body: &[u8], + response: &ScannerScopedDirtyUsageAckResponse, +) -> Result, std::num::TryFromIntError> { + let mut body = CanonicalBodyBuilder::new(b"rustfs-scoped-dirty-usage-ack-response-v1\0"); + body.push_bytes(request_body)?; + body.push_u32(response.protocol_version); + body.push_str(&response.owner_id)?; + body.push_str(&response.instance_id)?; + body.push_bool(response.supported); + body.push_u32(response.max_entries); + body.push_u32(response.max_request_bytes); + body.push_u64(response.cleared); + Ok(body.finish()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::proto_gen::node_service::ScannerScopedDirtyUsageEntry; + + fn request() -> ScannerScopedDirtyUsageAckRequest { + ScannerScopedDirtyUsageAckRequest { + challenge: vec![1; 16].into(), + protocol_version: 1, + owner_id: "11111111-1111-1111-1111-111111111111".into(), + instance_id: "a".repeat(32), + scope: 1, + probe_only: false, + entries: vec![ScannerScopedDirtyUsageEntry { + bucket: "photos".into(), + bucket_incarnation: vec![2; 16].into(), + generation: 8, + }], + } + } + + #[test] + fn scoped_dirty_usage_binds_every_request_field() { + let base = request(); + let baseline = canonical_scoped_dirty_usage_request(&base).expect("valid request"); + for field in 0..9 { + let mut changed = base.clone(); + match field { + 0 => changed.challenge = vec![3; 16].into(), + 1 => changed.protocol_version += 1, + 2 => changed.owner_id = "22222222-2222-2222-2222-222222222222".into(), + 3 => changed.instance_id = "b".repeat(32), + 4 => changed.scope += 1, + 5 => changed.probe_only = true, + 6 => changed.entries[0].bucket = "videos".into(), + 7 => changed.entries[0].bucket_incarnation = vec![3; 16].into(), + _ => changed.entries[0].generation += 1, + } + assert!(canonical_scoped_dirty_usage_request(&changed).map_or(true, |body| body != baseline)); + } + } + + #[test] + fn scoped_dirty_usage_binds_capability_and_ack_to_exact_request() { + let request = canonical_scoped_dirty_usage_request(&request()).expect("valid request"); + let response = ScannerScopedDirtyUsageAckResponse { + protocol_version: 1, + owner_id: "owner".into(), + instance_id: "process".into(), + supported: true, + max_entries: 32, + max_request_bytes: 8192, + cleared: 1, + response_proof: vec![1; 32].into(), + }; + let baseline = canonical_scoped_dirty_usage_response(&request, &response).expect("valid response"); + for field in 0..7 { + let mut changed = response.clone(); + match field { + 0 => changed.protocol_version += 1, + 1 => changed.owner_id.push('x'), + 2 => changed.instance_id.push('x'), + 3 => changed.supported = false, + 4 => changed.max_entries += 1, + 5 => changed.max_request_bytes += 1, + _ => changed.cleared += 1, + } + assert_ne!( + canonical_scoped_dirty_usage_response(&request, &changed).expect("response variant"), + baseline + ); + } + assert_ne!( + canonical_scoped_dirty_usage_response(b"another request", &response).expect("request variant"), + baseline + ); + } + + #[test] + fn scoped_dirty_usage_rejects_overflow_unknown_and_duplicate_entries() { + let base = request(); + let mut invalid = base.clone(); + invalid.entries = vec![base.entries[0].clone(); SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize + 1]; + assert_eq!(validate_scoped_dirty_usage_request(&invalid), Err(ScopedDirtyUsageRequestError::TooLarge)); + invalid = base.clone(); + invalid.entries[0].bucket = "x".repeat(SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize); + assert_eq!(validate_scoped_dirty_usage_request(&invalid), Err(ScopedDirtyUsageRequestError::TooLarge)); + invalid = base.clone(); + invalid.entries.push(base.entries[0].clone()); + assert_eq!( + validate_scoped_dirty_usage_request(&invalid), + Err(ScopedDirtyUsageRequestError::InvalidEntries) + ); + invalid = base; + invalid.entries[0].bucket_incarnation = vec![0; 16].into(); + assert_eq!( + validate_scoped_dirty_usage_request(&invalid), + Err(ScopedDirtyUsageRequestError::InvalidIdentity) + ); + } +} diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index f85a2e0b6..28cf638e9 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -90,8 +90,9 @@ pub use scanner::{ }; pub use scanner_io::{ ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, - acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change, - scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation, + acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket, + record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, + scanner_maintenance_generation, }; pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER}; use std::sync::atomic::{AtomicU64, Ordering}; diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index c8353bae9..0d5e82eeb 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -883,8 +883,9 @@ pub(crate) use cache::{ }; pub use dirty_usage::{ ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, - acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change, - scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation, + acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket, + record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, + scanner_maintenance_generation, }; #[cfg(test)] pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests}; diff --git a/crates/scanner/src/scanner_io/dirty_usage.rs b/crates/scanner/src/scanner_io/dirty_usage.rs index a5978e263..18db78a81 100644 --- a/crates/scanner/src/scanner_io/dirty_usage.rs +++ b/crates/scanner/src/scanner_io/dirty_usage.rs @@ -52,6 +52,112 @@ pub enum ScannerDirtyUsageAckError { ProcessChanged, #[error("scanner dirty usage generation cannot be acknowledged")] InvalidGeneration, + #[error("scanner dirty usage bucket incarnation fence is unavailable")] + IncarnationUnavailable, +} + +/// A scoped ACK requires storage-owned lifecycle and incarnation fences. +/// Callers must only send ACKs backed by durable per-bucket publication. +pub fn acknowledge_scoped_dirty_usage( + instance_id: &str, + entries: &[(&crate::storage_api::EcstoreBucketMetadataMutationGuard, u64)], + probe_only: bool, +) -> std::result::Result { + // Lock order: sorted bucket lifecycle/metadata fences (caller), then dirty map. + // No await or storage operation occurs while the dirty map is locked. + let (cleared, pending) = { + let mut dirty = dirty_usage_buckets(); + let checked = entries + .iter() + .map(|(guard, generation)| { + guard + .checked_bucket_incarnation() + .map(|(bucket, _)| (bucket, *generation)) + .map_err(|_| ScannerDirtyUsageAckError::IncarnationUnavailable) + }) + .collect::, _>>()?; + let cleared = apply_scoped_dirty_usage_ack( + instance_id, + scanner_activity_epoch(), + DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire), + &mut dirty, + &checked, + probe_only, + )?; + if cleared > 0 { + advance_generation(&DIRTY_USAGE_BUCKET_GENERATION); + } + (cleared, dirty.len()) + }; + if !probe_only { + global_metrics().record_scanner_dirty_usage_cycle_clear(usize_to_u64_saturated(cleared), usize_to_u64_saturated(pending)); + } + Ok(usize_to_u64_saturated(cleared)) +} + +fn apply_scoped_dirty_usage_ack( + instance_id: &str, + current_instance: &str, + current_generation: u64, + dirty: &mut DirtyUsageBuckets, + entries: &[(&str, u64)], + probe_only: bool, +) -> std::result::Result { + if instance_id != current_instance { + return Err(ScannerDirtyUsageAckError::ProcessChanged); + } + if current_generation == u64::MAX + || entries + .iter() + .any(|(_, generation)| *generation == 0 || *generation == u64::MAX || *generation > current_generation) + { + return Err(ScannerDirtyUsageAckError::InvalidGeneration); + } + let mut cleared = 0; + if !probe_only { + for (bucket, generation) in entries { + if dirty.get(*bucket) == Some(generation) { + dirty.remove(*bucket); + cleared += 1; + } + } + } + Ok(cleared) +} + +#[cfg(test)] +mod scoped_dirty_usage_tests { + use super::*; + + #[test] + fn scoped_dirty_usage_preserves_uncovered_newer_and_replayed_generations() { + let mut dirty = HashMap::from([("hot".to_string(), 7), ("cold".to_string(), 8)]); + assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], true), Ok(0)); + assert_eq!(dirty.len(), 2); + assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], false), Ok(1)); + assert_eq!(dirty.get("hot"), Some(&7)); + assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], false), Ok(0)); + dirty.insert("cold".to_string(), 9); + assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 9, &mut dirty, &[("cold", 8)], false), Ok(0)); + assert_eq!(dirty.get("cold"), Some(&9)); + } + + #[test] + fn scoped_dirty_usage_rejects_restart_and_invalid_batch_before_clearing() { + let original = HashMap::from([("hot".to_string(), 7), ("cold".to_string(), 8)]); + let mut dirty = original.clone(); + assert_eq!( + apply_scoped_dirty_usage_ack("old", "new", 8, &mut dirty, &[("cold", 8)], false), + Err(ScannerDirtyUsageAckError::ProcessChanged) + ); + for generation in [0, 9, u64::MAX] { + assert_eq!( + apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8), ("hot", generation)], false), + Err(ScannerDirtyUsageAckError::InvalidGeneration) + ); + assert_eq!(dirty, original); + } + } } pub(super) fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> { diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index f81065d59..7b2cda493 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -38,8 +38,8 @@ pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::object_opts_fr #[cfg(test)] pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys as ecstore_init_bucket_metadata_sys; pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{ - get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config, - get_replication_config as ecstore_get_replication_config, + BucketMetadataMutationGuard as EcstoreBucketMetadataMutationGuard, get_lifecycle_config as ecstore_get_lifecycle_config, + get_object_lock_config as ecstore_get_object_lock_config, get_replication_config as ecstore_get_replication_config, }; pub(crate) use rustfs_ecstore::api::bucket::replication::{ ReplicateObjectInfo, ReplicationConfig as EcstoreReplicationConfig, diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index 78ce81e5b..d82d67f20 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -208,6 +208,7 @@ const EVENT_PEER_ADDR_UNAVAILABLE: &str = "peer_addr_unavailable"; const EVENT_RPC_SIGNATURE_VERIFICATION_FAILED: &str = "rpc_signature_verification_failed"; const EVENT_GRPC_TRACE_CONTEXT_PROPAGATION_FAILED: &str = "grpc_trace_context_propagation_failed"; const HEAL_CONTROL_TONIC_RPC_PATH: &str = "/node_service.HealControlService/HealControl"; +const SCANNER_SCOPED_DIRTY_USAGE_ACK_TONIC_RPC_PATH: &str = "/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"; const TIER_MUTATION_PREPARE_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/PrepareTierMutation"; const TIER_MUTATION_COMMIT_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/CommitTierMutation"; const TIER_MUTATION_ABORT_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/AbortTierMutation"; @@ -1856,6 +1857,7 @@ fn process_connection( ); let rpc_service = RpcRequestPathService::new( Routes::new(node_service) + .add_service(InterceptedService::new(storage::tonic_service::make_scanner_control_server(), check_auth)) .add_service(heal_control_service) .add_service(tier_mutation_control_service) .prepare(), @@ -2259,6 +2261,7 @@ fn check_auth(req: Request<()>) -> std::result::Result, Status> { .strip_prefix(TONIC_RPC_PREFIX) .and_then(|suffix| suffix.strip_prefix('/')) .or_else(|| (target.uri.path() == HEAL_CONTROL_TONIC_RPC_PATH).then_some("HealControl")) + .or_else(|| (target.uri.path() == SCANNER_SCOPED_DIRTY_USAGE_ACK_TONIC_RPC_PATH).then_some("ScannerScopedDirtyUsageAck")) .or_else(|| (target.uri.path() == TIER_MUTATION_PREPARE_TONIC_RPC_PATH).then_some("PrepareTierMutation")) .or_else(|| (target.uri.path() == TIER_MUTATION_COMMIT_TONIC_RPC_PATH).then_some("CommitTierMutation")) .or_else(|| (target.uri.path() == TIER_MUTATION_ABORT_TONIC_RPC_PATH).then_some("AbortTierMutation")) @@ -3427,6 +3430,50 @@ mod tests { rustfs_common::set_global_local_node_name(&previous_node_name).await; } + #[tokio::test] + #[serial_test::serial] + async fn scoped_dirty_usage_peer_probe_reaches_handler_through_production_auth() { + let _ = rustfs_credentials::set_global_rpc_secret("rpc-http-test-secret".to_string()); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind scoped ACK auth test"); + let addr = listener.local_addr().expect("listener address"); + let previous_node_name = rustfs_common::get_global_local_node_name().await; + rustfs_common::set_global_local_node_name(&addr.to_string()).await; + let node = InterceptedService::new(NodeServiceServer::new(make_server()), check_auth); + let scanner = InterceptedService::new(storage::tonic_service::make_scanner_control_server(), check_auth); + let service = RpcRequestPathService::new(Routes::new(node).add_service(scanner).prepare()); + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.expect("accept test connection"); + ConnBuilder::new(TokioExecutor::new()) + .serve_connection(TokioIo::new(socket), TowerToHyperService::new(service)) + .await + .expect("serve scoped ACK auth test"); + }); + let host = rustfs_utils::XHost::try_from(addr.to_string()).expect("peer address"); + let client = storage::PeerRestClient::new(host, format!("http://{addr}")); + let result = client + .scanner_scoped_dirty_usage_capability( + "11111111-1111-1111-1111-111111111111".to_string(), + "a".repeat(32), + vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry { + bucket: "photos".into(), + bucket_incarnation: vec![1; 16].into(), + generation: 8, + }], + ) + .await; + client.evict_connection().await; + server.abort(); + let _ = server.await; + rustfs_common::set_global_local_node_name(&previous_node_name).await; + let error = result + .expect_err("probe must fail closed without the requested storage owner") + .to_string(); + assert!( + error.contains("storage layer is not initialized") || error.contains("scoped dirty usage peer or process changed"), + "signed probe must pass production path authentication and reach owner validation: {error}" + ); + } + #[tokio::test] #[serial_test::serial] async fn peer_rest_heal_control_uses_production_auth_and_keeps_validation_errors_online() { diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 0dd18424e..111726af3 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -493,6 +493,13 @@ impl std::fmt::Debug for NodeService { } } +pub(crate) fn make_scanner_control_server() -> scanner_control_service_server::ScannerControlServiceServer { + let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize; + scanner_control_service_server::ScannerControlServiceServer::new(make_server()) + .max_decoding_message_size(limit) + .max_encoding_message_size(limit) +} + pub fn make_server() -> NodeService { let context = runtime_sources::current_app_context(); make_server_for_context(context) @@ -1087,6 +1094,74 @@ impl NodeService { } } +#[tonic::async_trait] +impl scanner_control_service_server::ScannerControlService for NodeService { + async fn scanner_scoped_dirty_usage_ack( + &self, + request: Request, + ) -> Result, Status> { + use rustfs_protos::scoped_dirty_usage::*; + static ADMISSION: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(4); + + let canonical = + canonical_scoped_dirty_usage_request(request.get_ref()).map_err(|err| Status::invalid_argument(err.to_string()))?; + verify_tonic_canonical_body_digest(&request, &canonical) + .map_err(|_| Status::permission_denied("scoped dirty usage authentication failed"))?; + let _admission = ADMISSION + .try_acquire() + .map_err(|_| Status::resource_exhausted("scoped dirty usage receiver is busy"))?; + let request = request.into_inner(); + let store = self + .resolve_object_store() + .ok_or_else(|| Status::unavailable("storage layer is not initialized"))?; + if store.id.is_nil() + || request.owner_id != store.id.to_string() + || request.instance_id != rustfs_scanner::scanner_activity_epoch() + { + return Err(Status::failed_precondition("scoped dirty usage peer or process changed")); + } + let cleared = timeout(Duration::from_secs(30), async { + // Strict bucket order is validated before admission. Acquire every + // lifecycle/metadata fence before clearing any dirty record. + let mut guards = Vec::with_capacity(request.entries.len()); + for entry in &request.entries { + let incarnation = Uuid::from_slice(entry.bucket_incarnation.as_ref()) + .map_err(|_| Status::invalid_argument("invalid bucket incarnation"))?; + let guard = + crate::storage::storage_api::acquire_scanner_bucket_incarnation_fence(&entry.bucket, incarnation, store.id) + .await + .map_err(|_| Status::failed_precondition("trusted bucket incarnation is unavailable"))?; + guards.push(guard); + } + let entries = guards + .iter() + .zip(&request.entries) + .map(|(guard, entry)| (guard, entry.generation)) + .collect::>(); + rustfs_scanner::acknowledge_scoped_dirty_usage(&request.instance_id, &entries, request.probe_only) + .map_err(|err| Status::failed_precondition(err.to_string())) + }) + .await + .map_err(|_| Status::deadline_exceeded("scoped dirty usage incarnation validation timed out"))??; + let mut response = ScannerScopedDirtyUsageAckResponse { + protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION, + owner_id: request.owner_id, + instance_id: request.instance_id, + supported: true, + max_entries: SCOPED_DIRTY_USAGE_MAX_ENTRIES, + max_request_bytes: SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES, + cleared, + response_proof: Bytes::new(), + }; + let body = canonical_scoped_dirty_usage_response(&canonical, &response) + .map_err(|_| Status::internal("scoped dirty usage response is too large"))?; + response.response_proof = sign_tonic_rpc_response_proof(&body) + .map_err(|_| Status::unavailable("scoped dirty usage response authentication is unavailable"))? + .into(); + Ok(Response::new(response)) + } +} + #[tonic::async_trait] impl Node for NodeService { async fn ping(&self, request: Request) -> Result, Status> { @@ -2623,6 +2698,7 @@ mod tests { use rustfs_kms::KmsServiceManager; use rustfs_protos::CanonicalMutationBody as _; use rustfs_protos::models::PingBodyBuilder; + use rustfs_protos::proto_gen::node_service::scanner_control_service_server::ScannerControlService as _; use rustfs_protos::proto_gen::node_service::{ BackgroundHealStatusRequest, BatchGenerallyLockRequest, CancelDecommissionRequest, CheckPartsRequest, ClearDecommissionRequest, ControlPlaneErrorCode, DeleteBucketMetadataRequest, DeleteBucketRequest, DeletePathsRequest, @@ -5990,6 +6066,74 @@ mod tests { ); } + fn scoped_dirty_usage_request() -> rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest { + rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest { + challenge: vec![7; 16].into(), + protocol_version: 1, + owner_id: "11111111-1111-1111-1111-111111111111".into(), + instance_id: "a".repeat(32), + scope: 1, + probe_only: false, + entries: vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry { + bucket: "photos".into(), + bucket_incarnation: vec![1; 16].into(), + generation: 8, + }], + } + } + + #[tokio::test] + async fn scoped_dirty_usage_authenticates_before_storage_and_rejects_tampering() { + use rustfs_protos::scoped_dirty_usage::canonical_scoped_dirty_usage_request; + let service = create_test_node_service(); + let unsigned = service + .scanner_scoped_dirty_usage_ack(Request::new(scoped_dirty_usage_request())) + .await + .expect_err("unsigned ACK must not access storage"); + assert_eq!(unsigned.code(), tonic::Code::PermissionDenied); + for field in 0..9 { + let mut signed = Request::new(scoped_dirty_usage_request()); + let canonical = canonical_scoped_dirty_usage_request(signed.get_ref()).expect("canonical request"); + set_tonic_canonical_body_digest(&mut signed, &canonical).expect("digest"); + mark_v2_authenticated(&mut signed); + match field { + 0 => signed.get_mut().challenge = vec![3; 16].into(), + 1 => signed.get_mut().owner_id = "22222222-2222-2222-2222-222222222222".into(), + 2 => signed.get_mut().instance_id = "b".repeat(32), + 3 => signed.get_mut().probe_only = true, + 4 => signed.get_mut().entries[0].bucket = "videos".into(), + 5 => signed.get_mut().entries[0].bucket_incarnation = vec![2; 16].into(), + 6 => signed.get_mut().entries[0].generation += 1, + 7 => signed.get_mut().scope += 1, + _ => signed.get_mut().protocol_version += 1, + } + let error = service + .scanner_scoped_dirty_usage_ack(signed) + .await + .expect_err("tampered ACK must fail"); + assert_eq!( + error.code(), + if field < 7 { + tonic::Code::PermissionDenied + } else { + tonic::Code::InvalidArgument + } + ); + } + let mut signed = Request::new(scoped_dirty_usage_request()); + let canonical = canonical_scoped_dirty_usage_request(signed.get_ref()).expect("canonical request"); + set_tonic_canonical_body_digest(&mut signed, &canonical).expect("digest"); + mark_v2_authenticated(&mut signed); + assert_eq!( + service + .scanner_scoped_dirty_usage_ack(signed) + .await + .expect_err("missing owner cannot advertise capability") + .code(), + tonic::Code::Unavailable + ); + } + #[tokio::test] async fn test_scanner_activity_requires_body_bound_auth_before_storage_lookup() { let service = create_test_node_service(); @@ -6485,6 +6629,62 @@ mod tests { ) } + #[tokio::test] + async fn scoped_dirty_usage_transport_rejects_oversized_unknown_and_duplicate_fields() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind scoped ACK transport test"); + let addr = listener.local_addr().expect("test listener address"); + let (shutdown, stopped) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(super::make_scanner_control_server()) + .serve_with_incoming_shutdown(TcpListenerStream::new(listener), async { + let _ = stopped.await; + }) + .await + .expect("scoped ACK transport server"); + }); + let client = reqwest::Client::builder() + .no_proxy() + .http2_prior_knowledge() + .build() + .expect("HTTP/2 client"); + let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize; + for tag in [0x78, 0x0a] { + // Unknown varint field 15, or repeated empty singular challenge: + // both decode to a tiny default struct despite the large wire body. + for oversized in [false, true] { + let mut payload = [tag, 0].repeat(if oversized { (limit - 4) / 2 } else { limit / 2 }); + if oversized { + // Unknown fixed32 field 15 makes a valid cap+1 protobuf. + payload.extend_from_slice(&[0x7d, 0, 0, 0, 0]); + } + assert_eq!(payload.len(), limit + usize::from(oversized)); + let mut frame = vec![0]; + frame.extend_from_slice(&u32::try_from(payload.len()).expect("bounded test payload").to_be_bytes()); + frame.extend_from_slice(&payload); + let response = client + .post(format!("http://{addr}/node_service.ScannerControlService/ScannerScopedDirtyUsageAck")) + .header("content-type", "application/grpc") + .header("te", "trailers") + .body(frame) + .send() + .await + .expect("send raw protobuf frame"); + let status = response.headers().get("grpc-status").expect("gRPC failure status"); + assert_eq!( + status.to_str().expect("status text"), + if oversized { "11" } else { "3" }, + "cap+1 must fail in the codec, while cap bytes reach request validation" + ); + } + } + drop(client); + shutdown.send(()).expect("stop test server"); + server.await.expect("join test server"); + } + #[tokio::test] async fn heal_control_transport_enforces_codec_limit_and_fails_closed() { let Some(mut client) = connect_test_heal_control_client().await else { diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 9e1e92dc4..2216db7f4 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -379,7 +379,7 @@ pub(crate) mod tonic_service_consumer { #[cfg(test)] pub(crate) use super::super::tonic_service::{heal_topology_fingerprint, make_heal_control_server_for_source}; pub(crate) use super::super::tonic_service::{ - make_heal_control_server_with_cache, make_server, make_tier_mutation_control_server, + make_heal_control_server_with_cache, make_scanner_control_server, make_server, make_tier_mutation_control_server, }; } @@ -1704,6 +1704,14 @@ pub(crate) async fn acquire_bucket_metadata_transaction_lock( ecstore_bucket::metadata_sys::acquire_bucket_metadata_transaction_lock(bucket).await } +pub(crate) async fn acquire_scanner_bucket_incarnation_fence( + bucket: &str, + incarnation: uuid::Uuid, + owner_id: uuid::Uuid, +) -> Result { + ecstore_bucket::metadata_sys::acquire_scanner_bucket_incarnation_fence(bucket, incarnation, owner_id).await +} + pub(crate) async fn update_bucket_targets_under_transaction_lock( guard: &ecstore_bucket::metadata_sys::BucketMetadataMutationGuard, bucket: &str, diff --git a/rustfs/src/storage/tonic_service.rs b/rustfs/src/storage/tonic_service.rs index 539361507..c7d2ee28a 100644 --- a/rustfs/src/storage/tonic_service.rs +++ b/rustfs/src/storage/tonic_service.rs @@ -13,6 +13,7 @@ // limitations under the License. pub(crate) use crate::storage::rpc::node_service::make_heal_control_server_with_cache; +pub(crate) use crate::storage::rpc::node_service::make_scanner_control_server; #[cfg(test)] pub(crate) use crate::storage::rpc::node_service::{heal::heal_topology_fingerprint, make_heal_control_server_for_source}; pub use crate::storage::rpc::{make_heal_control_server, make_server, make_tier_mutation_control_server}; diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index db459b72d..4f824f8da 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -176,7 +176,7 @@ pub(crate) mod server { heal_topology_fingerprint, make_heal_control_server_for_source, }; pub(crate) use crate::storage::storage_api::tonic_service_consumer::{ - make_heal_control_server_with_cache, make_server, make_tier_mutation_control_server, + make_heal_control_server_with_cache, make_scanner_control_server, make_server, make_tier_mutation_control_server, }; } } From 7ba5cd6888b367423357ef3e5211798c87bab3be Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 5 Sep 2026 17:06:52 +0800 Subject: [PATCH 12/12] chore(deps): refresh SDKs and verify clock skew behavior (#7174) chore(deps): refresh SDKs and pin clock skew regression coverage Refresh compatible dependencies for Scanner/Heal V2 batch 1 and verify the production S3 retry/signing path with a deterministic clock. Co-authored-by: heihutu Co-authored-by: zhi22915