fix(ecstore): make transitioned cleanup crash-safe (#6978)

* fix(ecstore): fence transitioned object cleanup

* fix(ecstore): address ILM recovery review findings

* fix(ecstore): complete crash-safe tier cleanup recovery

* test(ecstore): avoid typo false positive

* fix(ecstore): stabilize decommission error buckets

* fix(ecstore): stabilize transition delete validation

* fix(ecstore): resume authorized tier delete dispatch

* fix(ecstore): satisfy feature clippy
This commit is contained in:
cxymds
2026-09-01 19:09:22 +08:00
committed by GitHub
parent bd66fa9dca
commit 6e26769265
46 changed files with 16999 additions and 1865 deletions
+2 -2
View File
@@ -65,7 +65,7 @@ command = ['sh', '-c', 'echo RUST_MIN_STACK=33554432 >> "$NEXTEST_ENV"']
# --- default profile (local): serialize the flaky groups, never retry --------
[[profile.default.scripts]]
filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)'
filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)'
setup = 'ecstore-large-stack'
[[profile.default.scripts]]
@@ -183,7 +183,7 @@ fail-fast = false
path = "junit.xml"
[[profile.ci.scripts]]
filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)'
filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)'
setup = 'ecstore-large-stack'
[[profile.ci.scripts]]
+15
View File
@@ -75,6 +75,10 @@ pub mod bucket {
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
inspect_transition_transaction_for_operator,
};
#[cfg(feature = "test-util")]
pub use crate::bucket::lifecycle::transition_transaction::{
TransitionTransactionRecoveryStats, recover_transition_transaction_records,
};
}
pub mod evaluator {
@@ -99,6 +103,17 @@ pub mod bucket {
pub use crate::bucket::lifecycle::tier_delete_journal::{
persist_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
};
#[cfg(feature = "test-util")]
pub mod test_util {
/// Model a single-node, all-v6 fleet after its capability probe has completed.
///
/// Call this only once while constructing an isolated test store, before any
/// tier-delete journal permit or background worker can be active.
pub fn install_all_v6_fleet_capability_proof() {
crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
}
}
}
pub mod tier_last_day_stats {
@@ -41,7 +41,7 @@ use crate::bucket::lifecycle::tier_free_version_recovery::{
DEFAULT_FREE_VERSION_RECOVERY_LIMIT, FreeVersionRecoveryStats, recover_tier_free_versions_with_cancel,
};
use crate::bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent_with_manager_and_identity};
use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_with_lease_idempotent};
use crate::bucket::lifecycle::transition_transaction::run_transition_transaction_recovery_loop;
use crate::bucket::object_lock::ObjectLockApi;
use crate::bucket::versioning::VersioningApi as _;
@@ -50,7 +50,10 @@ use crate::disk::error::DiskError;
use crate::disk::{DeleteOptions, Disk, DiskAPI, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, STORAGE_FORMAT_FILE};
use crate::error::Error;
use crate::error::StorageError;
use crate::error::{is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, is_network_or_host_down};
use crate::error::{
is_err_object_not_found, is_err_read_quorum, is_err_strict_volume_not_found, is_err_version_not_found,
is_network_or_host_down,
};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use crate::object_api::{ObjectEncryptionResolver, ReadPlan};
use crate::services::tier::{
@@ -586,25 +589,217 @@ impl ExpiryOp for FreeVersionTask {
}
}
async fn delete_free_version_remote_object(
async fn acquire_free_version_tier_lease(
oi: &ObjectInfo,
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
) -> Result<(), std::io::Error> {
) -> Result<(TierOperationLease, bool), std::io::Error> {
let version_id_exact = validate_transition_remote_version(oi)?;
let identity = tier_destination_id_from_metadata(&oi.user_defined)?
.ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?;
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
let lease =
TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, &oi.transitioned_object.tier, identity)
.await
.map_err(std::io::Error::other)?;
Ok((lease, version_id_exact))
}
async fn delete_free_version_remote_object_with_lease(
oi: &ObjectInfo,
lease: &TierOperationLease,
version_id_exact: bool,
) -> Result<(), std::io::Error> {
delete_object_from_remote_tier_with_lease_idempotent(
&oi.transitioned_object.name,
&oi.transitioned_object.version_id,
&oi.transitioned_object.tier,
identity,
tier_config_mgr,
lease,
version_id_exact,
)
.await?;
Ok(())
}
fn free_version_physical_topology_generation(api: &ECStore) -> String {
let mut hasher = Sha256::new();
for pool in &api.pools {
hasher.update(pool.pool_idx.to_be_bytes());
hasher.update(pool.disk_set.len().to_be_bytes());
for set in &pool.disk_set {
hasher.update(set.set_index.to_be_bytes());
}
}
rustfs_utils::crypto::hex(hasher.finalize().as_slice())
}
fn free_version_remote_tuple_matches(candidate: &ObjectInfo, expected: &ObjectInfo) -> std::io::Result<bool> {
if candidate.transitioned_object.tier != expected.transitioned_object.tier
|| candidate.transitioned_object.name != expected.transitioned_object.name
{
return Ok(false);
}
let candidate_identity = tier_destination_id_from_metadata(&candidate.user_defined)?
.ok_or_else(|| std::io::Error::other("tier free-version is missing its backend identity"))?;
let expected_identity = tier_destination_id_from_metadata(&expected.user_defined)?
.ok_or_else(|| std::io::Error::other("tier free-version task is missing its backend identity"))?;
if candidate_identity != expected_identity {
return Ok(false);
}
if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|| expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
{
return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"tier free-version remote version state is unknown",
));
}
Ok(candidate.transition_version_state == expected.transition_version_state
&& candidate.transitioned_object.version_id == expected.transitioned_object.version_id)
}
async fn scan_exact_free_version_targets(
api: &ECStore,
oi: &ObjectInfo,
local_object: &str,
) -> std::io::Result<Vec<(Arc<SetDisks>, FileInfo)>> {
let mut targets = Vec::new();
for pool in &api.pools {
for set in &pool.disk_set {
let versions = match set.load_file_info_versions_exact(&oi.bucket, &oi.name).await {
Ok(Some(versions)) => versions,
Ok(None) => continue,
Err(err) if is_err_strict_volume_not_found(&err) => continue,
Err(err) => return Err(std::io::Error::other(err)),
};
for version in versions.versions.iter().chain(versions.free_versions.iter()) {
let candidate = ObjectInfo::from_file_info(version, &oi.bucket, &oi.name, true);
if free_version_remote_tuple_matches(&candidate, oi)? {
if candidate.transitioned_object.free_version {
// Data movement can leave the same remote tuple in
// several physical pools. Ordinary deletion assigns a
// fresh local free-version UUID to each copy, but all
// of those markers own the same idempotent remote
// DELETE. Consume them together while holding every
// physical object lock; treating their local UUIDs as
// conflicting would strand cleanup forever.
let mut actual = version.clone();
actual.name = local_object.to_string();
targets.push((Arc::clone(set), actual));
} else {
return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"a live transitioned source still references the free-version remote tuple",
));
}
}
}
}
}
Ok(targets)
}
fn free_version_cleanup_fences_current(
topology_generation: &str,
api: &ECStore,
bucket_guard: &rustfs_lock::NamespaceLockGuard,
object_guards: &[crate::store::ObjectLockDiagGuard],
lease: &TierOperationLease,
cancel: &CancellationToken,
deadline: tokio::time::Instant,
) -> bool {
!cancel.is_cancelled()
&& tokio::time::Instant::now() < deadline
&& !bucket_guard.is_lock_lost()
&& object_guards.iter().all(|guard| !guard.is_lock_lost())
&& lease.is_current_generation()
&& free_version_physical_topology_generation(api) == topology_generation
}
async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel: &CancellationToken) -> std::io::Result<bool> {
const FREE_VERSION_REMOTE_DEADLINE: StdDuration = StdDuration::from_secs(30);
let topology_generation = free_version_physical_topology_generation(&api);
let bucket_guard = api
.acquire_bucket_lifecycle_read_lock(&oi.bucket)
.await
.map_err(std::io::Error::other)?;
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
let local_object = encode_dir_object(&oi.name);
let object_guards = api
.acquire_all_physical_object_write_locks("tier_free_version_cleanup", &oi.bucket, &local_object)
.await
.map_err(std::io::Error::other)?;
let targets = scan_exact_free_version_targets(&api, oi, &local_object).await?;
if targets.is_empty() {
return Ok(false);
}
let deadline = tokio::time::Instant::now() + FREE_VERSION_REMOTE_DEADLINE;
if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"tier free-version cleanup fence is invalid before remote delete",
));
}
tokio::select! {
_ = cancel.cancelled() => {
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled"));
}
result = tokio::time::timeout_at(
deadline,
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact),
) => {
result
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out"))??;
}
}
if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
// Remote DELETE is idempotent, but a changed fence makes the local
// outcome ambiguous. Keep every marker for a fully fenced retry.
return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"tier free-version cleanup fence changed after remote delete",
));
}
let mut first_error = None;
for (set, actual) in &targets {
let mut delete_request = FileInfo {
name: local_object.clone(),
version_id: actual.version_id,
..Default::default()
};
delete_request.set_tier_free_version();
if let Err(err) = set
.delete_object_version(&oi.bucket, &local_object, &delete_request, false)
.await
&& first_error.is_none()
{
first_error = Some(std::io::Error::other(err));
}
}
let remaining = scan_exact_free_version_targets(&api, oi, &local_object).await?;
if !remaining.is_empty() {
return Err(first_error.unwrap_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"tier free-version cleanup remained on at least one physical set",
)
}));
}
if let Some(err) = first_error {
return Err(err);
}
Ok(true)
}
#[cfg(all(test, feature = "test-util"))]
async fn delete_free_version_remote_object(
oi: &ObjectInfo,
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
) -> Result<(), std::io::Error> {
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await
}
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
@@ -618,8 +813,11 @@ where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = T>,
{
delete_free_version_remote_object(oi, tier_config_mgr).await?;
Ok(delete_local().await)
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await?;
let result = delete_local().await;
drop(lease);
Ok(result)
}
struct NewerNoncurrentTask {
@@ -690,6 +888,10 @@ impl ExpiryState {
usize::try_from(self.stats.pending_tasks().max(0)).unwrap_or(usize::MAX)
}
pub fn active_tasks(&self) -> usize {
usize::try_from(self.stats.active_tasks().max(0)).unwrap_or(usize::MAX)
}
fn send_expiry_task(&self, wrkr: Sender<Option<ExpiryOpType>>, task: ExpiryOpType) -> bool {
let queued = wrkr.try_send(Some(task)).is_ok();
if queued {
@@ -826,7 +1028,7 @@ impl ExpiryState {
}
pub async fn resize_workers(n: usize, api: Arc<ECStore>) {
let expiry_state = runtime_sources::expiry_state_handle();
let expiry_state = api.ctx.expiry_state();
if n == expiry_state.read().await.tasks_tx.len() || n < 1 {
return;
}
@@ -867,7 +1069,7 @@ impl ExpiryState {
stats: Arc<ExpiryStats>,
recovery_notify: Arc<Notify>,
) {
let cancel_token = runtime_sources::background_services_cancel_token().unwrap_or_else(|| {
let cancel_token = api.ctx.background_cancel_token().unwrap_or_else(|| {
static FALLBACK: std::sync::OnceLock<tokio_util::sync::CancellationToken> = std::sync::OnceLock::new();
FALLBACK.get_or_init(tokio_util::sync::CancellationToken::new).clone()
});
@@ -968,119 +1170,33 @@ impl ExpiryState {
else if v.as_any().is::<FreeVersionTask>() {
let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("FreeVersionTask downcast failed");
let oi = v.0.clone();
if let Err(err) = delete_free_version_remote_object(&oi, &api.tier_config_mgr()).await {
recovery_notify.notify_one();
debug!(
bucket = %oi.bucket,
object = %oi.name,
remote_object = %oi.transitioned_object.name,
remote_version_id = %oi.transitioned_object.version_id,
tier = %oi.transitioned_object.tier,
error = ?err,
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
reason = "remote_tier_delete_failed",
"Lifecycle worker skipped remote tier delete"
);
continue;
}
let local_object = encode_dir_object(&oi.name);
let mut fi = FileInfo {
name: local_object.clone(),
version_id: oi.version_id,
..Default::default()
};
// This removes an existing internal cleanup marker. Keeping
// `deleted` false makes duplicate tasks return not-found
// instead of creating an ordinary delete marker.
fi.set_tier_free_version();
let mut deleted_locally = false;
for pool in &api.pools {
let set = pool.get_disks_by_key(&local_object);
let ns_lock = match set.new_ns_lock(&oi.bucket, &local_object).await {
Ok(lock) => lock,
Err(err) => {
recovery_notify.notify_one();
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
pool_index = pool.pool_idx,
set_index = set.set_index,
error = ?err,
reason = "local_free_version_lock_failed",
"Lifecycle worker failed to create local free-version cleanup lock"
);
continue;
}
};
let _object_lock_guard =
match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await {
Ok(guard) => guard,
Err(err) => {
recovery_notify.notify_one();
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
pool_index = pool.pool_idx,
set_index = set.set_index,
error = ?err,
reason = "local_free_version_lock_failed",
"Lifecycle worker failed to acquire local free-version cleanup lock"
);
continue;
}
};
match set
.delete_object_version(&oi.bucket, &local_object, &fi, false)
.await
{
Ok(()) => {
deleted_locally = true;
break;
}
Err(err) if is_err_version_not_found(&err) || is_err_object_not_found(&err) => continue,
Err(err) => {
recovery_notify.notify_one();
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
remote_object = %oi.transitioned_object.name,
remote_version_id = %oi.transitioned_object.version_id,
tier = %oi.transitioned_object.tier,
error = ?err,
reason = "local_free_version_delete_failed",
"Lifecycle worker failed local free-version cleanup"
);
break;
}
}
}
if !deleted_locally {
debug!(
match cleanup_free_version_exact(api.clone(), &oi, &cancel_token).await {
Ok(true) => {}
Ok(false) => debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
remote_object = %oi.transitioned_object.name,
remote_version_id = %oi.transitioned_object.version_id,
tier = %oi.transitioned_object.tier,
reason = "local_free_version_missing",
"Lifecycle worker could not find transitioned free version locally"
);
"Lifecycle worker found that the exact free-version was already absent"
),
Err(err) => {
recovery_notify.notify_one();
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
remote_object = %oi.transitioned_object.name,
remote_version_id = %oi.transitioned_object.version_id,
tier = %oi.transitioned_object.tier,
error = ?err,
reason = "free_version_exact_cleanup_deferred",
"Lifecycle worker retained the exact free-version for a fenced retry"
);
}
}
}
else {
@@ -1152,8 +1268,8 @@ fn set_recovered_free_version_enqueue_observer(
RecoveredFreeVersionEnqueueObserverGuard
}
pub async fn enqueue_recovered_free_version(oi: ObjectInfo) -> bool {
let expiry_state = runtime_sources::expiry_state_handle();
pub async fn enqueue_recovered_free_version(api: &ECStore, oi: ObjectInfo) -> bool {
let expiry_state = api.ctx.expiry_state();
let queued = enqueue_recovered_free_version_with_state(&expiry_state, oi).await;
#[cfg(test)]
@@ -2580,8 +2696,8 @@ fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<(
}
Some(tokio::spawn(async move {
let cancel_token = runtime_sources::background_services_cancel_token().unwrap_or_default();
let expiry_state = runtime_sources::expiry_state_handle();
let cancel_token = api.ctx.background_cancel_token().unwrap_or_default();
let expiry_state = api.ctx.expiry_state();
run_tier_free_version_recovery_loop(
cancel_token,
expiry_state,
@@ -6229,9 +6345,18 @@ mod tests {
rustfs_utils::crypto::hex(old_identity),
);
oi.user_defined = Arc::new(metadata.clone());
let lease_observed_during_local_delete = Arc::new(std::sync::atomic::AtomicBool::new(false));
delete_free_version_remote_object_then(&oi, &manager, {
let local_delete_calls = Arc::clone(&local_delete_calls);
let lease_observed_during_local_delete = Arc::clone(&lease_observed_during_local_delete);
let manager = manager.clone();
move || async move {
assert_eq!(
crate::services::tier::tier::TierConfigMgr::active_operation_lease_count(&manager, "WARM").await,
1,
"the identity-bound tier lease must span the exact local marker delete"
);
lease_observed_during_local_delete.store(true, Ordering::Relaxed);
local_delete_calls.fetch_add(1, Ordering::Relaxed);
}
})
@@ -6239,6 +6364,12 @@ mod tests {
.expect("matching destination identity should allow idempotent remote cleanup");
assert_eq!(old_backend.remove_count().await, 1);
assert_eq!(local_delete_calls.load(Ordering::Relaxed), 1);
assert!(lease_observed_during_local_delete.load(Ordering::Relaxed));
assert_eq!(
crate::services::tier::tier::TierConfigMgr::active_operation_lease_count(&manager, "WARM").await,
0,
"the tier lease should be released after the local marker delete completes"
);
let mut single_prefix_metadata = HashMap::new();
single_prefix_metadata.insert(
@@ -6472,6 +6603,7 @@ mod tests {
let state = ExpiryState::new();
let mut state = state.write().await;
let je = Jentry {
persisted_version: 0,
obj_name: "remote/object".to_string(),
version_id: "remote-version".to_string(),
tier_name: "WARM".to_string(),
@@ -6480,6 +6612,7 @@ mod tests {
version_state: rustfs_filemeta::TransitionVersionState::Exact,
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
source: None,
dispatch: None,
};
let err = state
@@ -6620,6 +6753,7 @@ mod tests {
let state = ExpiryState::new_with_unconsumed_worker_channel(1);
let mut state = state.write().await;
let je = Jentry {
persisted_version: 0,
obj_name: "remote/object".to_string(),
version_id: "remote-version".to_string(),
tier_name: "WARM".to_string(),
@@ -6628,6 +6762,7 @@ mod tests {
version_state: rustfs_filemeta::TransitionVersionState::Exact,
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
source: None,
dispatch: None,
};
state
@@ -6759,7 +6894,7 @@ mod tests {
};
assert!(
super::enqueue_recovered_free_version(oi).await,
super::enqueue_recovered_free_version(&ecstore, oi).await,
"the resized production worker queue should accept the task"
);
stop_tx.send(None).await.expect("worker stop signal should be delivered");
@@ -6875,12 +7010,12 @@ mod tests {
.await
.expect("free-version task should reach the worker");
tokio::time::timeout(StdDuration::from_secs(30), async {
while remote_backend.remove_count().await == 0 {
while stats.active_tasks() == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("worker should complete remote cleanup before taking the local lock");
.expect("worker should mark the cleanup task active before the lock assertion");
let completed_while_locked = tokio::time::timeout(StdDuration::from_millis(100), async {
while stats.active_tasks() != 0 {
tokio::task::yield_now().await;
@@ -6889,7 +7024,12 @@ mod tests {
.await;
assert!(
completed_while_locked.is_err(),
"local cleanup must wait while a competing object writer owns the namespace lock"
"the cleanup task must wait while a competing object writer owns the namespace lock"
);
assert_eq!(
remote_backend.remove_count().await,
0,
"the remote tuple must not be deleted before the all-physical namespace fence is acquired"
);
for disk_path in &disk_paths {
assert!(
@@ -6900,6 +7040,13 @@ mod tests {
}
drop(object_lock_guard);
tokio::time::timeout(StdDuration::from_secs(30), async {
while remote_backend.remove_count().await == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("worker should delete the remote tuple after acquiring the released namespace fence");
tx.send(None).await.expect("worker stop signal should be delivered");
worker.await.expect("free-version worker should stop cleanly");
@@ -6995,6 +7142,7 @@ mod tests {
.next()
.expect("seeded free version should be recoverable");
let stale_version_id = oi.version_id.expect("free version should have a concrete UUID");
let ordinary_marker_mod_time = OffsetDateTime::now_utc();
for disk_path in &disk_paths {
let metadata_path = disk_path.join(&bucket).join(object).join(STORAGE_FORMAT_FILE);
@@ -7017,7 +7165,7 @@ mod tests {
name: object.to_string(),
version_id: Some(stale_version_id),
deleted: true,
mod_time: Some(OffsetDateTime::now_utc()),
mod_time: Some(ordinary_marker_mod_time),
..Default::default()
})
.expect("same-ID ordinary marker should replace the stale free version");
@@ -7031,6 +7179,13 @@ mod tests {
.expect("same-ID ordinary marker metadata should be written");
}
assert!(
!super::cleanup_free_version_exact(Arc::clone(&ecstore), &oi, &CancellationToken::new())
.await
.expect("a stale task whose local UUID now names an ordinary marker should be an idempotent no-op"),
"the stale free-version task must not report local cleanup"
);
let state = ExpiryState::new();
let (stats, recovery_notify) = {
let state = state.read().await;
@@ -11522,7 +11677,7 @@ mod tests {
#[cfg(feature = "test-util")]
#[tokio::test]
async fn journal_replay_rejects_unknown_version_state_before_backend_io() {
async fn journal_replay_quarantines_legacy_unknown_version_state_before_backend_io() {
let (_disk_paths, ecstore) = setup_test_env().await;
let (backend, _) = register_recovery_mock_tier(&ecstore).await;
let identity = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
@@ -11530,6 +11685,7 @@ mod tests {
.expect("mock tier lease should be available")
.backend_identity();
let je = Jentry {
persisted_version: 0,
obj_name: "remote/object".to_string(),
version_id: "legacy-version".to_string(),
tier_name: "WARM".to_string(),
@@ -11538,25 +11694,28 @@ mod tests {
version_state: rustfs_filemeta::TransitionVersionState::Unknown,
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
source: None,
dispatch: None,
};
crate::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(ecstore.clone(), &je)
.await
.expect("legacy unknown journal should remain byte-compatible and persistable");
let err = crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
.await
.expect_err("unknown journal state must fail before backend IO");
.expect_err("legacy unknown journal must be quarantined before backend IO");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock);
assert_eq!(backend.remove_count().await, 0);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn journal_replay_deletes_confirmed_exact_provider_token() {
async fn rejected_upload_cleanup_retries_confirmed_exact_provider_token_without_legacy_journal() {
let (_disk_paths, ecstore) = setup_test_env().await;
let (backend, _) = register_recovery_mock_tier(&ecstore).await;
let lease = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
.await
.expect("mock tier lease should be available");
let identity = lease.backend_identity();
backend
.set_put_remote_version(Some("provider-version-token".to_string()))
.await;
@@ -11570,34 +11729,30 @@ mod tests {
.expect("confirmed remote candidate should be seeded");
backend.set_remove_failure(true);
backend.set_reject_non_empty_remote_versions(true);
let je = Jentry {
obj_name: "remote/object".to_string(),
version_id: "provider-version-token".to_string(),
tier_name: "WARM".to_string(),
backend_identity: Some(identity),
version_id_exact: true,
version_state: rustfs_filemeta::TransitionVersionState::Exact,
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
source: None,
};
crate::set_disk::cleanup_rejected_transition_upload_durably(
let err = crate::set_disk::cleanup_rejected_transition_upload_durably(
&lease,
&je.obj_name,
&je.version_id,
"remote/object",
"provider-version-token",
true,
Some(ecstore.clone()),
)
.await
.expect("failed immediate cleanup should remain durable in the journal");
assert!(backend.contains(&je.obj_name).await);
.expect_err("a failed immediate cleanup must remain owned by the caller's transition transaction");
assert_eq!(err.kind(), std::io::ErrorKind::Other);
assert!(backend.contains("remote/object").await);
backend.set_remove_failure(false);
crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
.await
.expect("identity-bound exact journal must retry confirmed candidate cleanup");
crate::set_disk::cleanup_rejected_transition_upload_durably(
&lease,
"remote/object",
"provider-version-token",
true,
Some(ecstore),
)
.await
.expect("the transaction retry must delete the same confirmed candidate");
assert!(!backend.contains(&je.obj_name).await);
assert!(!backend.contains("remote/object").await);
assert_eq!(backend.exact_remove_count(), 2);
assert_eq!(
backend.remove_versions().await,
@@ -11760,11 +11915,14 @@ mod tests {
};
let mut recovery_rx = recovery_rx.lock().await;
assert!(
super::enqueue_recovered_free_version(ObjectInfo {
bucket: "prefill".to_string(),
name: "prefill".to_string(),
..Default::default()
})
super::enqueue_recovered_free_version(
&ecstore,
ObjectInfo {
bucket: "prefill".to_string(),
name: "prefill".to_string(),
..Default::default()
},
)
.await,
"the production recovery queue should accept its first task"
);
@@ -33,6 +33,7 @@ const MANUAL_TRANSITION_CURSOR_MARKER_PROOF_MAX_SIZE: usize = 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum DurableIlmRecordKind {
TierDeleteJournal,
TierDeleteDispatchManifest,
TransitionTransaction,
ManualTransitionJob,
ManualTransitionScope,
@@ -54,6 +55,18 @@ pub(crate) const TIER_DELETE_JOURNAL_NAMESPACE: DurableIlmNamespace = DurableIlm
max_record_size: 64 * 1024,
kind: DurableIlmRecordKind::TierDeleteJournal,
};
pub(crate) const TIER_DELETE_JOURNAL_V6_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
name: "tier-delete-journal-v6",
prefix: "ilm/tier-delete-journal-v6/",
max_record_size: 64 * 1024,
kind: DurableIlmRecordKind::TierDeleteJournal,
};
pub(crate) const TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
name: "tier-delete-dispatch-manifest",
prefix: tier_delete_journal::TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
max_record_size: tier_delete_journal::MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE,
kind: DurableIlmRecordKind::TierDeleteDispatchManifest,
};
pub(crate) const TRANSITION_TRANSACTION_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
name: "transition-transaction",
prefix: "ilm/transition-transactions/records",
@@ -85,8 +98,10 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace
kind: DurableIlmRecordKind::ManualTransitionWorkerResult,
};
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 6] = [
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 8] = [
TIER_DELETE_JOURNAL_NAMESPACE,
TIER_DELETE_JOURNAL_V6_NAMESPACE,
TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
TRANSITION_TRANSACTION_NAMESPACE,
MANUAL_TRANSITION_JOB_NAMESPACE,
MANUAL_TRANSITION_SCOPE_NAMESPACE,
@@ -157,6 +172,15 @@ pub(crate) enum DurableIlmRecordCheckpoint {
content_sha256: String,
identity_sha256: String,
committed: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
dispatch_identity_sha256: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
state: Option<super::tier_sweeper::TierDeleteJournalState>,
},
TierDeleteDispatchManifest {
content_sha256: String,
identity_sha256: String,
state: tier_delete_journal::TierDeleteDispatchManifestState,
},
TransitionTransaction {
content_sha256: String,
@@ -195,6 +219,7 @@ impl DurableIlmRecordCheckpoint {
pub(crate) fn content_sha256(&self) -> &str {
match self {
Self::TierDeleteJournal { content_sha256, .. }
| Self::TierDeleteDispatchManifest { content_sha256, .. }
| Self::TransitionTransaction { content_sha256, .. }
| Self::ManualTransitionJob { content_sha256, .. }
| Self::ManualTransitionScope { content_sha256, .. }
@@ -228,6 +253,19 @@ impl DurableIlmRecordCheckpoint {
}
pub(crate) fn validate_successor(&self, next: &Self) -> Result<()> {
for checkpoint in [self, next] {
if let Self::TierDeleteJournal {
committed,
dispatch_identity_sha256,
state,
..
} = checkpoint
&& (state.is_some() != dispatch_identity_sha256.is_some()
|| state.is_some_and(|state| *committed != (state == super::tier_sweeper::TierDeleteJournalState::Committed)))
{
return Err(Error::other("durable ILM tier delete journal checkpoint is invalid"));
}
}
if self == next {
if let Self::ManualTransitionJob {
progress,
@@ -244,18 +282,64 @@ impl DurableIlmRecordCheckpoint {
let valid = match (self, next) {
(
Self::TierDeleteJournal {
content_sha256: previous_content,
identity_sha256: previous_identity,
committed: previous_committed,
dispatch_identity_sha256: previous_dispatch_identity,
state: previous_state,
..
},
Self::TierDeleteJournal {
content_sha256: next_content,
identity_sha256: next_identity,
committed: next_committed,
dispatch_identity_sha256: next_dispatch_identity,
state: next_state,
..
},
) => {
use super::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared};
let dispatch_identity_is_monotonic = match (previous_dispatch_identity, next_dispatch_identity) {
(Some(previous), Some(next)) => previous == next,
(None, None) => true,
// Old receipts did not record the v6 dispatch binding. A
// byte-identical observation may adopt the stronger proof,
// but an in-flight mutation must fail closed instead of
// guessing which operation owned the journal.
(None, Some(_)) => previous_content == next_content,
(Some(_), None) => false,
};
let state_is_monotonic = match (previous_state, next_state) {
(Some(previous), Some(next)) => {
previous == next || matches!((previous, next), (Prepared, Dispatched) | (Dispatched, Committed))
}
(None, None) => previous_committed == next_committed || (!previous_committed && *next_committed),
(None, Some(_)) => previous_content == next_content,
(Some(_), None) => false,
};
previous_identity == next_identity && dispatch_identity_is_monotonic && state_is_monotonic
}
(
Self::TierDeleteDispatchManifest {
identity_sha256: previous_identity,
state: previous_state,
..
},
Self::TierDeleteDispatchManifest {
identity_sha256: next_identity,
state: next_state,
..
},
) => {
use tier_delete_journal::TierDeleteDispatchManifestState::{
Aborted, Aborting, Completed, DispatchAuthorized, Preparing,
};
previous_identity == next_identity
&& (previous_committed == next_committed || (!previous_committed && *next_committed))
&& matches!(
(previous_state, next_state),
(Preparing, DispatchAuthorized | Aborting) | (Aborting, Aborted) | (DispatchAuthorized, Completed)
)
}
(
Self::TransitionTransaction {
@@ -351,6 +435,49 @@ impl DurableIlmRecordCheckpoint {
Err(Error::other("durable ILM record generation is not a monotonic successor"))
}
}
/// Whether `self` is an older generation of the same immutable record
/// that can reach `terminal` through one or more valid state transitions.
/// This is deliberately broader than `validate_successor`, which remains
/// adjacent-only for receipt advancement. Terminal cleanup uses this only
/// after the exact terminal ETag and terminal receipt were committed, to
/// purge older object versions exposed by that deletion.
pub(crate) fn is_predecessor_of_terminal(&self, terminal: &Self) -> bool {
if self == terminal || self.validate_successor(terminal).is_ok() {
return true;
}
match (self, terminal) {
(
Self::TierDeleteJournal {
identity_sha256: previous_identity,
dispatch_identity_sha256: previous_dispatch,
state: Some(super::tier_sweeper::TierDeleteJournalState::Prepared),
..
},
Self::TierDeleteJournal {
identity_sha256: terminal_identity,
dispatch_identity_sha256: terminal_dispatch,
state: Some(super::tier_sweeper::TierDeleteJournalState::Committed),
..
},
) => previous_identity == terminal_identity && previous_dispatch == terminal_dispatch,
(
Self::TierDeleteDispatchManifest {
identity_sha256: previous_identity,
state: tier_delete_journal::TierDeleteDispatchManifestState::Preparing,
..
},
Self::TierDeleteDispatchManifest {
identity_sha256: terminal_identity,
state:
tier_delete_journal::TierDeleteDispatchManifestState::Aborted
| tier_delete_journal::TierDeleteDispatchManifestState::Completed,
..
},
) => previous_identity == terminal_identity,
_ => false,
}
}
}
fn transition_state_distance(
@@ -750,10 +877,19 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
if tier_delete_journal::tier_delete_journal_object_name(&entry) != path {
return Err(Error::other("tier delete journal content does not match its path"));
}
let operation_id = path
let legacy_operation_id = path
.strip_prefix(namespace.prefix)
.and_then(|suffix| suffix.strip_suffix(".json"))
.ok_or_else(|| Error::other("tier delete journal path is invalid"))?;
// Legacy v1-v5 paths already expose a 64-hex operation id and
// must remain receipt-compatible. V6 uses an operation-scoped
// nested path, so derive a fixed, path-unique receipt id instead
// of embedding slashes in the receipt locator.
let operation_id = if entry.persisted_version == 6 {
hex_sha256(path.as_bytes(), ToOwned::to_owned)
} else {
legacy_operation_id.to_string()
};
let identity_sha256 = checkpoint_hash(&(
&entry.obj_name,
&entry.version_id,
@@ -763,13 +899,29 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
entry.version_state,
&entry.source,
))?;
let dispatch_identity_sha256 = entry.dispatch.as_ref().map(checkpoint_hash).transpose()?;
(
"operation_id",
operation_id.to_string(),
operation_id,
DurableIlmRecordCheckpoint::TierDeleteJournal {
content_sha256,
identity_sha256,
committed: entry.state == super::tier_sweeper::TierDeleteJournalState::Committed,
dispatch_identity_sha256,
state: (entry.persisted_version == 6).then_some(entry.state),
},
)
}
DurableIlmRecordKind::TierDeleteDispatchManifest => {
let (operation_id, identity_sha256, state) =
tier_delete_journal::validate_tier_delete_dispatch_manifest_record(path, data)?;
(
"operation_id",
hex_sha256(operation_id.as_bytes(), ToOwned::to_owned),
DurableIlmRecordCheckpoint::TierDeleteDispatchManifest {
content_sha256,
identity_sha256,
state,
},
)
}
@@ -956,6 +1108,87 @@ mod tests {
}
}
#[test]
fn tier_delete_dispatch_manifest_namespace_validates_monotonic_branches() {
use tier_delete_journal::TierDeleteDispatchManifestState::{Aborted, Aborting, Completed, DispatchAuthorized, Preparing};
let operation_id = Uuid::new_v4();
let checkpoint = |state| {
let (path, data) = tier_delete_journal::test_tier_delete_dispatch_manifest_record(operation_id, state);
let namespace = classify_durable_ilm_record(&path)
.expect("dispatch manifest namespace should classify")
.expect("dispatch manifest should be durable");
assert_eq!(namespace, &TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE);
validate_durable_ilm_record(&path, &data)
.expect("dispatch manifest should validate")
.checkpoint
};
let preparing = checkpoint(Preparing);
let authorized = checkpoint(DispatchAuthorized);
let completed = checkpoint(Completed);
let aborting = checkpoint(Aborting);
let aborted = checkpoint(Aborted);
preparing
.validate_successor(&authorized)
.expect("Preparing may become DispatchAuthorized");
authorized
.validate_successor(&completed)
.expect("DispatchAuthorized may become Completed");
preparing.validate_successor(&aborting).expect("Preparing may enter rollback");
aborting.validate_successor(&aborted).expect("Aborting may become Aborted");
assert!(authorized.validate_successor(&aborting).is_err());
assert!(completed.validate_successor(&authorized).is_err());
assert!(aborted.validate_successor(&preparing).is_err());
}
#[test]
fn tier_delete_journal_checkpoint_binds_dispatch_and_full_state_monotonically() {
use crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared};
let checkpoint = |content: &str, dispatch: Option<&str>, state| DurableIlmRecordCheckpoint::TierDeleteJournal {
content_sha256: content.repeat(64),
identity_sha256: "i".repeat(64),
committed: state == Some(Committed),
dispatch_identity_sha256: dispatch.map(|value| value.repeat(64)),
state,
};
let prepared = checkpoint("a", Some("d"), Some(Prepared));
let dispatched = checkpoint("b", Some("d"), Some(Dispatched));
let committed = checkpoint("c", Some("d"), Some(Committed));
prepared
.validate_successor(&dispatched)
.expect("Prepared may advance to Dispatched");
dispatched
.validate_successor(&committed)
.expect("Dispatched may advance to Committed");
assert!(prepared.validate_successor(&committed).is_err());
assert!(dispatched.validate_successor(&prepared).is_err());
let rebound = checkpoint("b", Some("e"), Some(Dispatched));
assert!(dispatched.validate_successor(&rebound).is_err());
let legacy: DurableIlmRecordCheckpoint = serde_json::from_value(serde_json::json!({
"kind": "tier_delete_journal",
"content_sha256": "a".repeat(64),
"identity_sha256": "i".repeat(64),
"committed": false
}))
.expect("legacy tier-delete checkpoint should remain decodable");
legacy
.validate_successor(&prepared)
.expect("byte-identical legacy receipt may adopt the stronger v6 proof");
let changed_legacy = DurableIlmRecordCheckpoint::TierDeleteJournal {
content_sha256: "z".repeat(64),
identity_sha256: "i".repeat(64),
committed: false,
dispatch_identity_sha256: None,
state: None,
};
assert!(changed_legacy.validate_successor(&prepared).is_err());
}
#[test]
fn manual_transition_job_checkpoint_compacts_legacy_progress_compatibly() {
let options = super::super::bucket_lifecycle_ops::ManualTransitionRunOptions::default();
+2 -2
View File
@@ -34,6 +34,6 @@ pub mod tier_sweeper;
pub mod transition_transaction;
pub(crate) use durable_namespace::{
DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record,
validate_durable_ilm_record,
DurableIlmRecordCheckpoint, ILM_META_PREFIX, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE, ValidatedDurableIlmRecord,
classify_durable_ilm_record, validate_durable_ilm_record,
};
File diff suppressed because it is too large Load Diff
@@ -172,7 +172,8 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
return Err(std::io::Error::other("free-version recovery limit must be greater than zero").into());
}
let page = list_tier_free_versions(api, limit, bucket_marker.clone(), object_marker.clone(), cancel_token.clone()).await?;
let page =
list_tier_free_versions(api.clone(), limit, bucket_marker.clone(), object_marker.clone(), cancel_token.clone()).await?;
let mut stats = FreeVersionRecoveryStats {
scanned: 0,
enqueued: 0,
@@ -190,7 +191,7 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
return Err(tier_free_version_recovery_cancelled());
}
retry_cursor.visit(&oi);
if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(oi).await) {
if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(&api, oi).await) {
let (bucket_marker, object_marker) = retry_cursor.retry_markers();
stats.truncated = true;
stats.next_bucket_marker = bucket_marker;
@@ -255,6 +255,7 @@ impl ObjSweeper {
}
if del_tier {
return Some(Jentry {
persisted_version: 0,
obj_name: self.remote_object.clone(),
version_id: self.transition_version_id.clone(),
tier_name: self.transition_tier.clone(),
@@ -266,6 +267,7 @@ impl ObjSweeper {
version_state: self.transition_version_state,
state: TierDeleteJournalState::Committed,
source: None,
dispatch: None,
});
}
None
@@ -298,9 +300,19 @@ impl ObjSweeper {
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
pub(crate) enum TierDeleteJournalState {
Prepared,
Dispatched,
Committed,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub(crate) struct TierDeleteDispatchBinding {
pub(crate) operation_id: Uuid,
pub(crate) manifest_object: String,
pub(crate) journal_set_sha256: String,
pub(crate) topology_generation: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub(crate) struct TierDeleteSourceIdentity {
@@ -342,6 +354,10 @@ impl TierDeleteSourceIdentity {
#[derive(Debug, Clone)]
#[allow(unused_assignments)]
pub struct Jentry {
/// On-disk format version when decoded. Newly constructed entries use 0;
/// the encoder chooses their format from the durable ownership fields.
/// Recovery uses this value to quarantine v1-v5 without rewriting them.
pub(crate) persisted_version: u8,
pub(crate) obj_name: String,
pub(crate) version_id: String,
pub(crate) tier_name: String,
@@ -350,6 +366,23 @@ pub struct Jentry {
pub(crate) version_state: rustfs_filemeta::TransitionVersionState,
pub(crate) state: TierDeleteJournalState,
pub(crate) source: Option<TierDeleteSourceIdentity>,
pub(crate) dispatch: Option<TierDeleteDispatchBinding>,
}
impl Jentry {
/// Whether this prepared transaction is eligible to become the sole
/// cleanup owner for its transitioned source. The caller may use this to
/// decide whether to persist it, but must not set `skip_free_version`
/// until persistence succeeds.
pub(crate) fn can_replace_tier_free_version(&self) -> bool {
self.state == TierDeleteJournalState::Prepared
&& self.backend_identity.is_some()
&& self.version_state != rustfs_filemeta::TransitionVersionState::Unknown
&& self
.source
.as_ref()
.is_some_and(TierDeleteSourceIdentity::has_stable_identity)
}
}
impl ExpiryOp for Jentry {
@@ -617,6 +650,7 @@ pub fn transitioned_force_delete_journal_entry(
}
Some(Jentry {
persisted_version: 0,
obj_name: transitioned.name.clone(),
version_id: transitioned.version_id.clone(),
tier_name: transitioned.tier.clone(),
@@ -628,6 +662,7 @@ pub fn transitioned_force_delete_journal_entry(
version_state: transition_version_state,
state: TierDeleteJournalState::Committed,
source: None,
dispatch: None,
})
}
@@ -673,17 +708,73 @@ mod test {
use rustfs_s3_client::signer_error::invalid_utf8_header_error;
use super::{
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED,
RemoteDeleteBreaker, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, Jentry,
RemoteDeleteBreaker, RemoteTierDeleteOutcome, TierDeleteJournalState, TierDeleteSourceIdentity,
delete_confirmed_transition_candidate_exact_with_manager_and_identity, delete_object_from_remote_tier_idempotent,
delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error,
is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, should_record_remote_delete_failure,
transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
};
use crate::storage_api_contracts::lifecycle::TransitionedObject;
use rustfs_filemeta::TransitionVersionState;
use std::io::{Error, ErrorKind};
use std::time::{Duration, Instant};
fn stable_prepared_journal() -> Jentry {
Jentry {
persisted_version: 0,
obj_name: "remote/object".to_string(),
version_id: "remote-version".to_string(),
tier_name: "WARM".to_string(),
backend_identity: Some([7; 32]),
version_id_exact: true,
version_state: TransitionVersionState::Exact,
state: TierDeleteJournalState::Prepared,
source: Some(TierDeleteSourceIdentity {
bucket: "bucket".to_string(),
object: "object".to_string(),
version_id: Some(uuid::Uuid::new_v4().to_string()),
versioned: true,
version_suspended: false,
data_dir: None,
etag: None,
mod_time: None,
}),
dispatch: None,
}
}
#[test]
fn only_stable_prepared_journal_can_replace_tier_free_version() {
let stable = stable_prepared_journal();
assert!(stable.can_replace_tier_free_version());
let mut committed = stable.clone();
committed.state = TierDeleteJournalState::Committed;
assert!(!committed.can_replace_tier_free_version());
let mut unbound = stable.clone();
unbound.backend_identity = None;
assert!(!unbound.can_replace_tier_free_version());
let mut unknown = stable.clone();
unknown.version_state = TransitionVersionState::Unknown;
assert!(!unknown.can_replace_tier_free_version());
let mut unstable = stable;
unstable.source = Some(TierDeleteSourceIdentity {
bucket: "bucket".to_string(),
object: "object".to_string(),
version_id: None,
versioned: false,
version_suspended: false,
data_dir: None,
etag: Some("etag-only".to_string()),
mod_time: None,
});
assert!(!unstable.can_replace_tier_free_version());
}
#[test]
fn signer_header_error_detection_matches_utf8_failures() {
let err = Error::new(
@@ -49,8 +49,8 @@ use rustfs_protos::proto_gen::node_service::{
ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
ScannerPublicationLeaseResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
tier_mutation_control_service_client::TierMutationControlServiceClient,
TierMutationControlResponse, TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest,
node_service_client::NodeServiceClient, tier_mutation_control_service_client::TierMutationControlServiceClient,
};
pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
use rustfs_protos::{TierMutationRpcPhase, evict_failed_connection};
@@ -462,6 +462,31 @@ pub struct PeerTierMutationOutcome {
pub applied: bool,
}
#[derive(Debug, thiserror::Error)]
#[error("{message}")]
struct TierMutationDefinitelyRejected {
message: String,
}
fn tier_mutation_definitely_rejected_error(message: String) -> Error {
Error::other(TierMutationDefinitelyRejected { message })
}
#[cfg(test)]
pub(crate) fn test_tier_mutation_definitely_rejected_error(message: &str) -> Error {
tier_mutation_definitely_rejected_error(message.to_string())
}
pub(crate) fn tier_mutation_error_is_definitely_rejected(error: &Error) -> bool {
matches!(
error,
Error::Io(io_error)
if io_error
.get_ref()
.is_some_and(|source| source.downcast_ref::<TierMutationDefinitelyRejected>().is_some())
)
}
fn validate_tier_mutation_response_proof(
version: u32,
phase: TierMutationRpcPhase,
@@ -469,6 +494,16 @@ fn validate_tier_mutation_response_proof(
canonical_payload: &[u8],
response: &TierMutationControlResponse,
) -> Result<()> {
if response.response_proof.len() > rustfs_protos::TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE {
return Err(Error::other("peer tier mutation response proof exceeds size limit"));
}
if response
.error_info
.as_ref()
.is_some_and(|error| error.len() > rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE)
{
return Err(Error::other("peer tier mutation error response exceeds size limit"));
}
let canonical_response =
rustfs_protos::canonical_tier_mutation_rpc_response_body(rustfs_protos::TierMutationRpcResponseProofInput {
version,
@@ -479,6 +514,7 @@ fn validate_tier_mutation_response_proof(
state: response.state,
applied: response.applied,
error_info: response.error_info.as_deref(),
failure_class: response.failure_class,
})
.map_err(|_| Error::other("tier mutation response length cannot be represented"))?;
verify_tonic_rpc_response_proof(&canonical_response, &response.response_proof)
@@ -500,9 +536,9 @@ fn validate_tier_mutation_payload_len(phase: TierMutationRpcPhase, payload_len:
TierMutationRpcPhase::Commit => rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE,
TierMutationRpcPhase::Abort => {
if payload_len == 0 {
return Ok(());
return Err(Error::other("tier mutation abort payload is empty"));
}
return Err(Error::other("tier mutation abort payload must be empty"));
rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE
}
_ => return Err(Error::other("tier mutation rpc phase is unsupported")),
};
@@ -521,8 +557,29 @@ fn tier_mutation_phase_label(phase: TierMutationRpcPhase) -> &'static str {
}
}
fn tier_mutation_control_status_error(phase: TierMutationRpcPhase, status: tonic::Status) -> Error {
Error::other(format!("peer tier mutation {} RPC failed: {status}", tier_mutation_phase_label(phase)))
fn tier_mutation_control_status_error(phase: TierMutationRpcPhase, requested_version: u32, status: tonic::Status) -> Error {
let message = format!("peer tier mutation {} RPC failed: {status}", tier_mutation_phase_label(phase));
let legacy_rejection = format!("unsupported tier mutation peer protocol version: {requested_version}");
// RUSTFS_COMPAT_TODO(backlog-2097-tier-mutation-v4-error-text): retain this exact v3-server rejection classifier for mixed-version peers. Remove after every supported peer returns the signed v4 failure class.
if requested_version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
&& status.code() == tonic::Code::FailedPrecondition
&& status.message().as_bytes() == legacy_rejection.as_bytes()
{
return tier_mutation_definitely_rejected_error(message);
}
Error::other(message)
}
fn tier_mutation_failed_response_error(version: u32, failure_class: i32, error_info: Option<String>) -> Error {
let message = error_info.unwrap_or_else(|| "peer tier mutation failed without an error".to_string());
if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
&& TierMutationFailureClass::try_from(failure_class).ok() == Some(TierMutationFailureClass::PreDispatchRejected)
{
return tier_mutation_definitely_rejected_error(message);
}
// Missing/zero, unknown, and explicit Ambiguous are deliberately the same
// fail-closed result: the coordinator must include this peer in Abort.
Error::other(message)
}
impl PeerRestClient {
@@ -1315,8 +1372,12 @@ impl PeerRestClient {
.await
}
pub async fn abort_tier_mutation(&self, mutation_id: Uuid) -> Result<PeerTierMutationOutcome> {
self.tier_mutation_control(TierMutationRpcPhase::Abort, mutation_id, Bytes::new())
pub async fn abort_tier_mutation(
&self,
mutation_id: Uuid,
canonical_prepare_payload: Bytes,
) -> Result<PeerTierMutationOutcome> {
self.tier_mutation_control(TierMutationRpcPhase::Abort, mutation_id, canonical_prepare_payload)
.await
}
@@ -1350,7 +1411,7 @@ impl PeerRestClient {
client
.prepare_tier_mutation(request)
.await
.map_err(|status| tier_mutation_control_status_error(phase, status))?
.map_err(|status| tier_mutation_control_status_error(phase, version, status))?
.into_inner()
}
TierMutationRpcPhase::Commit => {
@@ -1363,7 +1424,7 @@ impl PeerRestClient {
client
.commit_tier_mutation(request)
.await
.map_err(|status| tier_mutation_control_status_error(phase, status))?
.map_err(|status| tier_mutation_control_status_error(phase, version, status))?
.into_inner()
}
TierMutationRpcPhase::Abort => {
@@ -1376,18 +1437,19 @@ impl PeerRestClient {
client
.abort_tier_mutation(request)
.await
.map_err(|status| tier_mutation_control_status_error(phase, status))?
.map_err(|status| tier_mutation_control_status_error(phase, version, status))?
.into_inner()
}
_ => return Err(Error::other("tier mutation rpc phase is unsupported")),
};
validate_tier_mutation_response_proof(version, phase, mutation_id, &canonical_payload, &response)?;
if !response.success {
return Err(Error::other(
response
.error_info
.unwrap_or_else(|| "peer tier mutation failed without an error".to_string()),
));
return Err(tier_mutation_failed_response_error(version, response.failure_class, response.error_info));
}
if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
&& response.failure_class != TierMutationFailureClass::Unspecified as i32
{
return Err(Error::other("successful peer tier mutation response carried a failure class"));
}
let state = decode_tier_mutation_peer_state(response.state)?;
Ok(PeerTierMutationOutcome {
@@ -3332,6 +3394,7 @@ mod tests {
state: i32,
applied: bool,
error_info: Option<&'a str>,
failure_class: i32,
}
fn signed_tier_mutation_response(input: TierMutationResponseFixture<'_>) -> TierMutationControlResponse {
@@ -3345,6 +3408,7 @@ mod tests {
state: input.state,
applied: input.applied,
error_info: input.error_info,
failure_class: input.failure_class,
})
.expect("small tier mutation response should encode");
let response_proof =
@@ -3355,6 +3419,7 @@ mod tests {
applied: input.applied,
error_info: input.error_info.map(str::to_string),
response_proof: response_proof.into(),
failure_class: input.failure_class,
}
}
@@ -3372,6 +3437,7 @@ mod tests {
state: TierMutationPeerState::Prepared as i32,
applied: true,
error_info: None,
failure_class: TierMutationFailureClass::Unspecified as i32,
});
validate_tier_mutation_response_proof(
rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
@@ -3396,6 +3462,10 @@ mod tests {
applied: false,
..response.clone()
},
TierMutationControlResponse {
failure_class: TierMutationFailureClass::Ambiguous as i32,
..response.clone()
},
] {
let err = validate_tier_mutation_response_proof(
rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
@@ -3419,6 +3489,44 @@ mod tests {
assert!(err.to_string().contains("invalid tier mutation response proof"));
}
#[test]
fn tier_mutation_response_rejects_oversized_proof_and_error_before_verification() {
let mutation_id = Uuid::new_v4();
let payload = b"tier-mutation-prepare";
let oversized_proof = TierMutationControlResponse {
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: None,
response_proof: vec![0; rustfs_protos::TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE + 1].into(),
failure_class: TierMutationFailureClass::Ambiguous as i32,
};
let err = validate_tier_mutation_response_proof(
rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
TierMutationRpcPhase::Prepare,
mutation_id,
payload,
&oversized_proof,
)
.expect_err("oversized proof must fail before cryptographic verification");
assert!(err.to_string().contains("response proof exceeds size limit"));
let oversized_error = TierMutationControlResponse {
response_proof: Bytes::new(),
error_info: Some("e".repeat(rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE + 1)),
..oversized_proof
};
let err = validate_tier_mutation_response_proof(
rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
TierMutationRpcPhase::Prepare,
mutation_id,
payload,
&oversized_error,
)
.expect_err("oversized error detail must fail before proof construction");
assert!(err.to_string().contains("error response exceeds size limit"));
}
#[test]
fn tier_mutation_peer_state_decode_fails_closed() {
assert_eq!(
@@ -3463,8 +3571,17 @@ mod tests {
)
.is_err()
);
validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 0).expect("empty abort payload should fit");
assert!(validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 1).is_err());
assert!(validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 0).is_err());
validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 1).expect("non-empty abort payload should fit");
validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE)
.expect("max abort payload should fit");
assert!(
validate_tier_mutation_payload_len(
TierMutationRpcPhase::Abort,
rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE + 1,
)
.is_err()
);
}
#[test]
@@ -3479,7 +3596,7 @@ mod tests {
tonic::Status::deadline_exceeded("peer tier mutation control timed out"),
tonic::Status::unavailable("peer tier mutation control unavailable"),
] {
let err = tier_mutation_control_status_error(phase, status);
let err = tier_mutation_control_status_error(phase, rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION, status);
let rendered = err.to_string();
assert!(rendered.contains(&format!("peer tier mutation {label} RPC failed")), "{rendered}");
assert!(
@@ -3493,6 +3610,61 @@ mod tests {
}
}
#[test]
fn tier_mutation_v4_to_v3_rejection_classification_requires_exact_status_and_message() {
let version = rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION;
let exact = format!("unsupported tier mutation peer protocol version: {version}");
let rejected = tier_mutation_control_status_error(
TierMutationRpcPhase::Prepare,
version,
tonic::Status::failed_precondition(exact.clone()),
);
assert!(tier_mutation_error_is_definitely_rejected(&rejected));
for status in [
tonic::Status::failed_precondition(format!("{exact}.")),
tonic::Status::failed_precondition(format!("unsupported tier mutation peer protocol version: {}", version - 1)),
tonic::Status::invalid_argument(exact.clone()),
tonic::Status::unimplemented(exact),
] {
let ambiguous = tier_mutation_control_status_error(TierMutationRpcPhase::Prepare, version, status);
assert!(
!tier_mutation_error_is_definitely_rejected(&ambiguous),
"near-text, wrong-code, and Unimplemented failures must remain ambiguous"
);
}
}
#[test]
fn tier_mutation_v4_failure_class_is_typed_and_fails_closed() {
let version = rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION;
let rejected = tier_mutation_failed_response_error(
version,
TierMutationFailureClass::PreDispatchRejected as i32,
Some("rejected".to_string()),
);
assert!(tier_mutation_error_is_definitely_rejected(&rejected));
for failure_class in [
TierMutationFailureClass::Unspecified as i32,
TierMutationFailureClass::Ambiguous as i32,
99,
] {
let ambiguous = tier_mutation_failed_response_error(version, failure_class, None);
assert!(
!tier_mutation_error_is_definitely_rejected(&ambiguous),
"missing, unknown, and explicit ambiguous classes must trigger Abort fanout"
);
}
let v3_ignores_v4_class = tier_mutation_failed_response_error(
rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
TierMutationFailureClass::PreDispatchRejected as i32,
Some("legacy failure".to_string()),
);
assert!(!tier_mutation_error_is_definitely_rejected(&v3_ignores_v4_class));
}
#[tokio::test]
async fn peer_rest_client_rejects_oversized_tier_prepare_before_dialing() {
let client = test_peer_client();
+589 -19
View File
@@ -18,7 +18,8 @@ use crate::bucket::utils::is_meta_bucketname;
use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::bucket::{
lifecycle::{
DurableIlmRecordCheckpoint, ILM_META_PREFIX, LifecycleExpiryConfigs, ValidatedDurableIlmRecord,
DurableIlmRecordCheckpoint, ILM_META_PREFIX, LifecycleExpiryConfigs, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
ValidatedDurableIlmRecord,
bucket_lifecycle_audit::LcEventSrc,
bucket_lifecycle_ops::{
LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_for_data_movement, apply_expiry_rule_in,
@@ -26,6 +27,7 @@ use crate::bucket::{
},
classify_durable_ilm_record, get_expiry_configs,
lifecycle::IlmAction,
tier_delete_journal::durable_ilm_v6_topology_generation,
validate_durable_ilm_record,
},
metadata_sys,
@@ -49,6 +51,9 @@ use crate::error::{
use crate::layout::endpoints::EndpointServerPools;
use crate::object_api::{DecommissionCapacityOptions, GetObjectReader, ObjectOptions};
use crate::runtime::sources as runtime_sources;
use crate::services::notification_sys::{
acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches, tier_delete_journal_topology_generation,
};
use crate::services::rebalance::{REBAL_META_NAME, RebalanceMeta, is_rebalance_conflicting_with_decommission};
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
use crate::storage_api_contracts::{
@@ -2384,10 +2389,12 @@ struct DecommissionDurableIlmReceipt {
id: String,
checkpoint: DurableIlmRecordCheckpoint,
terminal_checkpoint: Option<DurableIlmRecordCheckpoint>,
#[serde(default, skip_serializing_if = "Option::is_none")]
fleet_topology_generation: Option<String>,
}
impl DecommissionDurableIlmReceipt {
fn new(path: &str, record: &ValidatedDurableIlmRecord) -> Self {
fn new(path: &str, record: &ValidatedDurableIlmRecord, fleet_topology_generation: Option<String>) -> Self {
Self {
source_path: path.to_string(),
namespace: record.namespace.to_string(),
@@ -2395,6 +2402,7 @@ impl DecommissionDurableIlmReceipt {
id: record.id.clone(),
checkpoint: record.checkpoint.clone(),
terminal_checkpoint: None,
fleet_topology_generation,
}
}
@@ -2433,6 +2441,16 @@ impl DecommissionDurableIlmReceipt {
))
})?;
}
if self
.fleet_topology_generation
.as_deref()
.is_some_and(|generation| !is_sha256_checksum(generation))
{
return Err(Error::other_with_context(
"receipt fleet topology generation is invalid",
format!("source path `{}` {}", self.source_path, self.context()),
));
}
Ok(())
}
@@ -2541,6 +2559,21 @@ fn merge_decommission_durable_ilm_receipts(
id: existing.id.clone(),
checkpoint,
terminal_checkpoint,
fleet_topology_generation: match (
existing.fleet_topology_generation.as_ref(),
incoming.fleet_topology_generation.as_ref(),
) {
(Some(existing_generation), Some(incoming_generation)) if existing_generation == incoming_generation => {
Some(existing_generation.clone())
}
(None, None) => None,
_ => {
return Err(Error::other_with_context(
"durable ILM receipt fleet topology conflict",
format!("source path `{}` {}", existing.source_path, existing.context()),
));
}
},
};
merged.validate()?;
Ok(merged)
@@ -7050,6 +7083,15 @@ pub(crate) struct DecommissionCapacityOwner {
pub(crate) mutation_id: Option<uuid::Uuid>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct DecommissionDurableIlmCheckpointTarget {
pub(crate) source_pool_index: usize,
pub(crate) target_pool_index: usize,
pub(crate) capacity_owner: DecommissionCapacityOwner,
pub(crate) already_committed: bool,
pub(crate) target_etag: Option<String>,
}
impl DecommissionCapacityOwner {
pub(crate) fn apply_to(self, opts: &mut ObjectOptions) {
opts.src_pool_idx = self.source_pool_index;
@@ -7457,6 +7499,7 @@ impl DecommissionCapacityLockOrderBarrier {
Self { state }
}
#[cfg(feature = "test-util")]
pub(crate) async fn wait_until_owner_paused(&self) {
tokio::time::timeout(std::time::Duration::from_secs(30), self.state.owner_arrived.notified())
.await
@@ -7469,6 +7512,7 @@ impl DecommissionCapacityLockOrderBarrier {
.expect("external mutation should release capacity before waiting for the object namespace");
}
#[cfg(feature = "test-util")]
pub(crate) async fn wait_until_external_object_capacity_probe_acquired(&self) {
tokio::time::timeout(
std::time::Duration::from_secs(30),
@@ -7478,6 +7522,7 @@ impl DecommissionCapacityLockOrderBarrier {
.expect("external object mutation should acquire its no-active capacity probe");
}
#[cfg(feature = "test-util")]
pub(crate) async fn wait_until_external_object_commit_phase_started(&self) {
tokio::time::timeout(
std::time::Duration::from_secs(30),
@@ -7502,24 +7547,29 @@ impl DecommissionCapacityLockOrderBarrier {
.expect("external heal should attempt the target namespace lock after capacity admission");
}
#[cfg(feature = "test-util")]
pub(crate) fn release_owner(&self) {
self.state.owner_release.notify_one();
}
#[cfg(feature = "test-util")]
pub(crate) fn pause_external_object_commit_phase(&self) {
self.state.external_object_commit_phase_paused.store(true, Ordering::Release);
}
#[cfg(feature = "test-util")]
pub(crate) fn release_external_object_commit_phase(&self) {
self.state.external_object_commit_phase_release.notify_one();
}
#[cfg(feature = "test-util")]
pub(crate) fn pause_external_object_capacity_probe(&self) {
self.state
.external_object_capacity_probe_paused
.store(true, Ordering::Release);
}
#[cfg(feature = "test-util")]
pub(crate) fn release_external_object_capacity_probe(&self) {
self.state.external_object_capacity_probe_release.notify_one();
}
@@ -8315,6 +8365,122 @@ impl ECStore {
.await
}
/// Finish the capacity transaction for an identity-preserving temporary
/// replacement whose target bytes are already durably present. This is
/// the crash-recovery half of `run_decommission_capacity_temporary_mutation`:
/// it never writes the target again and only resolves a pending intent
/// owned by the exact deterministic mutation id.
pub(crate) async fn reconcile_decommission_capacity_after_equivalent_temporary_target(
&self,
owner: DecommissionCapacityOwner,
target_pool_index: usize,
expected_data_bytes: usize,
) -> Result<()> {
let mut save_guard = self.pool_meta_save_gate.lock().await;
let (pool_meta_guard, mut snapshot) = self
.acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent temporary target reconciliation failed")
.await?;
let source_pool_index = owner.source_pool_index;
let mutation_id = owner
.mutation_id
.ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target mutation identity is missing"))?;
let (target_layout, pending_physical_bytes, pending_mutation_id, already_reconciled) = {
let reservation = snapshot
.pools
.get(source_pool_index)
.and_then(|pool| pool.decommission.as_ref())
.and_then(|info| info.capacity_reservation.as_ref())
.filter(|reservation| reservation.admits_cleanup_owner(owner))
.ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target owner is stale"))?;
let target = reservation
.targets
.iter()
.find(|target| target.pool_index == target_pool_index)
.ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target allocation is missing"))?;
(
target.layout,
target.pending_physical_bytes,
target.pending_mutation_id,
target
.temporary_mutations
.iter()
.any(|mutation| mutation.mutation_id == mutation_id),
)
};
if pending_physical_bytes == 0 {
// Either the successful attempt already saved its progress, or a
// byte-neutral replacement had no inflight delta to record.
ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation fence failed")?;
return Ok(());
}
if pending_mutation_id != Some(mutation_id) {
return Err(decommission_capacity_blocked_error(
"equivalent temporary target pending intent belongs to another mutation",
));
}
if already_reconciled {
return Err(decommission_capacity_blocked_error(
"equivalent temporary target has both pending and reconciled state",
));
}
let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?;
if pending_physical_bytes < expected_target_physical_bytes {
return Err(decommission_capacity_blocked_error(
"equivalent temporary target pending capacity is smaller than the committed checkpoint",
));
}
resolve_decommission_target_pending(
&mut snapshot,
source_pool_index,
target_pool_index,
expected_target_physical_bytes,
mutation_id,
)?;
// The replacement is byte-non-growing and its exact bytes were read
// before this call, so no new physical delta is inferred on replay.
// A prior successful progress save would have taken the idempotent
// pending==0 return above.
record_decommission_target_inflight(
&mut snapshot,
source_pool_index,
target_pool_index,
0,
mutation_id,
OffsetDateTime::now_utc(),
)?;
let outcome = snapshot
.save_no_lock_armed(
self.pools.clone(),
&mut save_guard,
pool_meta_guard.lock_lost_signal(),
&[source_pool_index],
)
.await?;
ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?;
let persisted_info = outcome
.committed
.pools
.get(source_pool_index)
.and_then(|pool| pool.decommission.as_ref())
.cloned()
.ok_or_else(|| decommission_metadata_not_initialized_error("publish equivalent temporary target reconciliation"))?;
{
let mut pool_meta = self.pool_meta.write().await;
let pool_count = pool_meta.pools.len();
pool_meta.version = pool_meta.version.max(outcome.committed.version);
let info = pool_meta
.pools
.get_mut(source_pool_index)
.and_then(|pool| pool.decommission.as_mut())
.ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
info.capacity_reservation = persisted_info.capacity_reservation;
info.capacity_blocked_reason = persisted_info.capacity_blocked_reason;
}
ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?;
outcome.disarm();
Ok(())
}
pub(crate) async fn has_decommission_capacity_temporary_mutation_state(
&self,
target_pool_index: usize,
@@ -12847,6 +13013,7 @@ impl ECStore {
let receipt_path = decommission_durable_ilm_receipt_path(run_token, path, source_record.id_kind, &source_record.id);
let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
let mut proof = None::<DecommissionDurableIlmReceipt>;
let mut nonterminal_receipt_found = false;
for pool_idx in 0..self.pools.len() {
if pool_idx == source_pool_idx {
continue;
@@ -12890,23 +13057,35 @@ impl ECStore {
receipt.context()
)));
}
source_record
.checkpoint
.validate_successor(&receipt.checkpoint)
.map_err(|err| {
Error::other(format!(
"terminal durable ILM decommission receipt does not cover source at path `{path}` {}: {err}",
source_record.context()
))
})?;
if receipt.terminal_checkpoint.is_some() {
if let Some(terminal_checkpoint) = receipt.terminal_checkpoint.as_ref() {
if !source_record.checkpoint.is_predecessor_of_terminal(terminal_checkpoint) {
return Err(Error::other_with_context(
"terminal durable ILM decommission receipt does not cover source",
format!("path `{path}` {}", source_record.context()),
));
}
proof = Some(match proof {
Some(existing) => merge_decommission_durable_ilm_receipts(&existing, &receipt)?,
None => receipt,
});
} else {
source_record
.checkpoint
.validate_successor(&receipt.checkpoint)
.map_err(|err| {
Error::other_with_context(
"durable ILM decommission receipt does not cover source",
format!("path `{path}` {}: {err}", source_record.context()),
)
})?;
nonterminal_receipt_found = true;
}
}
Ok(proof)
// A terminal receipt on one target must not hide another target copy
// whose receipt was installed later and has not reached terminal yet.
// Returning no proof makes recovery advance every outstanding copy
// before source cleanup can treat the operation as complete.
if nonterminal_receipt_found { Ok(None) } else { Ok(proof) }
}
async fn verify_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> {
@@ -13050,6 +13229,7 @@ impl ECStore {
pool_idx: usize,
receipt_path: &str,
record: &ValidatedDurableIlmRecord,
fleet_topology_generation: Option<&str>,
terminal: bool,
) -> Result<bool> {
let stage = if terminal { "terminal" } else { "progress" };
@@ -13085,6 +13265,12 @@ impl ECStore {
))
})?;
Self::validate_decommission_durable_ilm_receipt_locator(receipt_path, &locator, &receipt)?;
if receipt.fleet_topology_generation.as_deref() != fleet_topology_generation {
return Err(Error::other_with_context(
"durable ILM decommission receipt fleet topology mismatch",
format!("path `{}` {}", receipt.source_path, receipt.context()),
));
}
receipt.checkpoint.validate_successor(&record.checkpoint).map_err(|err| {
Error::other(format!(
"{stage} durable ILM record generation mismatch at path `{}` {}: {err}",
@@ -13197,6 +13383,7 @@ impl ECStore {
let stage = if terminal { "terminal" } else { "progress" };
let record = validate_durable_ilm_record(path, data)
.map_err(|err| Error::other(format!("{stage} durable ILM record is invalid at path `{path}`: {err}")))?;
let fleet_topology_generation = durable_ilm_v6_topology_generation(path, data)?;
let active_source_pool_indices = active_runs.iter().map(|(pool_idx, _)| *pool_idx).collect::<Vec<_>>();
let mut terminal_target_pool_indices = Vec::new();
for (source_pool_idx, run_token) in active_runs {
@@ -13205,7 +13392,13 @@ impl ECStore {
for pool_idx in 0..self.pools.len() {
if pool_idx != source_pool_idx {
let found = self
.advance_durable_ilm_decommission_receipt(pool_idx, &receipt_path, &record, terminal)
.advance_durable_ilm_decommission_receipt(
pool_idx,
&receipt_path,
&record,
fleet_topology_generation.as_deref(),
terminal,
)
.await?;
receipt_found |= found;
if terminal
@@ -13227,6 +13420,240 @@ impl ECStore {
Ok(Some(terminal_target_pool_indices))
}
/// Resolve the exact receipt-bearing target copies on which a v6 dispatch
/// manifest may advance while a decommission reservation is active.
///
/// This is intentionally not a general capacity bypass. The target copy
/// must still be covered by the active source reservation and its durable
/// receipt, the ETag must be the caller's exact CAS generation, and the
/// replacement must be a byte-non-growing adjacent manifest checkpoint.
pub(crate) async fn decommission_durable_ilm_checkpoint_targets(
&self,
path: &str,
next_data: &[u8],
expected_etag: &str,
) -> Result<Option<Vec<DecommissionDurableIlmCheckpointTarget>>> {
let active_runs = {
let pool_meta = self.pool_meta.read().await;
let mut active_runs = Vec::new();
for (source_pool_index, pool) in pool_meta.pools.iter().enumerate() {
let Some(info) = pool
.decommission
.as_ref()
.filter(|info| info.has_decommission_state() && !info.complete)
else {
continue;
};
let Some(start_time) = info.start_time else {
continue;
};
let reservation = info.capacity_reservation.clone().ok_or_else(|| {
decommission_capacity_blocked_error(format!(
"active decommission source pool {source_pool_index} has no reservation for durable ILM checkpoint"
))
})?;
if !reservation.active() {
return Err(decommission_capacity_blocked_error(format!(
"active decommission source pool {source_pool_index} has a released reservation for durable ILM checkpoint"
)));
}
active_runs.push((
source_pool_index,
decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time),
reservation,
));
}
active_runs
};
if active_runs.is_empty() {
return Ok(None);
}
let next_record = validate_durable_ilm_record(path, next_data)?;
if next_record.namespace != TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE.name {
return Ok(None);
}
let next_fleet_topology_generation = durable_ilm_v6_topology_generation(path, next_data)?;
let mut targets = Vec::<DecommissionDurableIlmCheckpointTarget>::new();
let mut missing_source_receipts = 0usize;
let mut stale_target_etag_mismatch = false;
for (source_pool_index, run_token, reservation) in active_runs {
let receipt_path = decommission_durable_ilm_receipt_path(&run_token, path, next_record.id_kind, &next_record.id);
let mut source_receipt_found = false;
for allocation in &reservation.targets {
let receipt_data = match read_config_limited_preserve_empty(
self.pools[allocation.pool_index].clone(),
&receipt_path,
DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
)
.await
{
Ok(data) => data,
Err(err)
if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
|| is_err_object_not_found(&err)
|| is_err_version_not_found(&err) =>
{
continue;
}
Err(err) => return Err(err),
};
let receipt = DecommissionDurableIlmReceipt::decode(&receipt_data)?;
let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &receipt)?;
if receipt.source_path != path
|| receipt.namespace != next_record.namespace
|| receipt.id_kind != next_record.id_kind
|| receipt.id != next_record.id
{
return Err(Error::other_with_context(
"durable ILM checkpoint receipt identity does not authorize source",
format!("path `{path}` {}", next_record.context()),
));
}
if receipt.fleet_topology_generation != next_fleet_topology_generation {
return Err(Error::other_with_context(
"durable ILM checkpoint receipt fleet topology does not authorize source",
format!("path `{path}` {}", next_record.context()),
));
}
receipt
.checkpoint
.validate_successor(&next_record.checkpoint)
.map_err(|err| {
Error::other_with_context(
"durable ILM checkpoint receipt is not a predecessor of the requested generation",
format!(
"target pool {}, path `{path}` {}; receipt checkpoint {:?}, requested checkpoint {:?}: {err}",
allocation.pool_index,
next_record.context(),
receipt.checkpoint,
next_record.checkpoint
),
)
})?;
let (target_data, metadata) = read_config_limited_preserve_empty_with_metadata(
self.pools[allocation.pool_index].clone(),
path,
TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE.max_record_size,
)
.await?;
let target_record = validate_durable_ilm_record(path, &target_data)?;
if target_record.namespace != next_record.namespace
|| target_record.id_kind != next_record.id_kind
|| target_record.id != next_record.id
{
return Err(Error::other_with_context(
"durable ILM checkpoint target identity does not match source",
format!("path `{path}` {}", next_record.context()),
));
}
let already_committed = target_data.as_slice() == next_data;
let target_etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| {
Error::other_with_context("durable ILM checkpoint target is missing an ETag", format!("path `{path}`"))
})?;
if already_committed {
receipt
.checkpoint
.validate_successor(&target_record.checkpoint)
.map_err(|err| {
Error::other_with_context(
"durable ILM checkpoint receipt is not a predecessor of the committed target generation",
format!("path `{path}` {}: {err}", next_record.context()),
)
})?;
} else {
if target_record.checkpoint != receipt.checkpoint {
return Err(Error::other_with_context(
"durable ILM checkpoint target is not the receipt generation",
format!("path `{path}` {}", next_record.context()),
));
}
target_record
.checkpoint
.validate_successor(&next_record.checkpoint)
.map_err(|err| {
Error::other_with_context(
"durable ILM checkpoint target is not a predecessor of the requested generation",
format!("path `{path}` {}: {err}", next_record.context()),
)
})?;
if next_data.len() > target_data.len() {
return Err(decommission_capacity_blocked_error(format!(
"durable ILM checkpoint update at `{path}` grows from {} to {} bytes",
target_data.len(),
next_data.len()
)));
}
stale_target_etag_mismatch |= target_etag != expected_etag;
}
source_receipt_found = true;
if let Some(existing) = targets
.iter_mut()
.find(|target| target.target_pool_index == allocation.pool_index)
{
if existing.already_committed != already_committed {
return Err(Error::other_with_context(
"durable ILM checkpoint target state changed during authorization",
format!("path `{path}`"),
));
}
continue;
}
let base_owner = DecommissionCapacityOwner {
source_pool_index,
operation_id: reservation.operation_id,
generation: reservation.generation,
owner_nonce: reservation.owner_nonce,
mutation_id: None,
};
let mutation_id = decommission_capacity_mutation_id(
base_owner,
RUSTFS_META_BUCKET,
path,
Some(next_record.checkpoint.content_sha256()),
false,
None,
);
targets.push(DecommissionDurableIlmCheckpointTarget {
source_pool_index,
target_pool_index: allocation.pool_index,
capacity_owner: base_owner.with_mutation_id(mutation_id),
already_committed,
target_etag: Some(target_etag),
});
}
if !source_receipt_found {
missing_source_receipts = missing_source_receipts.saturating_add(1);
}
}
if missing_source_receipts > 0 {
return Err(decommission_capacity_blocked_error(format!(
"durable ILM checkpoint at `{path}` is missing receipt coverage for {missing_source_receipts} active source(s)"
)));
}
if targets.is_empty() {
return Err(decommission_capacity_blocked_error(format!(
"durable ILM checkpoint at `{path}` has no receipt-bearing reservation target"
)));
}
if stale_target_etag_mismatch && !targets.iter().any(|target| target.already_committed) {
return Err(Error::PreconditionFailed);
}
// After a partial multi-target commit, an aggregate read may return
// the ETag of the already-advanced target while another authorized
// target still has the predecessor ETag. The exact committed bytes
// plus every target's receipt/checkpoint proof authorize repairing
// that predecessor with its own target-local CAS. Without an exact
// committed target, retain the caller-ETag requirement above.
targets.sort_unstable_by_key(|target| target.target_pool_index);
Ok(Some(targets))
}
pub(crate) async fn record_durable_ilm_decommission_progress(&self, path: &str, data: &[u8]) -> Result<()> {
self.advance_durable_ilm_decommission_receipts(path, data, false)
.await
@@ -13248,6 +13675,66 @@ impl ECStore {
self.advance_durable_ilm_decommission_receipts(path, data, true).await
}
/// Return true only when `data` is the exact copy still owned by an active
/// decommission source and a target-side terminal receipt authorizes that
/// source's later verified cleanup. Lifecycle recovery may then regard the
/// logical record as terminal without deleting the source checkpoint.
pub(crate) async fn durable_ilm_terminal_receipt_covers_active_source(&self, path: &str, data: &[u8]) -> Result<bool> {
let namespace = classify_durable_ilm_record(path)?
.ok_or_else(|| Error::other_with_context("path is not a durable ILM record", format!("path `{path}`")))?;
let source_record = validate_durable_ilm_record(path, data)?;
let active_runs = {
let pool_meta = self.pool_meta.read().await;
pool_meta
.pools
.iter()
.enumerate()
.filter_map(|(source_pool_index, pool)| {
pool.decommission
.as_ref()
.filter(|info| info.has_decommission_state() && !info.complete)
.and_then(|info| info.start_time)
.map(|start_time| {
(source_pool_index, decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time))
})
})
.collect::<Vec<_>>()
};
let mut covered_active_source = false;
for (source_pool_index, run_token) in active_runs {
let source_data =
match read_config_limited_preserve_empty(self.pools[source_pool_index].clone(), path, namespace.max_record_size)
.await
{
Ok(source_data) => source_data,
Err(err)
if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
|| is_err_object_not_found(&err)
|| is_err_version_not_found(&err) =>
{
continue;
}
Err(err) => return Err(err),
};
if source_data.as_slice() != data {
continue;
}
if self
.load_decommission_durable_ilm_terminal_receipt_for_run(source_pool_index, path, &source_record, &run_token)
.await?
.is_some()
{
covered_active_source = true;
} else {
// Every active source that still stores this exact generation
// needs complete terminal receipt coverage. One covered source
// cannot authorize lifecycle recovery to skip another.
return Ok(false);
}
}
Ok(covered_active_source)
}
async fn cleanup_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> {
for (pool_idx, receipt_path) in self.list_decommission_durable_ilm_receipts(source_pool_idx).await? {
match delete_config(self.pools[pool_idx].clone(), &receipt_path).await {
@@ -13316,12 +13803,20 @@ impl ECStore {
.map_err(|err| Error::other(format!("failed to read source durable ILM record at path `{path}`: {err}")))?;
let source_record = validate_durable_ilm_record(path, &source)
.map_err(|err| Error::other(format!("source durable ILM record is invalid at path `{path}`: {err}")))?;
let source_fleet_topology_generation = durable_ilm_v6_topology_generation(path, &source)?;
let target = self
.load_decommissioned_durable_ilm_target(source_pool_idx, path, namespace.max_record_size, &source_record.context())
.await?;
let manifest_receipt = if let Some((target_pool_idx, target)) = target {
let target_record = validate_decommission_durable_ilm_copy(path, &source_record, &target)?;
let receipt = DecommissionDurableIlmReceipt::new(path, &target_record);
let target_fleet_topology_generation = durable_ilm_v6_topology_generation(path, &target)?;
if target_fleet_topology_generation != source_fleet_topology_generation {
return Err(Error::other_with_context(
"target durable ILM fleet topology generation differs from source",
format!("path `{path}` {}", source_record.context()),
));
}
let receipt = DecommissionDurableIlmReceipt::new(path, &target_record, target_fleet_topology_generation);
self.persist_decommission_durable_ilm_receipt_for_run(target_pool_idx, &receipt, run_token)
.await?;
receipt
@@ -13335,9 +13830,34 @@ impl ECStore {
))
})?
};
if manifest_receipt.fleet_topology_generation != source_fleet_topology_generation {
return Err(Error::other_with_context(
"terminal durable ILM receipt fleet topology generation does not cover source",
format!("path `{path}` {}", source_record.context()),
));
}
let fleet_proof = if let Some(expected_generation) = source_fleet_topology_generation.as_deref() {
let proof = acquire_tier_delete_journal_fleet_proof()
.ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable for source cleanup"))?;
if tier_delete_journal_topology_generation(&proof) != expected_generation
|| !tier_delete_journal_fleet_proof_matches(&proof)
{
return Err(Error::other("tier delete journal v6 fleet generation changed before source cleanup"));
}
Some(proof)
} else {
None
};
self.persist_decommission_durable_ilm_receipt_for_run(source_pool_idx, &manifest_receipt, run_token)
.await?;
if fleet_proof
.as_ref()
.is_some_and(|proof| !tier_delete_journal_fleet_proof_matches(proof))
{
return Err(Error::other("tier delete journal v6 fleet proof changed before source cleanup"));
}
let cleanup_result = data_movement::cleanup_source_entry_if_unchanged(
source_set,
RUSTFS_META_BUCKET,
@@ -13354,7 +13874,16 @@ impl ECStore {
source_record.context()
))
});
resolve_decommission_entry_cleanup_delete_result(cleanup_result, RUSTFS_META_BUCKET, path)
let cleanup_result = resolve_decommission_entry_cleanup_delete_result(cleanup_result, RUSTFS_META_BUCKET, path);
if fleet_proof
.as_ref()
.is_some_and(|proof| !tier_delete_journal_fleet_proof_matches(proof))
{
return Err(Error::other(
"tier delete journal v6 fleet proof changed during source cleanup; exact source outcome requires verification",
));
}
cleanup_result
}
#[cfg(all(test, feature = "test-util"))]
@@ -13390,7 +13919,26 @@ impl ECStore {
record: &ValidatedDurableIlmRecord,
terminal: bool,
) -> Result<String> {
let mut receipt = DecommissionDurableIlmReceipt::new(source_path, record);
let fleet_topology_generation = match read_config_limited_preserve_empty(
self.pools[target_pool_idx].clone(),
source_path,
classify_durable_ilm_record(source_path)?
.ok_or_else(|| Error::other_with_context("path is not a durable ILM record", format!("path `{source_path}`")))?
.max_record_size,
)
.await
{
Ok(target_data) => durable_ilm_v6_topology_generation(source_path, &target_data)?,
Err(err)
if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
|| is_err_object_not_found(&err)
|| is_err_version_not_found(&err) =>
{
None
}
Err(err) => return Err(err),
};
let mut receipt = DecommissionDurableIlmReceipt::new(source_path, record, fleet_topology_generation);
if terminal {
receipt.terminal_checkpoint = Some(record.checkpoint.clone());
}
@@ -17361,11 +17909,15 @@ mod pools_tests {
content_sha256: "b".repeat(64),
identity_sha256: "c".repeat(64),
committed: false,
dispatch_identity_sha256: None,
state: None,
};
let terminal_checkpoint = DurableIlmRecordCheckpoint::TierDeleteJournal {
content_sha256: "d".repeat(64),
identity_sha256: "c".repeat(64),
committed: true,
dispatch_identity_sha256: None,
state: None,
};
let incoming = DecommissionDurableIlmReceipt {
source_path,
@@ -17374,6 +17926,7 @@ mod pools_tests {
id: operation_id,
checkpoint: checkpoint.clone(),
terminal_checkpoint: None,
fleet_topology_generation: None,
};
let existing = DecommissionDurableIlmReceipt {
terminal_checkpoint: Some(terminal_checkpoint.clone()),
@@ -17384,7 +17937,24 @@ mod pools_tests {
.expect("retry receipt must merge with a terminal receipt");
assert_eq!(merged.checkpoint, checkpoint);
assert_eq!(merged.terminal_checkpoint, Some(terminal_checkpoint));
assert_eq!(merged.terminal_checkpoint, Some(terminal_checkpoint.clone()));
let topology_bound = DecommissionDurableIlmReceipt {
fleet_topology_generation: Some("e".repeat(64)),
..incoming
};
let mixed_error = merge_decommission_durable_ilm_receipts(&existing, &topology_bound)
.expect_err("a topology-bound v6 receipt must not mask an unbound receipt")
.to_string();
assert!(mixed_error.contains("fleet topology conflict"));
let topology_existing = DecommissionDurableIlmReceipt {
terminal_checkpoint: Some(terminal_checkpoint),
..topology_bound.clone()
};
let topology_merged = merge_decommission_durable_ilm_receipts(&topology_existing, &topology_bound)
.expect("receipts bound to the same fleet topology should merge");
assert_eq!(topology_merged.fleet_topology_generation, Some("e".repeat(64)));
}
#[test]
@@ -17409,7 +17979,7 @@ mod pools_tests {
assert!(job_bytes.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE);
let record = validate_durable_ilm_record(&path, &job_bytes).expect("large manual job should validate");
let expected_checkpoint = record.checkpoint.clone();
let mut receipt = DecommissionDurableIlmReceipt::new(&path, &record);
let mut receipt = DecommissionDurableIlmReceipt::new(&path, &record, None);
receipt.terminal_checkpoint = Some(record.checkpoint);
let encoded = receipt.encode().expect("bounded progress proof should fit the receipt limit");
+16 -2
View File
@@ -37,8 +37,9 @@ use crate::{
runtime::instance::{InstanceContext, bootstrap_ctx},
runtime::sources as runtime_sources,
set_disk::{PreparedGetObjectMetadata, SetDisks},
store::init_format::{
check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum,
store::{
RemoteTuplePublicationFence,
init_format::{check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum},
},
};
use futures::{
@@ -625,6 +626,19 @@ impl Sets {
.put_object_with_old_current_size(bucket, object, data, opts)
.await
}
pub(crate) async fn put_object_with_old_current_size_for_data_movement(
&self,
bucket: &str,
object: &str,
data: &mut PutObjReader,
opts: &ObjectOptions,
publication_fence: RemoteTuplePublicationFence,
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
self.get_disks_by_key(object)
.put_object_with_old_current_size_for_data_movement(bucket, object, data, opts, publication_fence)
.await
}
}
#[async_trait::async_trait]
+49 -21
View File
@@ -27,7 +27,7 @@ use crate::storage_api_contracts::{
namespace::NamespaceLocking as _,
object::{HTTPPreconditions, ObjectOperations as _},
};
use crate::store::{ECStore, ObjectLockDiagGuard, SourceCleanupMutationFence};
use crate::store::{DecommissionFixedReadAnchor, ECStore, SourceCleanupMutationFence};
use bytes::Bytes;
use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
@@ -161,7 +161,7 @@ pub fn mark_multipart_upload_completed(flag: &Arc<AtomicBool>) {
flag.store(false, Ordering::Relaxed);
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
struct DataMovementMultipartAbortBarrierState {
bucket: String,
object: String,
@@ -169,17 +169,17 @@ struct DataMovementMultipartAbortBarrierState {
release: tokio::sync::Notify,
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pub(crate) struct DataMovementMultipartAbortBarrier {
state: Arc<DataMovementMultipartAbortBarrierState>,
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
static DATA_MOVEMENT_MULTIPART_ABORT_BARRIER: std::sync::OnceLock<
std::sync::Mutex<Option<Arc<DataMovementMultipartAbortBarrierState>>>,
> = std::sync::OnceLock::new();
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
impl DataMovementMultipartAbortBarrier {
pub(crate) fn install(bucket: &str, object: &str) -> Self {
let state = Arc::new(DataMovementMultipartAbortBarrierState {
@@ -204,7 +204,7 @@ impl DataMovementMultipartAbortBarrier {
}
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
impl Drop for DataMovementMultipartAbortBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
@@ -218,7 +218,7 @@ impl Drop for DataMovementMultipartAbortBarrier {
}
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
async fn pause_data_movement_multipart_before_abort(bucket: &str, object: &str) {
let barrier = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
@@ -1518,7 +1518,7 @@ pub(crate) async fn migrate_decommission_object(
capacity_owner: Option<DecommissionCapacityOwner>,
) -> Result<()> {
let source = rd.object_info.clone();
let _mutation_fence = store
let mutation_fence = store
.acquire_decommission_object_mutation_fence(&bucket, &source.name)
.await?;
let current = find_data_movement_target_info(store.as_ref(), pool_idx, &bucket, &source)
@@ -1537,7 +1537,7 @@ pub(crate) async fn migrate_decommission_object(
op_label,
None,
capacity_owner,
Some(&_mutation_fence),
Some(mutation_fence),
)
.await
}
@@ -1588,8 +1588,9 @@ async fn migrate_object_inner(
op_label: &str,
lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
capacity_owner: Option<DecommissionCapacityOwner>,
mutation_fence: Option<&ObjectLockDiagGuard>,
mutation_fence: Option<DecommissionFixedReadAnchor>,
) -> Result<()> {
let mut mutation_fence = mutation_fence;
let object_info = rd.object_info.clone();
let capacity_owner = capacity_owner.map(|owner| {
let version_id = object_info.version_id.map(|version_id| version_id.to_string());
@@ -1605,6 +1606,13 @@ async fn migrate_object_inner(
});
owner.with_mutation_id(mutation_id)
});
// Capture the exact source/tier identity before any client-paced read, but
// defer both the tier lease and source/target write locks to the final
// publication. Decommission already owns main's fixed-domain mutation
// fence, so reacquiring that domain as a write lock would self-deadlock.
let remote_tuple_publication_fence = store
.acquire_remote_tuple_publication_fence(&bucket, pool_idx, &object_info, false)
.await?;
let has_part_checksums = object_info
.parts
.iter()
@@ -1656,8 +1664,8 @@ async fn migrate_object_inner(
}
let mut cleanup_opts =
data_movement_abort_opts(pool_idx, source_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
if let Some(fence) = mutation_fence {
fence.add_namespace_lock_fence(&mut cleanup_opts);
if let Some(anchor) = mutation_fence.as_ref() {
anchor.guard().add_namespace_lock_fence(&mut cleanup_opts);
}
if let Some(fence) = multipart_mutation_fence.as_ref() {
fence.add_namespace_lock_fence(&mut cleanup_opts);
@@ -1684,7 +1692,12 @@ async fn migrate_object_inner(
}
}
let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
.handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
.handle_new_multipart_upload_with_pool_idx(
&bucket,
&object_info.name,
&new_multipart_opts,
mutation_fence.as_ref().map(DecommissionFixedReadAnchor::guard),
)
.await
{
Ok(res) => res,
@@ -1797,15 +1810,20 @@ async fn migrate_object_inner(
if let Some(fence) = multipart_mutation_fence.as_ref() {
fence.add_namespace_lock_fence(&mut complete_multipart_opts);
}
let remote_tuple_publication_fence = match mutation_fence.take() {
Some(anchor) => remote_tuple_publication_fence.under_fixed_read_anchor(anchor)?,
None => remote_tuple_publication_fence,
};
if let Err(err) = store
.clone()
.complete_multipart_upload_for_data_movement(
(target_pool_idx, mutation_fence),
.complete_multipart_upload_for_data_movement_with_publication_fence(
target_pool_idx,
&bucket,
&object_info.name,
&res.upload_id,
parts,
&complete_multipart_opts,
remote_tuple_publication_fence,
)
.await
{
@@ -1849,8 +1867,8 @@ async fn migrate_object_inner(
if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) {
let mut abort_opts =
data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
if let Some(fence) = mutation_fence {
fence.add_namespace_lock_fence(&mut abort_opts);
if let Some(anchor) = mutation_fence.as_ref() {
anchor.guard().add_namespace_lock_fence(&mut abort_opts);
}
if let Some(fence) = multipart_mutation_fence.as_ref() {
fence.add_namespace_lock_fence(&mut abort_opts);
@@ -1923,12 +1941,12 @@ async fn migrate_object_inner(
if let Err(primary_err) = multipart_result {
if should_abort_multipart_upload(&abort_multipart_flag) {
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pause_data_movement_multipart_before_abort(&bucket, &object_info.name).await;
let mut abort_opts =
data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
if let Some(fence) = mutation_fence {
fence.add_namespace_lock_fence(&mut abort_opts);
if let Some(anchor) = mutation_fence.as_ref() {
anchor.guard().add_namespace_lock_fence(&mut abort_opts);
}
if let Some(fence) = multipart_mutation_fence.as_ref() {
fence.add_namespace_lock_fence(&mut abort_opts);
@@ -1982,8 +2000,18 @@ async fn migrate_object_inner(
if let Some(signal) = lock_lost_signal {
put_opts.add_namespace_lock_lost_signal(signal);
}
let remote_tuple_publication_fence = match mutation_fence.take() {
Some(anchor) => remote_tuple_publication_fence.under_fixed_read_anchor(anchor)?,
None => remote_tuple_publication_fence,
};
let (target_pool_idx, put_result) = store
.put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence)
.put_object_for_data_movement_with_publication_fence(
&bucket,
&object_info.name,
&mut data,
&put_opts,
remote_tuple_publication_fence,
)
.await
.map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?;
if let Err(err) = put_result {
@@ -278,3 +278,17 @@ fn reduce_errs_buckets_identical_other_messages_together() {
assert_eq!(count, 3);
assert_eq!(err, Some(DiskError::other("can not get client")));
}
#[test]
fn stable_io_context_buckets_by_cause_and_preserves_diagnostic_source() {
let first = StorageError::other_with_context("tier mutation intent changed", "mutation-a");
let second = StorageError::other_with_context("tier mutation intent changed", "mutation-b");
assert_eq!(first, second, "diagnostic identity must not split quorum buckets");
let StorageError::Io(io_error) = first else {
panic!("stable context must remain an io error");
};
assert_eq!(io_error.to_string(), "tier mutation intent changed");
let context = io_error.get_ref().expect("stable context must remain downcastable");
assert_eq!(context.source().expect("diagnostic source must be retained").to_string(), "mutation-a");
}
+37
View File
@@ -23,6 +23,36 @@ use s3s::S3ErrorCode;
pub type Error = StorageError;
pub type Result<T> = core::result::Result<T, Error>;
/// Keeps high-cardinality diagnostic detail in the error source while making
/// the rendered `io::Error` stable for quorum aggregation.
#[derive(Debug)]
struct StableIoContextError {
message: &'static str,
source: Box<dyn std::error::Error + Send + Sync>,
}
impl std::fmt::Display for StableIoContextError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(self.message)
}
}
impl std::error::Error for StableIoContextError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(self.source.as_ref())
}
}
pub(crate) fn stable_io_error<E>(message: &'static str, source: E) -> std::io::Error
where
E: Into<Box<dyn std::error::Error + Send + Sync>>,
{
std::io::Error::other(StableIoContextError {
message,
source: source.into(),
})
}
/// Storage layer error type covering disk, volume, bucket, object, multipart,
/// erasure-coding, and operational error conditions.
///
@@ -264,6 +294,13 @@ impl StorageError {
StorageError::Io(std::io::Error::other(error))
}
pub(crate) fn other_with_context<E>(message: &'static str, source: E) -> Self
where
E: Into<Box<dyn std::error::Error + Send + Sync>>,
{
StorageError::Io(stable_io_error(message, source))
}
pub fn is_not_found(&self) -> bool {
matches!(
self,
+9 -15
View File
@@ -301,36 +301,23 @@ pub enum LifecycleDeleteAllPhase {
#[doc(hidden)]
#[derive(Default)]
pub struct LifecycleDeleteAllJournalState {
prepared: HashMap<String, crate::bucket::lifecycle::tier_sweeper::Jentry>,
mutation_started: bool,
}
impl Debug for LifecycleDeleteAllJournalState {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("LifecycleDeleteAllJournalState")
.field("prepared_count", &self.prepared.len())
.field("mutation_started", &self.mutation_started)
.finish()
}
}
impl LifecycleDeleteAllJournalState {
pub(crate) fn contains(&self, name: &str) -> bool {
self.prepared.contains_key(name)
}
pub(crate) fn insert(&mut self, name: String, entry: crate::bucket::lifecycle::tier_sweeper::Jentry) {
self.prepared.insert(name, entry);
}
pub(crate) fn prepared_entries(&self) -> Vec<crate::bucket::lifecycle::tier_sweeper::Jentry> {
self.prepared.values().cloned().collect()
}
pub(crate) fn mark_mutation_started(&mut self) {
self.mutation_started = true;
}
#[cfg(test)]
pub(crate) fn mutation_started(&self) -> bool {
self.mutation_started
}
@@ -706,6 +693,12 @@ pub struct ObjectOptions {
pub lifecycle_delete_all: Option<LifecycleDeleteAllRequest>,
#[doc(hidden)]
pub lifecycle_delete_all_journal: Option<Arc<parking_lot::Mutex<LifecycleDeleteAllJournalState>>>,
/// Whole-operation authorization created only by consuming a validated
/// v6 dispatch-manifest permit. Clones share the authorization, not the
/// one-shot permit itself.
#[doc(hidden)]
pub tier_delete_dispatch_authorization:
Option<crate::bucket::lifecycle::tier_delete_journal::TierDeleteDispatchAuthorization>,
/// RustFS-only compare-and-set condition checked under the object write lock.
pub expected_current_version_id: Option<String>,
/// Persisted bucket incarnation observed before authorization.
@@ -847,6 +840,7 @@ impl std::fmt::Debug for ObjectOptions {
.field("version_id", &self.version_id.is_some())
.field("lifecycle_delete_all", &self.lifecycle_delete_all.is_some())
.field("lifecycle_delete_all_journal", &self.lifecycle_delete_all_journal.is_some())
.field("tier_delete_dispatch_authorization", &self.tier_delete_dispatch_authorization.is_some())
.field("expected_current_version_id", &self.expected_current_version_id.is_some())
.field("expected_bucket_incarnation_id", &self.expected_bucket_incarnation_id)
.field("no_lock", &self.no_lock)
@@ -972,7 +966,7 @@ impl ObjectOptions {
}
#[cfg(test)]
pub(crate) fn add_namespace_lock_fence_for_test(&mut self, fence: &NamespaceLockFence) {
pub(crate) fn add_namespace_lock_fence(&mut self, fence: &NamespaceLockFence) {
self.namespace_lock_fence
.get_or_insert_with(NamespaceLockFence::new)
.extend(fence);
+581 -74
View File
@@ -29,10 +29,14 @@ use rustfs_madmin::metrics::RealtimeMetrics;
use rustfs_madmin::net::NetInfo;
use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
use rustfs_utils::XHost;
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
use std::future::Future;
use std::hash::{Hash, Hasher};
use std::sync::{Arc, Mutex, OnceLock};
use std::sync::{
Arc, Mutex, OnceLock,
atomic::{AtomicBool, AtomicUsize, Ordering},
};
use std::time::{Duration, Instant, SystemTime};
use tokio::time::{sleep, timeout};
use tokio_util::sync::CancellationToken;
@@ -52,6 +56,20 @@ const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
const TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION: u32 = 3;
type CrossPoolFencePolicyResult = Result<BTreeMap<String, Uuid>>;
fn cross_pool_fence_policy_results(
peer_epochs: BTreeMap<String, Uuid>,
minimum_version: u32,
) -> (CrossPoolFencePolicyResult, CrossPoolFencePolicyResult) {
let journal_result = if minimum_version >= TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION {
Ok(peer_epochs.clone())
} else {
Err(Error::other("tier delete journal v6 policy capability version is unsupported"))
};
(Ok(peer_epochs), journal_result)
}
#[derive(Clone, Debug)]
pub struct ScannerPublicationLeaseGrant {
@@ -107,15 +125,91 @@ struct FleetCapabilityProof {
topology_fingerprint: String,
peer_epochs: Arc<BTreeMap<String, Uuid>>,
expires_at: Instant,
generation: Arc<FleetCapabilityProofGeneration>,
}
impl FleetCapabilityProof {
fn new(topology_fingerprint: String, peer_epochs: Arc<BTreeMap<String, Uuid>>, expires_at: Instant) -> Self {
Self {
topology_fingerprint,
peer_epochs,
expires_at,
generation: FleetCapabilityProofGeneration::fresh(),
}
}
fn token(&self) -> FleetCapabilityProofToken {
FleetCapabilityProofToken {
topology_fingerprint: self.topology_fingerprint.clone(),
peer_epochs: self.peer_epochs.clone(),
}
}
#[cfg(any(test, feature = "test-util"))]
fn with_fresh_generation(&self) -> Self {
Self::new(self.topology_fingerprint.clone(), Arc::clone(&self.peer_epochs), self.expires_at)
}
}
/// Admission generation for effects that must not straddle a fleet-proof
/// replacement. Revocation is deliberately non-blocking: it closes admission
/// immediately, while the proof slot withholds the successor generation until
/// every admitted operation has drained.
#[derive(Default)]
struct FleetCapabilityProofGeneration {
accepting: AtomicBool,
active: AtomicUsize,
}
impl FleetCapabilityProofGeneration {
fn fresh() -> Arc<Self> {
Arc::new(Self {
accepting: AtomicBool::new(true),
active: AtomicUsize::new(0),
})
}
fn try_acquire(self: &Arc<Self>) -> Option<FleetCapabilityProofPermit> {
if !self.accepting.load(Ordering::Acquire) {
return None;
}
self.active.fetch_add(1, Ordering::AcqRel);
if self.accepting.load(Ordering::Acquire) {
Some(FleetCapabilityProofPermit {
generation: Arc::clone(self),
})
} else {
self.release();
None
}
}
fn revoke(&self) {
self.accepting.store(false, Ordering::Release);
}
fn is_accepting(&self) -> bool {
self.accepting.load(Ordering::Acquire)
}
fn is_drained(&self) -> bool {
self.active.load(Ordering::Acquire) == 0
}
fn release(&self) {
let previous = self.active.fetch_sub(1, Ordering::AcqRel);
debug_assert!(previous > 0, "fleet capability permit count underflow");
}
}
struct FleetCapabilityProofPermit {
generation: Arc<FleetCapabilityProofGeneration>,
}
impl Drop for FleetCapabilityProofPermit {
fn drop(&mut self) {
self.generation.release();
}
}
#[derive(Clone, PartialEq, Eq)]
@@ -127,6 +221,7 @@ struct FleetCapabilityProofToken {
#[derive(Default)]
struct FleetCapabilityProofState {
proof: Option<FleetCapabilityProof>,
draining_generation: Option<Arc<FleetCapabilityProofGeneration>>,
topology_conflict: bool,
}
@@ -136,8 +231,17 @@ pub(crate) struct RemoteVersionStateFleetProofToken(FleetCapabilityProofToken);
#[derive(Clone, PartialEq, Eq)]
pub struct CrossPoolFenceFleetProofToken(FleetCapabilityProofToken);
/// A point-in-time proof that every current storage member implements the v6
/// dispatch-manifest policy. It intentionally has no `Clone` implementation:
/// one acquisition authorizes one manifest construction attempt.
pub(crate) struct TierDeleteJournalFleetProofToken {
token: FleetCapabilityProofToken,
_permit: FleetCapabilityProofPermit,
}
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
static TIER_DELETE_JOURNAL_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
@@ -148,8 +252,35 @@ fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCa
REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
}
fn replace_fleet_capability_proof(slot: &std::sync::RwLock<FleetCapabilityProofState>, proof: Option<FleetCapabilityProof>) {
slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
fn tier_delete_journal_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
TIER_DELETE_JOURNAL_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
}
fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) {
if let Some(proof) = state.proof.take() {
proof.generation.revoke();
if !proof.generation.is_drained() {
state.draining_generation = Some(proof.generation);
}
}
if state
.draining_generation
.as_ref()
.is_some_and(|generation| generation.is_drained())
{
state.draining_generation = None;
}
}
fn revoke_fleet_capability_proof(slot: &std::sync::RwLock<FleetCapabilityProofState>) {
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
revoke_fleet_capability_proof_state(&mut state);
}
fn mark_fleet_capability_topology_conflict(slot: &std::sync::RwLock<FleetCapabilityProofState>) {
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
state.topology_conflict = true;
revoke_fleet_capability_proof_state(&mut state);
}
fn publish_fleet_capability_probe_result(
@@ -161,21 +292,42 @@ fn publish_fleet_capability_probe_result(
match result {
Ok(peer_epochs) => {
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
let peer_epochs = state
if let Some(current) = state
.proof
.as_ref()
.as_mut()
.filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
.map(|proof| Arc::clone(&proof.peer_epochs))
.unwrap_or_else(|| Arc::new(peer_epochs));
state.proof = Some(FleetCapabilityProof {
topology_fingerprint: topology_fingerprint.to_string(),
peer_epochs,
expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
});
{
current.expires_at = observed_at + REMOTE_VERSION_STATE_PROOF_TTL;
return None;
}
if let Some(previous) = state.proof.take() {
previous.generation.revoke();
if !previous.generation.is_drained() {
state.draining_generation = Some(previous.generation);
}
}
if state
.draining_generation
.as_ref()
.is_some_and(|generation| generation.is_drained())
{
state.draining_generation = None;
}
if state.draining_generation.is_some() {
return Some(Error::other(
"fleet capability proof successor waits for the previous generation to drain",
));
}
state.proof = Some(FleetCapabilityProof::new(
topology_fingerprint.to_string(),
Arc::new(peer_epochs),
observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
));
None
}
Err(err) => {
replace_fleet_capability_proof(slot, None);
revoke_fleet_capability_proof(slot);
Some(err)
}
}
@@ -216,7 +368,72 @@ pub fn cross_pool_fence_fleet_proof_matches(proof: &CrossPoolFenceFleetProofToke
fleet_capability_proof_matches(cross_pool_fence_fleet_proof_slot(), &proof.0)
}
#[cfg(test)]
pub(crate) fn acquire_tier_delete_journal_fleet_proof() -> Option<TierDeleteJournalFleetProofToken> {
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
let state = tier_delete_journal_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
acquire_tier_delete_journal_fleet_proof_from(&state, expected_topology, Instant::now())
}
fn acquire_tier_delete_journal_fleet_proof_from(
state: &FleetCapabilityProofState,
expected_topology: &str,
now: Instant,
) -> Option<TierDeleteJournalFleetProofToken> {
let token = acquire_fleet_capability_proof_from(state, expected_topology, now)?;
let permit = state.proof.as_ref()?.generation.try_acquire()?;
Some(TierDeleteJournalFleetProofToken { token, _permit: permit })
}
pub(crate) fn tier_delete_journal_fleet_proof_matches(proof: &TierDeleteJournalFleetProofToken) -> bool {
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
return false;
};
let state = tier_delete_journal_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
tier_delete_journal_fleet_proof_matches_at(&state, proof, expected_topology, Instant::now())
}
fn tier_delete_journal_fleet_proof_matches_at(
state: &FleetCapabilityProofState,
proof: &TierDeleteJournalFleetProofToken,
expected_topology: &str,
now: Instant,
) -> bool {
proof._permit.generation.is_accepting()
&& fleet_capability_proof_matches_at(state, &proof.token, expected_topology, now)
&& state
.proof
.as_ref()
.is_some_and(|current| Arc::ptr_eq(&current.generation, &proof._permit.generation))
}
pub(crate) fn tier_delete_journal_topology_generation(proof: &TierDeleteJournalFleetProofToken) -> String {
stable_tier_delete_journal_topology_generation(&proof.token.topology_fingerprint)
}
#[cfg(all(test, feature = "test-util"))]
pub(crate) fn tier_delete_journal_fleet_proof_has_inflight_for_test() -> bool {
let state = tier_delete_journal_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.proof.as_ref().is_some_and(|proof| !proof.generation.is_drained())
|| state
.draining_generation
.as_ref()
.is_some_and(|generation| !generation.is_drained())
}
fn stable_tier_delete_journal_topology_generation(topology_fingerprint: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(b"rustfs-tier-delete-journal-topology-v1\0");
hasher.update(topology_fingerprint.as_bytes());
rustfs_utils::crypto::hex(hasher.finalize().as_slice())
}
#[cfg(any(test, feature = "test-util"))]
pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
let topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY
.get()
@@ -226,18 +443,39 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
let mut state = cross_pool_fence_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = Instant::now();
let proof = if !state.topology_conflict && fleet_capability_proof_valid_at(state.proof.as_ref(), &topology, now) {
state.proof.clone()
} else {
Some(FleetCapabilityProof::new(
topology,
Arc::new(BTreeMap::new()),
now + Duration::from_secs(60 * 60),
))
};
state.topology_conflict = false;
state.proof = Some(FleetCapabilityProof {
topology_fingerprint: topology,
peer_epochs: Arc::new(BTreeMap::new()),
expires_at: Instant::now() + Duration::from_secs(60 * 60),
});
state.proof = proof.clone();
drop(state);
let mut journal_state = tier_delete_journal_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
debug_assert!(
journal_state
.proof
.as_ref()
.is_none_or(|current| current.generation.is_drained())
);
journal_state.topology_conflict = false;
journal_state.draining_generation = None;
journal_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation);
}
#[cfg(test)]
pub(crate) struct CrossPoolFenceFleetProofGuard {
previous_proof: Option<FleetCapabilityProof>,
previous_topology_conflict: bool,
previous_journal_proof: Option<FleetCapabilityProof>,
previous_journal_topology_conflict: bool,
}
#[cfg(test)]
@@ -246,8 +484,24 @@ impl Drop for CrossPoolFenceFleetProofGuard {
let mut state = cross_pool_fence_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.proof = self.previous_proof.take();
state.proof = self
.previous_proof
.take()
.as_ref()
.map(FleetCapabilityProof::with_fresh_generation);
state.draining_generation = None;
state.topology_conflict = self.previous_topology_conflict;
drop(state);
let mut journal_state = tier_delete_journal_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
journal_state.proof = self
.previous_journal_proof
.take()
.as_ref()
.map(FleetCapabilityProof::with_fresh_generation);
journal_state.draining_generation = None;
journal_state.topology_conflict = self.previous_journal_topology_conflict;
}
}
@@ -258,12 +512,29 @@ pub(crate) fn without_cross_pool_fence_fleet_proof_for_test() -> CrossPoolFenceF
let mut state = cross_pool_fence_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let mut journal_state = tier_delete_journal_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let guard = CrossPoolFenceFleetProofGuard {
previous_proof: state.proof.clone(),
previous_topology_conflict: state.topology_conflict,
previous_journal_proof: journal_state.proof.clone(),
previous_journal_topology_conflict: journal_state.topology_conflict,
};
state.proof = None;
if let Some(proof) = state.proof.take() {
proof.generation.revoke();
if !proof.generation.is_drained() {
state.draining_generation = Some(proof.generation);
}
}
state.topology_conflict = true;
if let Some(proof) = journal_state.proof.take() {
proof.generation.revoke();
if !proof.generation.is_drained() {
journal_state.draining_generation = Some(proof.generation);
}
}
journal_state.topology_conflict = true;
guard
}
@@ -275,11 +546,33 @@ pub fn rotate_cross_pool_fence_fleet_proof_for_test() -> bool {
let Some(current) = state.proof.as_ref() else {
return false;
};
state.proof = Some(FleetCapabilityProof {
topology_fingerprint: current.topology_fingerprint.clone(),
peer_epochs: Arc::new(current.peer_epochs.as_ref().clone()),
expires_at: current.expires_at,
});
let proof = FleetCapabilityProof::new(
current.topology_fingerprint.clone(),
Arc::new(current.peer_epochs.as_ref().clone()),
current.expires_at,
);
state.proof = Some(proof.clone());
drop(state);
let mut journal_state = tier_delete_journal_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
journal_state.topology_conflict = false;
if let Some(previous) = journal_state.proof.take() {
previous.generation.revoke();
if !previous.generation.is_drained() {
journal_state.draining_generation = Some(previous.generation);
}
}
if journal_state
.draining_generation
.as_ref()
.is_some_and(|generation| generation.is_drained())
{
journal_state.draining_generation = None;
}
if journal_state.draining_generation.is_none() {
journal_state.proof = Some(proof.with_fresh_generation());
}
true
}
@@ -291,15 +584,22 @@ fn fleet_capability_proof_matches(
return false;
};
let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner);
if state.topology_conflict {
return false;
}
state.proof.as_ref().is_some_and(|current| {
current.topology_fingerprint == *expected_topology
&& current.topology_fingerprint == proof.topology_fingerprint
&& Arc::ptr_eq(&current.peer_epochs, &proof.peer_epochs)
&& Instant::now() < current.expires_at
})
fleet_capability_proof_matches_at(&state, proof, expected_topology, Instant::now())
}
fn fleet_capability_proof_matches_at(
state: &FleetCapabilityProofState,
proof: &FleetCapabilityProofToken,
expected_topology: &str,
now: Instant,
) -> bool {
!state.topology_conflict
&& state.proof.as_ref().is_some_and(|current| {
current.topology_fingerprint == expected_topology
&& current.topology_fingerprint == proof.topology_fingerprint
&& Arc::ptr_eq(&current.peer_epochs, &proof.peer_epochs)
&& now < current.expires_at
})
}
fn fleet_capability_proof_valid_at(proof: Option<&FleetCapabilityProof>, expected_topology: &str, now: Instant) -> bool {
@@ -312,7 +612,7 @@ pub(crate) struct RemoteVersionStateFleetProofGuard;
#[cfg(test)]
impl Drop for RemoteVersionStateFleetProofGuard {
fn drop(&mut self) {
replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None);
revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot());
}
}
@@ -348,10 +648,12 @@ fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, pe
pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
for slot in [remote_version_state_fleet_proof_slot(), cross_pool_fence_fleet_proof_slot()] {
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
state.topology_conflict = true;
state.proof = None;
for slot in [
remote_version_state_fleet_proof_slot(),
cross_pool_fence_fleet_proof_slot(),
tier_delete_journal_fleet_proof_slot(),
] {
mark_fleet_capability_topology_conflict(slot);
}
}
return;
@@ -373,7 +675,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
}
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
};
let fence_result = match get_global_notification_sys() {
let fence_probe = match get_global_notification_sys() {
Some(notification_sys) => timeout(
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
@@ -382,13 +684,21 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
};
let (fence_result, journal_result) = match fence_probe {
Ok((peer_epochs, minimum_version)) => cross_pool_fence_policy_results(peer_epochs, minimum_version),
Err(err) => {
let message = err.to_string();
(Err(Error::other(message.clone())), Err(Error::other(message)))
}
};
let topology_conflict = remote_version_state_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.topology_conflict;
if topology_conflict {
replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None);
replace_fleet_capability_proof(cross_pool_fence_fleet_proof_slot(), None);
revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot());
revoke_fleet_capability_proof(cross_pool_fence_fleet_proof_slot());
revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot());
} else if let Some(err) = publish_fleet_capability_probe_result(
remote_version_state_fleet_proof_slot(),
&topology_fingerprint,
@@ -409,7 +719,25 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
capability = "cross_pool_fence_v2",
capability = "cross_pool_fence",
state = "failed_closed",
error = %err,
"notification capability probe"
);
}
if !topology_conflict
&& let Some(err) = publish_fleet_capability_probe_result(
tier_delete_journal_fleet_proof_slot(),
&topology_fingerprint,
journal_result,
Instant::now(),
)
{
debug!(
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
capability = "tier_delete_journal_v6_policy",
state = "failed_closed",
error = %err,
"notification capability probe"
@@ -483,7 +811,7 @@ impl NotificationSys {
Ok(peer_epochs)
}
async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result<(BTreeMap<String, Uuid>, u32)> {
if self.peer_clients.len() != self.peer_topology_hosts.len() {
return Err(Error::other("cross-pool fence capability fleet membership is incomplete"));
}
@@ -494,14 +822,21 @@ impl NotificationSys {
client.probe_cross_pool_fence(topology_fingerprint.to_string()).await
});
let mut peer_epochs = BTreeMap::new();
let mut minimum_version = u32::MAX;
for result in join_all(probes).await {
let (peer, version, epoch) = result?;
if version < CROSS_POOL_FENCE_SUPPORTED_VERSION {
return Err(Error::other("cross-pool fence capability version is unsupported"));
}
minimum_version = minimum_version.min(version);
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
}
Ok(peer_epochs)
// A single-node deployment has no remote member to lower the local
// policy version advertised by this binary.
if minimum_version == u32::MAX {
minimum_version = TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION;
}
Ok((peer_epochs, minimum_version))
}
}
@@ -1827,12 +2162,13 @@ impl NotificationSys {
join_all(futures).await
}
pub async fn abort_tier_mutation(&self, mutation_id: Uuid) -> Vec<NotificationPeerErr> {
pub async fn abort_tier_mutation(&self, mutation_id: Uuid, canonical_prepare_payload: Bytes) -> Vec<NotificationPeerErr> {
let mut futures = Vec::with_capacity(self.peer_clients.len());
for client in self.peer_clients.iter().cloned() {
let payload = canonical_prepare_payload.clone();
futures.push(async move {
if let Some(client) = client {
notification_peer_result(client.host.to_string(), client.abort_tier_mutation(mutation_id).await)
notification_peer_result(client.host.to_string(), client.abort_tier_mutation(mutation_id, payload).await)
} else {
unreachable_notification_peer_err()
}
@@ -2467,16 +2803,24 @@ fn aggregate_scanner_dirty_usage_acknowledgement_results(
mod tests {
use super::*;
#[test]
fn cross_pool_v2_remains_generic_but_cannot_authorize_v6_journal() {
let peers = BTreeMap::from([("node-b:9000".to_string(), Uuid::new_v4())]);
let (generic_v2, journal_v2) = cross_pool_fence_policy_results(peers.clone(), 2);
assert!(generic_v2.is_ok(), "v2 remains valid for existing cross-pool fencing");
assert!(journal_v2.is_err(), "a mixed v2/v3 fleet must fail closed for journal-v6 deletion");
let (generic_v3, journal_v3) = cross_pool_fence_policy_results(peers, 3);
assert!(generic_v3.is_ok());
assert!(journal_v3.is_ok(), "an all-v3 fleet may authorize journal-v6 deletion");
}
#[test]
fn remote_version_state_fleet_proof_rejects_stale_or_mismatched_membership() {
let now = Instant::now();
let mut peer_epochs = BTreeMap::new();
peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
let proof = FleetCapabilityProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(peer_epochs),
expires_at: now + Duration::from_secs(1),
};
let proof = FleetCapabilityProof::new("topology-a".to_string(), Arc::new(peer_epochs), now + Duration::from_secs(1));
assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now));
assert!(!fleet_capability_proof_valid_at(Some(&proof), "topology-b", now));
@@ -2495,11 +2839,7 @@ mod tests {
#[test]
fn remote_version_state_fleet_proof_accepts_single_node_membership() {
let now = Instant::now();
let proof = FleetCapabilityProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(BTreeMap::new()),
expires_at: now + Duration::from_secs(1),
};
let proof = FleetCapabilityProof::new("topology-a".to_string(), Arc::new(BTreeMap::new()), now + Duration::from_secs(1));
assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now));
}
@@ -2507,21 +2847,97 @@ mod tests {
#[test]
fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
let now = Instant::now();
let proof = FleetCapabilityProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
expires_at: now + Duration::from_secs(1),
};
let proof = FleetCapabilityProof::new(
"topology-a".to_string(),
Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
now + Duration::from_secs(1),
);
let captured = proof.token();
let restarted = FleetCapabilityProof {
topology_fingerprint: proof.topology_fingerprint.clone(),
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
expires_at: proof.expires_at,
};
let restarted = FleetCapabilityProof::new(
proof.topology_fingerprint.clone(),
Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
proof.expires_at,
);
assert!(captured != restarted.token());
}
#[test]
fn tier_delete_journal_generation_is_stable_across_members_and_process_restarts() {
let topology = "topology-a";
let now = Instant::now();
let node_a_view = FleetCapabilityProof::new(
topology.to_string(),
Arc::new(BTreeMap::from([("node-b".to_string(), Uuid::new_v4())])),
now + Duration::from_secs(1),
);
let node_b_view = FleetCapabilityProof::new(
topology.to_string(),
Arc::new(BTreeMap::from([("node-a".to_string(), Uuid::new_v4())])),
now + Duration::from_secs(1),
);
let restarted_node_a_view = FleetCapabilityProof::new(
topology.to_string(),
Arc::new(BTreeMap::from([("node-b".to_string(), Uuid::new_v4())])),
now + Duration::from_secs(1),
);
let generations = [&node_a_view, &node_b_view, &restarted_node_a_view]
.map(|proof| stable_tier_delete_journal_topology_generation(&proof.token().topology_fingerprint));
assert_eq!(generations[0], generations[1]);
assert_eq!(generations[0], generations[2]);
assert_ne!(
generations[0],
stable_tier_delete_journal_topology_generation("topology-b"),
"a real topology change must produce a different durable generation"
);
}
#[test]
fn tier_delete_journal_restart_revokes_old_token_but_fresh_token_recovers_same_generation() {
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
let now = Instant::now();
let original_peers = BTreeMap::from([("node-b".to_string(), Uuid::new_v4())]);
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original_peers), now).is_none());
let original = slot
.read()
.expect("proof slot should not poison")
.proof
.as_ref()
.expect("successful probe should publish proof")
.token();
let original_generation = stable_tier_delete_journal_topology_generation(&original.topology_fingerprint);
let restarted_peers = BTreeMap::from([("node-b".to_string(), Uuid::new_v4())]);
assert!(
publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted_peers), now + Duration::from_millis(1))
.is_none()
);
let state = slot.read().expect("proof slot should not poison");
let fresh = state
.proof
.as_ref()
.expect("restart probe should publish a fresh proof")
.token();
assert!(!fleet_capability_proof_matches_at(
&state,
&original,
"topology-a",
now + Duration::from_millis(2)
));
assert!(fleet_capability_proof_matches_at(
&state,
&fresh,
"topology-a",
now + Duration::from_millis(2)
));
assert_eq!(
original_generation,
stable_tier_delete_journal_topology_generation(&fresh.topology_fingerprint)
);
}
#[test]
fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
@@ -2561,15 +2977,106 @@ mod tests {
assert!(!Arc::ptr_eq(&original.peer_epochs, &replaced.peer_epochs));
}
#[test]
fn tier_delete_journal_successor_waits_for_inflight_generation_to_drain() {
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
let now = Instant::now();
let original_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original_peers), now).is_none());
let admitted = {
let state = slot.read().expect("proof slot should not poison");
acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now)
.expect("a fresh proof should admit one journal operation")
};
{
let state = slot.read().expect("proof slot should not poison");
assert!(
tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now),
"a freshly admitted journal proof must remain current"
);
assert!(
acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now + REMOTE_VERSION_STATE_PROOF_TTL,)
.is_none(),
"TTL expiry must stop new admission"
);
assert!(
!tier_delete_journal_fleet_proof_matches_at(
&state,
&admitted,
"topology-a",
now + REMOTE_VERSION_STATE_PROOF_TTL,
),
"TTL expiry must also stop an admitted proof at its next durable fence"
);
assert!(!admitted._permit.generation.is_drained());
}
let restarted_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
let blocked = publish_fleet_capability_probe_result(
&slot,
"topology-a",
Ok(restarted_peers.clone()),
now + Duration::from_millis(1),
)
.expect("a successor proof must wait for the admitted generation");
assert!(blocked.to_string().contains("previous generation to drain"));
{
let state = slot.read().expect("proof slot should not poison");
assert!(state.proof.is_none(), "new operations must remain closed while the predecessor drains");
assert!(state.draining_generation.is_some());
assert!(
!tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now + Duration::from_millis(1),),
"a restarted peer must revoke an admitted proof before its next durable fence"
);
}
drop(admitted);
assert!(
publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted_peers), now + Duration::from_millis(2),)
.is_none(),
"the successor may publish after the in-flight operation releases its permit"
);
let state = slot.read().expect("proof slot should not poison");
assert!(state.proof.is_some());
assert!(state.draining_generation.is_none());
}
#[test]
fn tier_delete_journal_topology_conflict_revokes_admitted_generation() {
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
let now = Instant::now();
let peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now).is_none());
let admitted = {
let state = slot.read().expect("proof slot should not poison");
acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now)
.expect("a fresh proof should admit one journal operation")
};
mark_fleet_capability_topology_conflict(&slot);
let state = slot.read().expect("proof slot should not poison");
assert!(state.topology_conflict);
assert!(state.proof.is_none());
assert!(state.draining_generation.is_some());
assert!(!admitted._permit.generation.is_accepting());
assert!(
!tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now),
"topology conflict must revoke an already admitted journal proof"
);
}
#[test]
fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
let now = Instant::now();
let mut state = FleetCapabilityProofState {
proof: Some(FleetCapabilityProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(BTreeMap::new()),
expires_at: now + Duration::from_secs(1),
}),
proof: Some(FleetCapabilityProof::new(
"topology-a".to_string(),
Arc::new(BTreeMap::new()),
now + Duration::from_secs(1),
)),
draining_generation: None,
topology_conflict: false,
};
assert!(acquire_fleet_capability_proof_from(&state, "topology-a", now).is_some());
@@ -3279,7 +3786,7 @@ mod tests {
assert_eq!(commit.len(), 1);
assert!(commit[0].err.is_some());
let abort = sys.abort_tier_mutation(mutation_id).await;
let abort = sys.abort_tier_mutation(mutation_id, Bytes::from_static(b"prepare")).await;
assert_eq!(abort.len(), 1);
assert!(abort[0].err.is_some());
}
@@ -356,7 +356,7 @@ impl ECStore {
};
run_guard.ensure_held("rebalance version migration")?;
let result = migrate_entry_version(
&RebalanceMigrationBackend::new(set.as_ref(), self.as_ref(), lock_lost_signal.clone()),
&RebalanceMigrationBackend::new(set.as_ref(), self.clone(), lock_lost_signal.clone()),
bucket.clone(),
pool_index,
version,
@@ -1478,6 +1478,7 @@ mod tests {
let (_temp_dirs, store, _unused_store) =
crate::services::rebalance::test_two_pool_stores(Some(active_rebalance_meta(REBALANCE_ID))).await;
prepare_rebalance_test_volumes(store.as_ref()).await;
crate::services::tier::test_util::register_mock_tier(&store.tier_config_mgr(), "WARM").await;
let source_set = store.pools[0].get_disks_by_key(object);
let target_set = store.pools[1].get_disks_by_key(object);
let version_id = uuid::Uuid::new_v4();
@@ -1520,14 +1521,17 @@ mod tests {
let entry = metacache_entry_from_source(source_set.as_ref(), bucket, object).await;
let run_signal_fence = RebalanceRunSignalTestFence::install(REBALANCE_ID);
let barrier = TieredMetadataCommitBarrier::install(bucket, object);
let task = spawn_real_rebalance_entry(
let mut task = spawn_real_rebalance_entry(
Arc::clone(&store),
Arc::clone(&source_set),
entry,
REBALANCE_ID,
Arc::new(RebalanceBucketConfigs::default()),
);
barrier.wait_until_paused().await;
tokio::select! {
_ = barrier.wait_until_paused() => {}
result = &mut task => panic!("rebalance exited before the tiered commit barrier: {result:?}"),
}
run_signal_fence.mark_lost();
barrier.release();
drop(barrier);
@@ -101,14 +101,14 @@ pub(crate) trait MigrationBackend: Send + Sync {
pub(crate) struct RebalanceMigrationBackend<'a> {
source: &'a SetDisks,
store: &'a ECStore,
store: std::sync::Arc<ECStore>,
lock_lost_signal: Option<std::sync::Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
}
impl<'a> RebalanceMigrationBackend<'a> {
pub(crate) fn new(
source: &'a SetDisks,
store: &'a ECStore,
store: std::sync::Arc<ECStore>,
lock_lost_signal: Option<std::sync::Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
) -> Self {
Self {
+1 -1
View File
@@ -134,7 +134,7 @@ pub(crate) async fn test_three_pool_stores_with_isolated_node_contexts(
test_pool_stores_with_contexts(rebalance_meta, true, 3, 2).await
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pub(crate) async fn test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(
rebalance_meta: Option<RebalanceMeta>,
) -> (
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(feature = "test-util")]
#[cfg(any(test, feature = "test-util"))]
pub mod test_util;
pub mod tier;
pub mod tier_admin;
File diff suppressed because it is too large Load Diff
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use std::sync::{Arc, LazyLock};
use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
use serde::{Deserialize, Serialize};
@@ -32,9 +32,34 @@ pub(crate) const TIER_MUTATION_INTENT_SCHEMA: &str = "rustfs-tier-mutation-inten
pub(crate) const MAX_TIER_MUTATION_INTENT_SIZE: usize = rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE;
pub(crate) const TIER_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/records";
pub(crate) const TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/coordinators";
pub(crate) const TIER_MUTATION_MUTEX_SHARDS: usize = 64;
const TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS: usize = 3;
pub(crate) type TierMutationDigest = [u8; 32];
static TIER_MUTATION_MUTEXES: LazyLock<[tokio::sync::Mutex<()>; TIER_MUTATION_MUTEX_SHARDS]> =
LazyLock::new(|| std::array::from_fn(|_| tokio::sync::Mutex::new(())));
/// Serializes every local phase and recovery action for one mutation id while
/// retaining bounded parallelism for unrelated mutations.
pub(crate) async fn acquire_tier_mutation_mutex(mutation_id: Uuid) -> tokio::sync::MutexGuard<'static, ()> {
TIER_MUTATION_MUTEXES[tier_mutation_mutex_shard_index(mutation_id)]
.lock()
.await
}
fn tier_mutation_mutex_shard_index(mutation_id: Uuid) -> usize {
let raw = mutation_id.as_u128();
let mut mixed = (raw as u64) ^ ((raw >> 64) as u64);
// MurmurHash3's 64-bit finalizer gives stable diffusion without allocating
// or relying on RandomState, whose seed differs between processes.
mixed ^= mixed >> 33;
mixed = mixed.wrapping_mul(0xff51_afd7_ed55_8ccd);
mixed ^= mixed >> 33;
mixed = mixed.wrapping_mul(0xc4ce_b9fe_1a85_ec53);
mixed ^= mixed >> 33;
(mixed as usize) & (TIER_MUTATION_MUTEX_SHARDS - 1)
}
pub(crate) type Result<T> = std::result::Result<T, TierMutationIntentError>;
#[derive(Debug, thiserror::Error)]
@@ -265,6 +290,30 @@ impl TierMutationIntent {
&& self.expires_at_unix_nanos == other.expires_at_unix_nanos
}
/// Reconstruct the exact Prepared record that originally produced this
/// intent. Abort RPCs are identity-bound to that payload; serializing an
/// Aborted terminal record would both violate the wire contract and use a
/// different revision if a missing peer has to persist a tombstone.
pub(crate) fn original_prepared(&self) -> Result<Self> {
if self.state == TierMutationIntentState::Prepared {
self.validate()?;
return Ok(self.clone());
}
let mut prepared = self.clone();
prepared.revision =
prepared
.revision
.checked_sub(1)
.filter(|revision| *revision != 0)
.ok_or(TierMutationIntentError::Corrupt(
"terminal intent cannot reconstruct its prepared revision",
))?;
prepared.state = TierMutationIntentState::Prepared;
prepared.committed_config_etag = None;
prepared.validate()?;
Ok(prepared)
}
pub(crate) fn encode(&self) -> Result<Vec<u8>> {
self.validate()?;
let intent_bytes = serde_json::to_vec(self)?;
@@ -439,6 +488,16 @@ where
load_tier_mutation_intent_record_with_etag_at_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
}
pub(crate) async fn load_tier_coordinator_mutation_intent_record_with_etag<S>(
api: Arc<S>,
mutation_id: Uuid,
) -> EcstoreResult<(TierMutationIntent, String)>
where
S: EcstoreObjectIO,
{
load_tier_mutation_intent_record_with_etag_at_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
}
async fn load_tier_mutation_intent_record_with_etag_at_prefix<S>(
api: Arc<S>,
prefix: &str,
@@ -507,6 +566,7 @@ where
.await
}
#[cfg(test)]
pub(crate) async fn delete_tier_mutation_intent_record<S>(api: Arc<S>, mutation_id: Uuid) -> EcstoreResult<()>
where
S: EcstoreObjectOperations,
@@ -514,13 +574,36 @@ where
delete_tier_mutation_intent_record_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
}
pub(crate) async fn delete_tier_coordinator_mutation_intent_record<S>(api: Arc<S>, mutation_id: Uuid) -> EcstoreResult<()>
pub(crate) async fn delete_tier_mutation_intent_record_if_current<S>(
api: Arc<S>,
mutation_id: Uuid,
current_etag: &str,
) -> EcstoreResult<()>
where
S: EcstoreObjectOperations,
{
delete_tier_mutation_intent_record_with_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
delete_tier_mutation_intent_record_if_current_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id, current_etag)
.await
}
pub(crate) async fn delete_tier_coordinator_mutation_intent_record_if_current<S>(
api: Arc<S>,
mutation_id: Uuid,
current_etag: &str,
) -> EcstoreResult<()>
where
S: EcstoreObjectOperations,
{
delete_tier_mutation_intent_record_if_current_with_prefix(
api,
TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX,
mutation_id,
current_etag,
)
.await
}
#[cfg(test)]
async fn delete_tier_mutation_intent_record_with_prefix<S>(api: Arc<S>, prefix: &str, mutation_id: Uuid) -> EcstoreResult<()>
where
S: EcstoreObjectOperations,
@@ -533,6 +616,40 @@ where
}
}
async fn delete_tier_mutation_intent_record_if_current_with_prefix<S>(
api: Arc<S>,
prefix: &str,
mutation_id: Uuid,
current_etag: &str,
) -> EcstoreResult<()>
where
S: EcstoreObjectOperations,
{
if current_etag.trim().is_empty() {
return Err(Error::other("tier mutation intent current ETag is empty"));
}
let object =
tier_mutation_intent_record_object_name_with_prefix(prefix, mutation_id).map_err(tier_mutation_intent_store_error)?;
match api
.delete_object(
RUSTFS_META_BUCKET,
&object,
ObjectOptions {
http_preconditions: Some(HTTPPreconditions {
if_match: Some(current_etag.to_string()),
..Default::default()
}),
..Default::default()
},
)
.await
{
Ok(_) => Ok(()),
Err(err) if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) => Err(Error::ConfigNotFound),
Err(err) => Err(err),
}
}
pub(crate) async fn advance_tier_mutation_intent_record_idempotent<S>(
api: Arc<S>,
mutation_id: Uuid,
@@ -713,6 +830,7 @@ fn digest_is_empty(digest: &TierMutationDigest) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use std::time::Duration;
const OLD_IDENTITY: TierDestinationId = [1; 32];
const NEW_IDENTITY: TierDestinationId = [2; 32];
@@ -736,6 +854,61 @@ mod tests {
}
}
#[test]
fn mutation_mutex_uses_exactly_64_stable_shards() {
assert_eq!(TIER_MUTATION_MUTEX_SHARDS, 64);
assert_eq!(TIER_MUTATION_MUTEXES.len(), TIER_MUTATION_MUTEX_SHARDS);
let mutation_id = Uuid::parse_str("36e2220e-9ad2-495b-b3bc-c4d2caf70a31").expect("fixture uuid should parse");
let shard = tier_mutation_mutex_shard_index(mutation_id);
assert!(shard < TIER_MUTATION_MUTEX_SHARDS);
assert_eq!(shard, tier_mutation_mutex_shard_index(mutation_id));
}
#[tokio::test]
async fn mutation_mutex_serializes_the_same_id() {
let mutation_id = Uuid::new_v4();
let first = acquire_tier_mutation_mutex(mutation_id).await;
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (acquired_tx, mut acquired_rx) = tokio::sync::oneshot::channel();
let waiter = tokio::spawn(async move {
started_tx.send(()).expect("test receiver should remain alive");
let _second = acquire_tier_mutation_mutex(mutation_id).await;
acquired_tx.send(()).expect("test receiver should remain alive");
});
started_rx.await.expect("waiter should start");
assert!(
tokio::time::timeout(Duration::from_millis(25), &mut acquired_rx)
.await
.is_err(),
"the same mutation id must not enter concurrently"
);
drop(first);
tokio::time::timeout(Duration::from_secs(1), &mut acquired_rx)
.await
.expect("waiter should acquire after release")
.expect("waiter should report acquisition");
waiter.await.expect("waiter task should finish");
}
#[tokio::test]
async fn mutation_mutex_allows_different_shards_to_progress() {
let first_id = Uuid::new_v4();
let first_shard = tier_mutation_mutex_shard_index(first_id);
let second_id = (0..1024)
.map(|_| Uuid::new_v4())
.find(|candidate| tier_mutation_mutex_shard_index(*candidate) != first_shard)
.expect("a distinct shard should be easy to find");
let first = acquire_tier_mutation_mutex(first_id).await;
let _second = tokio::time::timeout(Duration::from_secs(1), acquire_tier_mutation_mutex(second_id))
.await
.expect("a different shard must not wait for the first mutation");
drop(first);
}
#[test]
fn intent_round_trip_preserves_committed_state() {
let mut intent = prepared_intent();
@@ -873,6 +1046,37 @@ mod tests {
));
}
#[test]
fn terminal_intent_reconstructs_original_prepared_abort_payload() {
for terminal in [TierMutationIntentState::Aborted, TierMutationIntentState::Committed] {
let original = prepared_intent();
let mut intent = original.clone();
let committed_etag = (terminal == TierMutationIntentState::Committed).then(|| "new-etag".to_string());
intent
.advance(terminal, committed_etag)
.expect("terminal transition should succeed");
let reconstructed = intent
.original_prepared()
.expect("terminal record should recover prepared payload");
assert_eq!(reconstructed, original);
assert!(intent.same_identity_as(&reconstructed));
}
}
#[test]
fn terminal_intent_with_initial_revision_fails_prepared_reconstruction() {
let mut corrupt = prepared_intent();
corrupt.state = TierMutationIntentState::Aborted;
assert!(matches!(
corrupt.original_prepared(),
Err(TierMutationIntentError::Corrupt(
"terminal intent cannot reconstruct its prepared revision"
))
));
}
#[test]
fn intent_validation_rejects_placeholder_identity() {
let mut intent = prepared_intent();
@@ -15,12 +15,13 @@
use std::sync::Arc;
use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase};
use time::OffsetDateTime;
use uuid::Uuid;
use super::tier::{TierConfigMgr, tier_config_abort_matches, tier_config_commit_matches, tier_config_etag_matches};
use super::tier_mutation_intent::{
MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, advance_tier_mutation_intent_record_idempotent,
load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, acquire_tier_mutation_mutex,
advance_tier_mutation_intent_record_idempotent, load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
};
use crate::error::{Error, StorageError};
use crate::store::ECStore;
@@ -57,6 +58,8 @@ pub enum TierMutationPeerError {
CommitProofMismatch,
#[error("tier mutation peer abort proof does not match the persisted tier configuration")]
AbortProofMismatch,
#[error("tier mutation peer prepared intent has expired")]
ExpiredIntent,
#[error("tier mutation peer runtime error: {0}")]
Runtime(#[source] AdminError),
#[error("tier mutation peer store error: {0}")]
@@ -79,6 +82,7 @@ pub async fn handle_tier_mutation_peer_request(
canonical_payload: &[u8],
) -> TierMutationPeerResult<TierMutationPeerOutcome> {
validate_peer_request_envelope(protocol_version, mutation_id, canonical_payload)?;
let _mutation_guard = acquire_tier_mutation_mutex(mutation_id).await;
match phase {
TierMutationRpcPhase::Prepare => handle_prepare(api, mutation_id, canonical_payload).await,
TierMutationRpcPhase::Commit => handle_commit(api, mutation_id, canonical_payload).await,
@@ -103,43 +107,57 @@ async fn handle_prepare(
}
let tier_config_mgr = api.tier_config_mgr();
match save_tier_mutation_intent_record_if_absent(api.clone(), &intent).await {
Ok(()) => {
TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &intent)
.await
.map_err(TierMutationPeerError::Runtime)?;
Ok(TierMutationPeerOutcome {
state: TierMutationPeerState::Prepared,
applied: true,
})
}
Err(Error::PreconditionFailed) => {
let existing = load_tier_mutation_intent_record(api, mutation_id).await?;
if !existing.same_identity_as(&intent) {
return Err(TierMutationPeerError::ConflictingIntent);
for _ in 0..3 {
let (stored, applied) = match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
Ok(existing) => {
if !existing.same_identity_as(&intent) {
return Err(TierMutationPeerError::ConflictingIntent);
}
(existing, false)
}
match existing.state {
TierMutationIntentState::Prepared => {
TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &existing)
.await
.map_err(TierMutationPeerError::Runtime)?;
Err(Error::ConfigNotFound) => {
let now = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).unwrap_or(i64::MAX);
if intent.expires_at_unix_nanos <= now {
return Err(TierMutationPeerError::ExpiredIntent);
}
TierMutationIntentState::Committed => {
TierConfigMgr::apply_committed_mutation_intent_block(&tier_config_mgr, &existing)
.await
.map_err(TierMutationPeerError::Runtime)?;
}
TierMutationIntentState::Aborted => {
TierConfigMgr::request_committed_mutation_refresh(&tier_config_mgr).await;
match save_tier_mutation_intent_record_if_absent(api.clone(), &intent).await {
Ok(()) => (intent.clone(), true),
Err(Error::PreconditionFailed) => continue,
Err(err) => return Err(err.into()),
}
}
Ok(TierMutationPeerOutcome {
state: peer_state_from_intent(existing.state),
applied: false,
})
Err(err) => return Err(err.into()),
};
match stored.state {
TierMutationIntentState::Prepared => {
TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &stored)
.await
.map_err(TierMutationPeerError::Runtime)?;
TierConfigMgr::wait_for_blocked_tier_operation_leases(&tier_config_mgr, &stored)
.await
.map_err(TierMutationPeerError::Runtime)?;
}
TierMutationIntentState::Committed => {
TierConfigMgr::apply_committed_mutation_intent_block(&tier_config_mgr, &stored)
.await
.map_err(TierMutationPeerError::Runtime)?;
}
TierMutationIntentState::Aborted => {
TierConfigMgr::clear_prepared_mutation_intent_block(&tier_config_mgr, mutation_id)
.await
.map_err(TierMutationPeerError::Runtime)?;
TierConfigMgr::request_committed_mutation_refresh(&tier_config_mgr).await;
}
}
Err(err) => Err(err.into()),
return Ok(TierMutationPeerOutcome {
state: peer_state_from_intent(stored.state),
applied,
});
}
Err(TierMutationPeerError::Store(Error::other(
"tier mutation prepare raced repeatedly with another decision",
)))
}
async fn handle_commit(
@@ -201,26 +219,106 @@ async fn handle_abort(
mutation_id: Uuid,
canonical_payload: &[u8],
) -> TierMutationPeerResult<TierMutationPeerOutcome> {
if !canonical_payload.is_empty() {
return Err(TierMutationPeerError::InvalidPayload("abort payload must be empty".to_string()));
let prepared = TierMutationIntent::decode(mutation_id, canonical_payload)
.map_err(|err| TierMutationPeerError::InvalidPayload(err.to_string()))?;
if prepared.state != TierMutationIntentState::Prepared {
return Err(TierMutationPeerError::InvalidPayload(
"abort payload must carry the original prepared intent".to_string(),
));
}
let existing = load_tier_mutation_intent_record(api.clone(), mutation_id).await?;
if existing.state == TierMutationIntentState::Prepared
&& !tier_config_abort_matches(api.clone(), &existing)
.await
.map_err(Error::other)?
{
return Err(TierMutationPeerError::AbortProofMismatch);
let mut tombstone = prepared.clone();
tombstone
.advance(TierMutationIntentState::Aborted, None)
.map_err(|err| TierMutationPeerError::InvalidPayload(err.to_string()))?;
for _ in 0..3 {
match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
Ok(existing) => {
if !existing.same_identity_as(&prepared) {
return Err(TierMutationPeerError::ConflictingIntent);
}
match existing.state {
TierMutationIntentState::Committed => {
return Ok(TierMutationPeerOutcome {
state: TierMutationPeerState::Committed,
applied: false,
});
}
TierMutationIntentState::Aborted => {
TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id)
.await
.map_err(TierMutationPeerError::Runtime)?;
TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
return Ok(TierMutationPeerOutcome {
state: TierMutationPeerState::Aborted,
applied: false,
});
}
TierMutationIntentState::Prepared => {}
}
if !tier_config_abort_matches(api.clone(), &prepared)
.await
.map_err(Error::other)?
{
return Err(TierMutationPeerError::AbortProofMismatch);
}
let advanced = advance_tier_mutation_intent_record_idempotent(
api.clone(),
mutation_id,
TierMutationIntentState::Aborted,
None,
)
.await;
let (intent, applied) = match advanced {
Ok(result) => result,
Err(err) => match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
Ok(current)
if current.same_identity_as(&prepared) && current.state != TierMutationIntentState::Prepared =>
{
(current, false)
}
_ => return Err(err.into()),
},
};
if intent.state == TierMutationIntentState::Aborted {
TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id)
.await
.map_err(TierMutationPeerError::Runtime)?;
}
return Ok(TierMutationPeerOutcome {
state: peer_state_from_intent(intent.state),
applied,
});
}
Err(Error::ConfigNotFound) => {
if !tier_config_abort_matches(api.clone(), &prepared)
.await
.map_err(Error::other)?
{
return Err(TierMutationPeerError::AbortProofMismatch);
}
match save_tier_mutation_intent_record_if_absent(api.clone(), &tombstone).await {
Ok(()) => {
TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id)
.await
.map_err(TierMutationPeerError::Runtime)?;
TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
return Ok(TierMutationPeerOutcome {
state: TierMutationPeerState::Aborted,
applied: true,
});
}
Err(Error::PreconditionFailed) => continue,
Err(err) => return Err(err.into()),
}
}
Err(err) => return Err(err.into()),
}
}
let (intent, applied) =
advance_tier_mutation_intent_record_idempotent(api.clone(), mutation_id, TierMutationIntentState::Aborted, None).await?;
if intent.state == TierMutationIntentState::Aborted {
TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
}
Ok(TierMutationPeerOutcome {
state: peer_state_from_intent(intent.state),
applied,
})
Err(TierMutationPeerError::Store(Error::other(
"tier mutation abort raced repeatedly with prepare",
)))
}
fn validate_peer_request_envelope(
@@ -228,7 +326,10 @@ fn validate_peer_request_envelope(
mutation_id: Uuid,
canonical_payload: &[u8],
) -> TierMutationPeerResult<()> {
if protocol_version != TIER_MUTATION_RPC_PROTOCOL_VERSION {
if !matches!(
protocol_version,
rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION | TIER_MUTATION_RPC_PROTOCOL_VERSION
) {
return Err(TierMutationPeerError::UnsupportedProtocolVersion(protocol_version));
}
if mutation_id.is_nil() {
@@ -276,6 +377,8 @@ mod tests {
#[test]
fn peer_request_envelope_fails_closed_on_old_version_nil_id_and_large_payload() {
let mutation_id = Uuid::new_v4();
validate_peer_request_envelope(rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION, mutation_id, b"payload")
.expect("v3 must remain accepted during the v4 rollout");
assert!(matches!(
validate_peer_request_envelope(TIER_MUTATION_RPC_PROTOCOL_VERSION + 1, mutation_id, b"payload"),
Err(TierMutationPeerError::UnsupportedProtocolVersion(_))
+4 -2
View File
@@ -859,7 +859,9 @@ static OBJECT_LOCK_DIAG_ENABLED: OnceLock<bool> = OnceLock::new();
mod core;
#[cfg(test)]
pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, disk_call_counters, rename_fanout_barrier};
pub(crate) use core::io_primitives::disk_call_counters;
#[cfg(all(test, feature = "test-util"))]
pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
mod ctx;
mod metadata;
mod ops;
@@ -872,7 +874,7 @@ pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
#[cfg(test)]
pub(crate) use ops::object::DeleteObjectCommitBarrier;
#[cfg(feature = "test-util")]
#[cfg(any(test, feature = "test-util"))]
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
pub(crate) use ops::object::body_cache_plaintext_len;
#[cfg(all(test, feature = "test-util"))]
+1 -1
View File
@@ -244,7 +244,7 @@ impl StaleMultipartCleanupGuard {
}
#[cfg(any(test, feature = "test-util"))]
#[derive(Clone, Copy, PartialEq, Eq)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MultipartCommitPause {
NewUploadBeforeLockLost,
PutPartBeforeLockAcquire,
File diff suppressed because it is too large Load Diff
+301 -8
View File
@@ -29,6 +29,25 @@ use std::future::Future;
const DELETED_BUCKETS_PREFIX: &str = ".deleted";
const SCANNER_BUCKET_LIST_SET_CONCURRENCY: usize = 4;
const EVENT_BUCKET_DELETE_BLOCKED: &str = "bucket_delete_blocked";
fn record_bucket_delete_blocker(bucket: &str, kind: BucketDeleteBlockerKind, residue: &BucketMetadataLessResidue) {
metrics::counter!("rustfs_bucket_delete_blockers_total", "kind" => kind.as_str()).increment(1);
debug!(
event = EVENT_BUCKET_DELETE_BLOCKED,
component = "ecstore",
subsystem = "bucket",
bucket,
blocker = kind.as_str(),
files = residue.files,
uuid_data_dirs = residue.uuid_data_dirs,
entries_scanned = residue.entries_scanned,
diagnostic_bytes_read = residue.diagnostic_bytes_read,
diagnostic_truncated = residue.diagnostic_truncated,
sample = residue.sample.as_deref().unwrap_or("<none>"),
"Bucket deletion was blocked by durable local state"
);
}
fn scanner_bucket_list_set_concurrency(set_count: usize) -> usize {
set_count.clamp(1, SCANNER_BUCKET_LIST_SET_CONCURRENCY)
@@ -156,6 +175,7 @@ where
async fn bucket_delete_local_blocker(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
budget: &mut BucketDeleteDiagnosticBudget,
) -> Result<Option<StorageError>> {
let local_disks = runtime_sources::local_disks_in(ctx).await;
let mut residue = BucketMetadataLessResidue::default();
@@ -164,18 +184,30 @@ async fn bucket_delete_local_blocker(
let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else {
continue;
};
let scan = scan_metadata_less_residue(&bucket_path?).await?;
let scan = scan_metadata_less_residue_with_budget(&bucket_path?, budget).await?;
if scan.xlmeta_found {
record_bucket_delete_blocker(bucket, scan.xlmeta_blocker.unwrap_or(BucketDeleteBlockerKind::UnknownXlMeta), &scan);
return Ok(Some(StorageError::BucketNotEmpty(bucket.to_string())));
}
residue.files = residue.files.saturating_add(scan.files);
residue.uuid_data_dirs = residue.uuid_data_dirs.saturating_add(scan.uuid_data_dirs);
residue.entries_scanned = residue.entries_scanned.saturating_add(scan.entries_scanned);
residue.diagnostic_bytes_read = residue.diagnostic_bytes_read.saturating_add(scan.diagnostic_bytes_read);
if residue.sample.is_none() {
residue.sample = scan.sample;
residue.sample = scan.sample.clone();
}
if scan.diagnostic_truncated {
residue.diagnostic_truncated = true;
record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::DiagnosticBudgetExceeded, &residue);
return Ok(Some(StorageError::BucketNotEmptyWithDetails {
bucket: bucket.to_string(),
details: residue.describe(),
}));
}
}
if residue.has_residue_without_xlmeta() {
record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::OrphanDirectory, &residue);
return Ok(Some(StorageError::BucketNotEmptyWithDetails {
bucket: bucket.to_string(),
details: residue.describe(),
@@ -783,12 +815,14 @@ impl ECStore {
}
}
};
let mut diagnostic_budget = None;
if bucket_exists {
validate_table_bucket_delete_guard(&self.ctx, bucket).await?;
if !opts.force {
if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket).await? {
let budget = diagnostic_budget.get_or_insert_with(BucketDeleteDiagnosticBudget::new);
if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket, budget).await? {
return Err(blocker);
}
delete_opts.force_if_empty = true;
@@ -827,7 +861,12 @@ impl ECStore {
{
if delete_opts.force_if_empty
&& matches!(&err, StorageError::BucketNotEmpty(_))
&& let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket).await?
&& let Some(blocker) = bucket_delete_local_blocker(
&self.ctx,
bucket,
diagnostic_budget.get_or_insert_with(BucketDeleteDiagnosticBudget::new),
)
.await?
{
return Err(blocker);
}
@@ -856,15 +895,17 @@ impl ECStore {
#[cfg(test)]
mod tests {
use super::{
SCANNER_BUCKET_LIST_SET_CONCURRENCY, await_bucket_namespace_operation, bucket_delete_metadata_cleanup_prefixes,
bucket_deleted_marker_prefix, bucket_deleted_marker_volume, run_bucket_usage_cleanup, run_physical_bucket_deletion,
scan_metadata_less_residue, scanner_bucket_list_set_concurrency, should_override_created_from_metadata,
BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES, BucketDeleteBlockerKind,
BucketDeleteDiagnosticBudget, SCANNER_BUCKET_LIST_SET_CONCURRENCY, await_bucket_namespace_operation,
bucket_delete_metadata_cleanup_prefixes, bucket_deleted_marker_prefix, bucket_deleted_marker_volume,
run_bucket_usage_cleanup, run_physical_bucket_deletion, scan_metadata_less_residue,
scan_metadata_less_residue_with_budget, scanner_bucket_list_set_concurrency, should_override_created_from_metadata,
validate_table_bucket_delete_allowed,
};
use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
use crate::bucket::metadata_sys;
use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE};
use crate::error::StorageError;
use crate::object_api::{ObjectOptions, PutObjReader};
use crate::runtime::instance::InstanceContext;
@@ -879,6 +920,7 @@ mod tests {
layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
};
use rustfs_data_usage::{BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageInfo};
use rustfs_filemeta::{FileInfo, FileMeta, TRANSITION_COMPLETE};
use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
use serial_test::serial;
use std::path::{Path, PathBuf};
@@ -892,6 +934,88 @@ mod tests {
static BUCKET_DELETE_TEST_ENV: OnceCell<(Vec<PathBuf>, Arc<ECStore>)> = OnceCell::const_new();
#[tokio::test(start_paused = true)]
async fn bucket_delete_diagnostic_budget_starts_with_first_scan_io_and_latches_once() {
let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
tokio::time::advance(Duration::from_secs(10)).await;
let first_polled = Arc::new(AtomicBool::new(false));
let first_polled_for_io = first_polled.clone();
let first = budget
.run_io(async move {
first_polled_for_io.store(true, Ordering::SeqCst);
Ok::<_, std::io::Error>(7_u8)
})
.await
.expect("the first diagnostic IO should succeed");
assert_eq!(first, Some(7));
assert!(first_polled.load(Ordering::SeqCst));
tokio::time::advance(Duration::from_millis(101)).await;
let expired_polled = Arc::new(AtomicBool::new(false));
let expired_polled_for_io = expired_polled.clone();
let expired = budget
.run_io(async move {
expired_polled_for_io.store(true, Ordering::SeqCst);
Ok::<_, std::io::Error>(9_u8)
})
.await
.expect("an expired diagnostic budget should not become an IO error");
assert_eq!(expired, None);
assert!(
!expired_polled.load(Ordering::SeqCst),
"the deadline must remain latched after the first scan IO"
);
}
#[tokio::test(start_paused = true)]
async fn bucket_delete_diagnostic_budget_times_out_its_first_pending_io() {
let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
let io_polled = Arc::new(AtomicBool::new(false));
let io_polled_for_future = io_polled.clone();
let result = budget
.run_io(std::future::poll_fn(move |_cx| {
io_polled_for_future.store(true, Ordering::SeqCst);
std::task::Poll::<std::io::Result<()>>::Pending
}))
.await
.expect("a diagnostic timeout should fail closed without an IO error");
assert_eq!(result, None);
assert!(
io_polled.load(Ordering::SeqCst),
"the first diagnostic IO must be polled before its timeout"
);
}
#[tokio::test(start_paused = true)]
async fn delayed_metadata_less_scans_still_detect_orphans_and_xlmeta() {
let root = tempfile::tempdir().expect("temporary delayed-scan roots should be created");
let orphan_root = root.path().join("orphan-root");
let xlmeta_root = root.path().join("xlmeta-root");
std::fs::create_dir_all(&orphan_root).expect("orphan root should be created");
std::fs::create_dir_all(&xlmeta_root).expect("xlmeta root should be created");
std::fs::write(orphan_root.join("orphan-part"), b"orphan").expect("orphan fixture should be written");
std::fs::write(xlmeta_root.join(STORAGE_FORMAT_FILE), b"invalid-xlmeta").expect("xl.meta fixture should be written");
let mut orphan_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
let mut xlmeta_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
tokio::time::advance(Duration::from_secs(60)).await;
let orphan = scan_metadata_less_residue_with_budget(&orphan_root, &mut orphan_budget)
.await
.expect("delayed orphan scan should complete");
assert!(orphan.has_residue_without_xlmeta());
assert!(!orphan.diagnostic_truncated);
let xlmeta = scan_metadata_less_residue_with_budget(&xlmeta_root, &mut xlmeta_budget)
.await
.expect("delayed xl.meta scan should complete");
assert!(xlmeta.xlmeta_found);
assert!(!xlmeta.diagnostic_truncated);
}
#[tokio::test(start_paused = true)]
async fn bucket_namespace_operation_fails_closed_after_lease_expiry() {
let ttl = Duration::from_millis(20);
@@ -1313,6 +1437,175 @@ mod tests {
assert!(sample.ends_with("/part.1"));
}
#[tokio::test]
async fn metadata_less_residue_scan_shares_one_entry_budget_across_roots() {
let root = tempfile::tempdir().expect("temporary diagnostic roots should be created");
let first_root = root.path().join("disk-a");
let second_root = root.path().join("disk-b");
tokio::fs::create_dir_all(&first_root)
.await
.expect("first diagnostic root should be created");
tokio::fs::create_dir_all(&second_root)
.await
.expect("second diagnostic root should be created");
for index in 0..3 {
std::fs::write(first_root.join(format!("first-{index}")), b"").expect("first-root fixture should be written");
std::fs::write(second_root.join(format!("second-{index}")), b"").expect("second-root fixture should be written");
}
let mut budget = BucketDeleteDiagnosticBudget::with_limits(4, Duration::from_secs(5));
let first = scan_metadata_less_residue_with_budget(&first_root, &mut budget)
.await
.expect("first root should fit the shared budget");
assert!(!first.diagnostic_truncated);
let second = scan_metadata_less_residue_with_budget(&second_root, &mut budget)
.await
.expect("second root should stop at the remaining shared budget");
assert!(second.diagnostic_truncated);
assert!(first.entries_scanned + second.entries_scanned <= 4);
}
#[tokio::test]
async fn metadata_less_residue_scan_honors_an_expired_request_deadline() {
let root = tempfile::tempdir().expect("temporary diagnostic root should be created");
std::fs::write(root.path().join("orphan"), b"").expect("deadline fixture should be written");
let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::ZERO);
let scan = scan_metadata_less_residue_with_budget(root.path(), &mut budget)
.await
.expect("an expired diagnostic budget should fail closed without an IO error");
assert!(scan.diagnostic_truncated);
assert_eq!(scan.entries_scanned, 0);
assert_eq!(scan.diagnostic_bytes_read, 0);
}
#[tokio::test]
async fn metadata_less_residue_scan_stops_at_diagnostic_budget() {
let root = tempfile::tempdir().expect("temporary bucket root should be created");
let bucket_path = root.path().join("bucket");
tokio::fs::create_dir_all(&bucket_path)
.await
.expect("budget fixture directory should be created");
for index in 0..(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES + 32) {
std::fs::write(bucket_path.join(format!("orphan-{index:05}")), b"").expect("budget fixture file should be created");
}
let residue = scan_metadata_less_residue(&bucket_path)
.await
.expect("budgeted residue scan should fail closed without an IO error");
assert!(residue.diagnostic_truncated);
assert!(residue.has_residue_without_xlmeta());
assert!(!residue.xlmeta_found);
assert!(residue.entries_scanned <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
assert!(residue.files <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
assert_eq!(residue.diagnostic_bytes_read, 0);
}
#[tokio::test]
async fn bucket_residue_scan_distinguishes_visible_and_tier_free_xlmeta() {
let root = tempfile::tempdir().expect("temporary bucket root should be created");
let bucket_path = root.path().join("bucket");
let visible_path = bucket_path.join("visible").join(STORAGE_FORMAT_FILE);
tokio::fs::create_dir_all(visible_path.parent().expect("visible xl.meta should have a parent"))
.await
.expect("visible object directory should be created");
let mut visible = FileMeta::new();
visible
.add_version(FileInfo {
version_id: Some(Uuid::new_v4()),
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
})
.expect("visible version should encode");
tokio::fs::write(&visible_path, visible.marshal_msg().expect("visible xl.meta should marshal"))
.await
.expect("visible xl.meta should be written");
let visible_scan = scan_metadata_less_residue(&bucket_path)
.await
.expect("visible xl.meta scan should succeed");
assert_eq!(visible_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::VisibleVersion));
tokio::fs::remove_dir_all(bucket_path.join("visible"))
.await
.expect("visible fixture should be removed");
let free_path = bucket_path.join("free").join(STORAGE_FORMAT_FILE);
tokio::fs::create_dir_all(free_path.parent().expect("free xl.meta should have a parent"))
.await
.expect("free-version object directory should be created");
let source_version_id = Uuid::new_v4();
let mut free = FileMeta::new();
free.add_version(FileInfo {
version_id: Some(source_version_id),
transition_status: TRANSITION_COMPLETE.to_string(),
transitioned_objname: "remote/object".to_string(),
transition_version_id: Some(Uuid::new_v4()),
transition_tier: "WARM".to_string(),
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
})
.expect("transitioned source should encode");
let mut delete = FileInfo {
version_id: Some(source_version_id),
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
};
delete.set_tier_free_version_id(&Uuid::new_v4().to_string());
free.delete_version(&delete)
.expect("transitioned source delete should create a free-version");
tokio::fs::write(&free_path, free.marshal_msg().expect("free-version xl.meta should marshal"))
.await
.expect("free-version xl.meta should be written");
let free_scan = scan_metadata_less_residue(&bucket_path)
.await
.expect("free-version xl.meta scan should succeed");
assert_eq!(free_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::TierFreeVersion));
tokio::fs::remove_dir_all(bucket_path.join("free"))
.await
.expect("free-version fixture should be removed");
let exact_limit_path = bucket_path.join("exact-limit").join(STORAGE_FORMAT_FILE);
tokio::fs::create_dir_all(exact_limit_path.parent().expect("exact-limit xl.meta should have a parent"))
.await
.expect("exact-limit object directory should be created");
let exact_limit = tokio::fs::File::create(&exact_limit_path)
.await
.expect("exact-limit xl.meta should be created");
exact_limit
.set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES)
.await
.expect("exact-limit xl.meta should be extended without allocating its contents");
let exact_limit_scan = scan_metadata_less_residue(&bucket_path)
.await
.expect("exact-limit xl.meta scan should remain fail closed");
assert_eq!(exact_limit_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
assert_eq!(exact_limit_scan.diagnostic_bytes_read, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
tokio::fs::remove_dir_all(bucket_path.join("exact-limit"))
.await
.expect("exact-limit fixture should be removed");
let oversized_path = bucket_path.join("oversized").join(STORAGE_FORMAT_FILE);
tokio::fs::create_dir_all(oversized_path.parent().expect("oversized xl.meta should have a parent"))
.await
.expect("oversized object directory should be created");
let oversized = tokio::fs::File::create(&oversized_path)
.await
.expect("oversized xl.meta should be created");
oversized
.set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES + 1)
.await
.expect("oversized xl.meta should be extended without allocating its contents");
let oversized_scan = scan_metadata_less_residue(&bucket_path)
.await
.expect("oversized xl.meta scan should remain fail closed");
assert_eq!(oversized_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
assert_eq!(oversized_scan.diagnostic_bytes_read, 0);
assert!(oversized_scan.diagnostic_bytes_read <= BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
}
#[tokio::test]
#[serial]
async fn scanner_bucket_listing_unions_every_erasure_set() {
File diff suppressed because it is too large Load Diff
+177 -10
View File
@@ -65,7 +65,7 @@ use http::HeaderMap;
use lazy_static::lazy_static;
use rand::RngExt as _;
use rustfs_config::server_config::Config;
use rustfs_filemeta::FileInfo;
use rustfs_filemeta::{FileInfo, FileMeta};
use rustfs_heal_contracts::heal_channel::{HealItemType, HealOpts};
use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
use rustfs_madmin::heal_commands::HealResultItem;
@@ -88,25 +88,105 @@ type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
pub const SCANNER_PUBLICATION_LEASE_TTL_MS: u64 = 60_000;
pub(crate) const BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES: u64 = 1024 * 1024;
pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES: usize = 4_096;
pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED: Duration = Duration::from_millis(100);
#[derive(Debug)]
pub(crate) struct BucketDeleteDiagnosticBudget {
deadline: Option<tokio::time::Instant>,
max_elapsed: Duration,
entries_remaining: usize,
}
impl BucketDeleteDiagnosticBudget {
pub(crate) fn new() -> Self {
Self::with_limits(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED)
}
fn with_limits(entries: usize, elapsed: Duration) -> Self {
Self {
deadline: None,
max_elapsed: elapsed,
entries_remaining: entries,
}
}
fn deadline(&mut self) -> tokio::time::Instant {
let max_elapsed = self.max_elapsed;
*self.deadline.get_or_insert_with(|| tokio::time::Instant::now() + max_elapsed)
}
fn claim_entry(&mut self) -> bool {
if self.entries_remaining == 0 {
return false;
}
let deadline = self.deadline();
if tokio::time::Instant::now() >= deadline {
return false;
}
self.entries_remaining -= 1;
true
}
async fn run_io<T, F>(&mut self, future: F) -> std::io::Result<Option<T>>
where
F: std::future::Future<Output = std::io::Result<T>>,
{
let deadline = self.deadline();
if tokio::time::Instant::now() >= deadline {
return Ok(None);
}
match tokio::time::timeout_at(deadline, future).await {
Ok(result) => result.map(Some),
Err(_) => Ok(None),
}
}
}
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub(crate) struct BucketMetadataLessResidue {
pub(crate) xlmeta_found: bool,
pub(crate) xlmeta_blocker: Option<BucketDeleteBlockerKind>,
pub(crate) files: usize,
pub(crate) uuid_data_dirs: usize,
pub(crate) entries_scanned: usize,
pub(crate) diagnostic_bytes_read: u64,
pub(crate) diagnostic_truncated: bool,
pub(crate) sample: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum BucketDeleteBlockerKind {
VisibleVersion,
TierFreeVersion,
UnknownXlMeta,
OrphanDirectory,
DiagnosticBudgetExceeded,
}
impl BucketDeleteBlockerKind {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::VisibleVersion => "visible_version",
Self::TierFreeVersion => "tier_free_version",
Self::UnknownXlMeta => "unknown_xlmeta",
Self::OrphanDirectory => "orphan_directory",
Self::DiagnosticBudgetExceeded => "diagnostic_budget_exceeded",
}
}
}
impl BucketMetadataLessResidue {
pub(crate) fn has_residue_without_xlmeta(&self) -> bool {
!self.xlmeta_found && self.files > 0
!self.xlmeta_found && (self.files > 0 || self.diagnostic_truncated)
}
pub(crate) fn describe(&self) -> String {
let sample = self.sample.as_deref().unwrap_or("<none>");
format!(
"metadata-less on-disk residue remains after empty-bucket verification: files={}, uuid_data_dirs={}, sample={sample}",
self.files, self.uuid_data_dirs
"metadata-less on-disk residue remains after empty-bucket verification: files={}, uuid_data_dirs={}, entries_scanned={}, diagnostic_bytes_read={}, diagnostic_truncated={}, sample={sample}",
self.files, self.uuid_data_dirs, self.entries_scanned, self.diagnostic_bytes_read, self.diagnostic_truncated,
)
}
}
@@ -145,28 +225,112 @@ pub(crate) async fn has_xlmeta_files(path: &std::path::Path) -> std::io::Result<
Ok(false)
}
#[cfg(test)]
pub(crate) async fn scan_metadata_less_residue(path: &std::path::Path) -> std::io::Result<BucketMetadataLessResidue> {
let mut budget = BucketDeleteDiagnosticBudget::new();
scan_metadata_less_residue_with_budget(path, &mut budget).await
}
async fn scan_metadata_less_residue_with_budget(
path: &std::path::Path,
budget: &mut BucketDeleteDiagnosticBudget,
) -> std::io::Result<BucketMetadataLessResidue> {
use crate::disk::STORAGE_FORMAT_FILE;
use tokio::fs;
use tokio::io::AsyncReadExt as _;
let mut scan = BucketMetadataLessResidue::default();
let mut stack = vec![path.to_path_buf()];
let mark_budget_exhausted = |scan: &mut BucketMetadataLessResidue| {
scan.diagnostic_truncated = true;
scan.sample.get_or_insert_with(|| "<diagnostic-budget-exceeded>".to_string());
};
while let Some(current_path) = stack.pop() {
let mut entries = match fs::read_dir(&current_path).await {
Ok(entries) => entries,
let mut entries = match budget.run_io(fs::read_dir(&current_path)).await {
Ok(Some(entries)) => entries,
Ok(None) => {
mark_budget_exhausted(&mut scan);
return Ok(scan);
}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
Err(err) => return Err(err),
};
while let Some(entry) = entries.next_entry().await? {
let file_type = entry.file_type().await?;
loop {
let entry = match budget.run_io(entries.next_entry()).await? {
Some(Some(entry)) => entry,
Some(None) => break,
None => {
mark_budget_exhausted(&mut scan);
return Ok(scan);
}
};
if !budget.claim_entry() {
mark_budget_exhausted(&mut scan);
return Ok(scan);
}
scan.entries_scanned = scan.entries_scanned.saturating_add(1);
let Some(file_type) = budget.run_io(entry.file_type()).await? else {
mark_budget_exhausted(&mut scan);
return Ok(scan);
};
let file_name = entry.file_name();
let file_name_str = file_name.to_string_lossy();
if file_name_str == STORAGE_FORMAT_FILE {
scan.xlmeta_found = true;
continue;
if scan.xlmeta_blocker.is_none() {
let entry_path = entry.path();
let Some(metadata) = budget.run_io(fs::metadata(&entry_path)).await? else {
mark_budget_exhausted(&mut scan);
scan.xlmeta_blocker = Some(BucketDeleteBlockerKind::DiagnosticBudgetExceeded);
return Ok(scan);
};
scan.xlmeta_blocker = Some(if metadata.len() > BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES {
BucketDeleteBlockerKind::UnknownXlMeta
} else {
match budget.run_io(fs::File::open(&entry_path)).await {
Ok(Some(file)) => {
let mut data = Vec::new();
let read = budget
.run_io(file.take(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES).read_to_end(&mut data))
.await;
scan.diagnostic_bytes_read = data.len() as u64;
match read {
Ok(Some(_)) => match FileMeta::load(&data) {
Ok(meta)
if !meta.versions.is_empty()
&& meta.versions.iter().all(|version| version.header.free_version()) =>
{
BucketDeleteBlockerKind::TierFreeVersion
}
Ok(meta) if !meta.versions.is_empty() => BucketDeleteBlockerKind::VisibleVersion,
Ok(_) | Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
},
Ok(None) => {
mark_budget_exhausted(&mut scan);
BucketDeleteBlockerKind::DiagnosticBudgetExceeded
}
Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
}
}
Ok(None) => {
mark_budget_exhausted(&mut scan);
BucketDeleteBlockerKind::DiagnosticBudgetExceeded
}
Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
}
});
let sample = entry_path
.strip_prefix(path)
.unwrap_or(entry_path.as_path())
.to_string_lossy()
.replace(std::path::MAIN_SEPARATOR, "/");
scan.sample = Some(sample);
}
return Ok(scan);
}
if file_type.is_dir() {
@@ -222,7 +386,10 @@ pub(crate) mod init_format;
pub(crate) mod list_objects;
mod multipart;
mod object;
pub(crate) use object::{ObjectLockDiagGuard, SourceCleanupMutationFence, tiered_data_movement_source_matches};
pub(crate) use object::{
DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence,
SourceCleanupMutationFence, tiered_data_movement_source_matches,
};
pub use object::{
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
SnapshotConsistencyError,
+69 -1
View File
@@ -846,6 +846,7 @@ impl ECStore {
.await
}
#[cfg(all(test, feature = "test-util"))]
pub(crate) async fn complete_multipart_upload_for_data_movement(
self: Arc<Self>,
target: (usize, Option<&ObjectLockDiagGuard>),
@@ -854,6 +855,44 @@ impl ECStore {
upload_id: &str,
uploaded_parts: Vec<CompletePart>,
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
self.complete_multipart_upload_for_data_movement_inner(target, bucket, object, upload_id, uploaded_parts, opts, None)
.await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn complete_multipart_upload_for_data_movement_with_publication_fence(
self: Arc<Self>,
target_pool_idx: usize,
bucket: &str,
object: &str,
upload_id: &str,
uploaded_parts: Vec<CompletePart>,
opts: &ObjectOptions,
publication_fence: RemoteTuplePublicationFence,
) -> Result<ObjectInfo> {
self.complete_multipart_upload_for_data_movement_inner(
(target_pool_idx, None),
bucket,
object,
upload_id,
uploaded_parts,
opts,
Some(publication_fence),
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn complete_multipart_upload_for_data_movement_inner(
self: Arc<Self>,
target: (usize, Option<&ObjectLockDiagGuard>),
bucket: &str,
object: &str,
upload_id: &str,
uploaded_parts: Vec<CompletePart>,
opts: &ObjectOptions,
publication_fence: Option<RemoteTuplePublicationFence>,
) -> Result<ObjectInfo> {
let (target_pool_idx, mutation_fence) = target;
check_complete_multipart_args(bucket, object, upload_id)?;
@@ -885,8 +924,36 @@ impl ECStore {
snapshot.add_lock_fences(&mut opts);
opts.object_lock_config_snapshot = Some(snapshot);
}
self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence)
let fixed_read_anchor = publication_fence
.as_ref()
.and_then(RemoteTuplePublicationFence::fixed_read_anchor_guard);
self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence.or(fixed_read_anchor))
.await;
// NewMultipart/UploadPart are staging only. Acquire and consume the
// non-cloneable publication capability immediately before Complete,
// then retain its guards until Complete has drained the commit path.
let publication_object = encode_dir_object(object);
let publication_guard = match publication_fence {
Some(publication_fence) => {
let guard = publication_fence
.into_commit_guard(target_pool_idx, bucket, &publication_object)
.await?;
guard.add_namespace_lock_fence(&mut opts);
opts.no_lock = true;
Some(guard)
}
None => {
if rustfs_utils::http::metadata_compat::contains_key_str(
&opts.user_defined,
rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
) {
return Err(Error::other(
"data movement multipart completion cannot publish transition ownership without a publication capability",
));
}
None
}
};
#[cfg(test)]
pause_data_movement_multipart_before_selected_completion(bucket).await;
let pool = self
@@ -911,6 +978,7 @@ impl ECStore {
},
)
.await;
drop(publication_guard);
let result = enqueue_transition_after_write(result, LcEventSrc::S3CompleteMultipartUpload).await;
if result.is_ok() {
list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await;
File diff suppressed because it is too large Load Diff
+32
View File
@@ -309,6 +309,17 @@ impl ECStore {
}
pub(super) async fn delete_prefix(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
let dispatch_scope = if opts.tier_delete_journal_api.is_some() {
let incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
let authorization = opts
.tier_delete_dispatch_authorization
.as_ref()
.ok_or_else(|| Error::other("prefix mutation is missing its dispatched-journal authorization"))?;
authorization.ensure_current(bucket, incarnation, object)?;
Some((authorization, incarnation))
} else {
None
};
if opts.lifecycle_delete_all.is_some() {
let mut preflight_opts = opts.clone();
preflight_opts
@@ -317,6 +328,9 @@ impl ECStore {
.ok_or(StorageError::PreconditionFailed)?
.phase = crate::object_api::LifecycleDeleteAllPhase::Preflight;
for pool in &self.pools {
if let Some((authorization, incarnation)) = dispatch_scope {
authorization.ensure_current(bucket, incarnation, object)?;
}
#[cfg(test)]
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Preflight, pool.pool_idx)?;
pool.delete_object(bucket, object, preflight_opts.clone()).await?;
@@ -326,6 +340,9 @@ impl ECStore {
.ok_or(StorageError::PreconditionFailed)?
.lock()
.mark_mutation_started();
if let Some((authorization, incarnation)) = dispatch_scope {
authorization.mark_mutation_started(bucket, incarnation, object)?;
}
let mut non_trigger_opts = opts.clone();
non_trigger_opts
.lifecycle_delete_all
@@ -333,6 +350,9 @@ impl ECStore {
.ok_or(StorageError::PreconditionFailed)?
.phase = crate::object_api::LifecycleDeleteAllPhase::History;
for pool in &self.pools {
if let Some((authorization, incarnation)) = dispatch_scope {
authorization.ensure_current(bucket, incarnation, object)?;
}
#[cfg(test)]
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::History, pool.pool_idx)?;
let mut pool_opts = non_trigger_opts.clone();
@@ -348,6 +368,9 @@ impl ECStore {
.phase = crate::object_api::LifecycleDeleteAllPhase::FinalPreflight;
let mut trigger_pools = Vec::new();
for (pool_index, pool) in self.pools.iter().enumerate() {
if let Some((authorization, incarnation)) = dispatch_scope {
authorization.ensure_current(bucket, incarnation, object)?;
}
#[cfg(test)]
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::FinalPreflight, pool.pool_idx)?;
let result = pool.delete_object(bucket, object, final_preflight_opts.clone()).await?;
@@ -366,6 +389,9 @@ impl ECStore {
.ok_or(StorageError::PreconditionFailed)?
.phase = crate::object_api::LifecycleDeleteAllPhase::Trigger;
for pool_index in trigger_pools {
if let Some((authorization, incarnation)) = dispatch_scope {
authorization.ensure_current(bucket, incarnation, object)?;
}
#[cfg(test)]
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Trigger, pool_index)?;
let mut pool_opts = trigger_opts.clone();
@@ -378,7 +404,13 @@ impl ECStore {
let mut first_error = None;
let mut first_volume_error = None;
let mut has_success = false;
if let Some((authorization, incarnation)) = dispatch_scope {
authorization.mark_mutation_started(bucket, incarnation, object)?;
}
for pool in &self.pools {
if let Some((authorization, incarnation)) = dispatch_scope {
authorization.ensure_current(bucket, incarnation, object)?;
}
let mut opts = opts.clone();
opts.delete_prefix = true;
match pool.delete_object(bucket, object, opts).await {
+23 -1
View File
@@ -697,6 +697,11 @@ impl HealChannelProcessor {
| HealRequestSource::Mrf => true,
});
// `no_lock` is an internal coordination hint. An admin request can
// carry the legacy field over the wire, but cannot use it as ambient
// authority to bypass storage namespace locking.
let no_lock = request.no_lock.unwrap_or(false) && request.source != HealRequestSource::Admin;
// Build HealOptions with all available fields
let options = HealOptions {
scan_mode: request.scan_mode.unwrap_or(HealScanMode::Normal),
@@ -705,7 +710,7 @@ impl HealChannelProcessor {
update_parity: request.update_parity.unwrap_or(true),
recursive,
dry_run: request.dry_run.unwrap_or(false),
no_lock: request.no_lock.unwrap_or(false),
no_lock,
timeout: request.timeout_seconds.map(std::time::Duration::from_secs),
pool_index: request.pool_index,
set_index: request.set_index,
@@ -990,6 +995,23 @@ mod tests {
assert!(heal_request.options.no_lock);
}
#[tokio::test]
async fn test_convert_to_heal_request_admin_cannot_bypass_object_lock() {
let heal_manager = create_test_heal_manager();
let processor = HealChannelProcessor::new(heal_manager);
let channel_request = HealChannelRequest {
id: "admin-no-lock".to_string(),
bucket: "test-bucket".to_string(),
object_prefix: Some("test-object".to_string()),
no_lock: Some(true),
source: HealRequestSource::Admin,
..Default::default()
};
let heal_request = processor.convert_to_heal_request(channel_request).unwrap();
assert!(!heal_request.options.no_lock, "admin nolock must not become storage lock authority");
}
#[tokio::test]
async fn test_convert_to_heal_request_scanner_defaults_recreate_missing_false() {
let heal_manager = create_test_heal_manager();
@@ -1521,6 +1521,8 @@ pub struct TierMutationControlResponse {
pub error_info: ::core::option::Option<::prost::alloc::string::String>,
#[prost(bytes = "bytes", tag = "5")]
pub response_proof: ::prost::bytes::Bytes,
#[prost(enumeration = "TierMutationFailureClass", tag = "6")]
pub failure_class: i32,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct GetLiveEventsRequest {
@@ -1611,6 +1613,35 @@ impl TierMutationPeerState {
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
#[repr(i32)]
pub enum TierMutationFailureClass {
Unspecified = 0,
PreDispatchRejected = 1,
Ambiguous = 2,
}
impl TierMutationFailureClass {
/// String value of the enum field names used in the ProtoBuf definition.
///
/// The values are not transformed in any way and thus are considered stable
/// (if the ProtoBuf definition does not change) and safe for programmatic use.
pub fn as_str_name(&self) -> &'static str {
match self {
Self::Unspecified => "TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED",
Self::PreDispatchRejected => "TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED",
Self::Ambiguous => "TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS",
}
}
/// Creates an enum from field names used in the ProtoBuf definition.
pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
match value {
"TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED" => Some(Self::Unspecified),
"TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED" => Some(Self::PreDispatchRejected),
"TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS" => Some(Self::Ambiguous),
_ => None,
}
}
}
/// Generated client implementations.
pub mod node_service_client {
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
+146 -55
View File
@@ -177,6 +177,7 @@ pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-re
pub const CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-cross-pool-fence-capability-v1\0";
pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024;
pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024;
pub const TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE;
pub const TIER_MUTATION_RPC_MAX_MESSAGE_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE + 4096;
pub fn heal_control_coordinator_epoch(topology_fingerprint: &str) -> Result<u64, &'static str> {
@@ -342,7 +343,16 @@ impl TierMutationRpcPhase {
}
}
pub const TIER_MUTATION_RPC_PROTOCOL_VERSION: u32 = 1;
// Version 2 required peer Prepare to block new tier-reference creators and
// drain their in-flight operation leases. Version 3 additionally binds Abort
// to the canonical Prepare intent so a missing-record Abort can persist an
// identity-bound tombstone and linearize against a delayed Prepare. Version 4
// signs a typed failure classification while retaining the exact v3 proof
// bytes for rolling compatibility.
pub const TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION: u32 = 3;
pub const TIER_MUTATION_RPC_PROTOCOL_VERSION: u32 = 4;
pub const TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE: usize = 1024;
pub const TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE: usize = 4096;
pub fn canonical_tier_mutation_rpc_body(
version: u32,
@@ -374,19 +384,23 @@ pub struct TierMutationRpcResponseProofInput<'a> {
pub state: i32,
pub applied: bool,
pub error_info: Option<&'a str>,
pub failure_class: i32,
}
pub fn canonical_tier_mutation_rpc_response_body(
input: TierMutationRpcResponseProofInput<'_>,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
const DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v1\0";
const V3_DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v1\0";
const V4_DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v2\0";
let phase = input.phase.as_wire_str().as_bytes();
let mutation_id = input.mutation_id.as_bytes();
let error_info = input.error_info.map(str::as_bytes);
let error_info_len = error_info.map_or(0, <[u8]>::len);
let is_v4 = input.version >= TIER_MUTATION_RPC_PROTOCOL_VERSION;
let domain = if is_v4 { V4_DOMAIN } else { V3_DOMAIN };
let mut body = Vec::with_capacity(
DOMAIN.len()
domain.len()
+ 4
+ 8
+ phase.len()
@@ -398,9 +412,10 @@ pub fn canonical_tier_mutation_rpc_response_body(
+ 1
+ 1
+ 8
+ error_info_len,
+ error_info_len
+ if is_v4 { 4 } else { 0 },
);
body.extend_from_slice(DOMAIN);
body.extend_from_slice(domain);
body.extend_from_slice(&input.version.to_be_bytes());
body.extend_from_slice(&u64::try_from(phase.len())?.to_be_bytes());
body.extend_from_slice(phase);
@@ -415,6 +430,9 @@ pub fn canonical_tier_mutation_rpc_response_body(
if let Some(error_info) = error_info {
body.extend_from_slice(error_info);
}
if is_v4 {
body.extend_from_slice(&input.failure_class.to_be_bytes());
}
Ok(body)
}
@@ -2137,10 +2155,10 @@ mod heal_control_tests {
#[cfg(test)]
mod tier_mutation_rpc_tests {
use super::{
TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase, TierMutationRpcResponseProofInput,
canonical_tier_mutation_rpc_body, canonical_tier_mutation_rpc_response_body,
TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION, TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase,
TierMutationRpcResponseProofInput, canonical_tier_mutation_rpc_body, canonical_tier_mutation_rpc_response_body,
};
use crate::proto_gen::node_service::TierMutationPeerState;
use crate::proto_gen::node_service::{TierMutationFailureClass, TierMutationPeerState};
use uuid::uuid;
#[test]
@@ -2155,7 +2173,7 @@ mod tier_mutation_rpc_tests {
)
.expect("small mutation body should encode");
let mut golden = b"rustfs-tier-mutation-rpc-v1\0".to_vec();
golden.extend_from_slice(&1_u32.to_be_bytes());
golden.extend_from_slice(&TIER_MUTATION_RPC_PROTOCOL_VERSION.to_be_bytes());
golden.extend_from_slice(&7_u64.to_be_bytes());
golden.extend_from_slice(b"prepare");
golden.extend_from_slice(mutation_id.as_bytes());
@@ -2165,8 +2183,13 @@ mod tier_mutation_rpc_tests {
assert_ne!(
baseline,
canonical_tier_mutation_rpc_body(2, TierMutationRpcPhase::Prepare, mutation_id, payload)
.expect("small mutation body should encode")
canonical_tier_mutation_rpc_body(
TIER_MUTATION_RPC_PROTOCOL_VERSION + 1,
TierMutationRpcPhase::Prepare,
mutation_id,
payload,
)
.expect("small mutation body should encode")
);
assert_ne!(
baseline,
@@ -2201,11 +2224,27 @@ mod tier_mutation_rpc_tests {
}
#[test]
fn canonical_tier_mutation_response_binds_request_state_and_error() {
fn tier_mutation_v3_request_and_response_golden_bytes_are_unchanged() {
let mutation_id = uuid!("12345678-1234-5678-9abc-def012345678");
let payload = b"canonical-intent-record";
let baseline = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
let request = canonical_tier_mutation_rpc_body(
TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
TierMutationRpcPhase::Prepare,
mutation_id,
payload,
)
.expect("v3 request should encode");
let mut request_golden = b"rustfs-tier-mutation-rpc-v1\0".to_vec();
request_golden.extend_from_slice(&TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION.to_be_bytes());
request_golden.extend_from_slice(&7_u64.to_be_bytes());
request_golden.extend_from_slice(b"prepare");
request_golden.extend_from_slice(mutation_id.as_bytes());
request_golden.extend_from_slice(&u64::try_from(payload.len()).expect("payload length should fit").to_be_bytes());
request_golden.extend_from_slice(payload);
assert_eq!(request, request_golden);
let response = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: payload,
@@ -2213,49 +2252,97 @@ mod tier_mutation_rpc_tests {
state: TierMutationPeerState::Prepared as i32,
applied: true,
error_info: None,
// v3 must ignore the field so its authenticated bytes stay exact.
failure_class: TierMutationFailureClass::PreDispatchRejected as i32,
})
.expect("small mutation response should encode");
.expect("v3 response should encode");
let mut response_golden = b"rustfs-tier-mutation-rpc-response-v1\0".to_vec();
response_golden.extend_from_slice(&TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION.to_be_bytes());
response_golden.extend_from_slice(&7_u64.to_be_bytes());
response_golden.extend_from_slice(b"prepare");
response_golden.extend_from_slice(mutation_id.as_bytes());
response_golden.extend_from_slice(&u64::try_from(payload.len()).expect("payload length should fit").to_be_bytes());
response_golden.extend_from_slice(payload);
response_golden.push(1);
response_golden.extend_from_slice(&(TierMutationPeerState::Prepared as i32).to_be_bytes());
response_golden.push(1);
response_golden.push(0);
response_golden.extend_from_slice(&0_u64.to_be_bytes());
assert_eq!(response, response_golden);
}
#[test]
fn canonical_tier_mutation_v4_response_binds_request_result_and_failure_class() {
let mutation_id = uuid!("12345678-1234-5678-9abc-def012345678");
let payload = b"canonical-intent-record";
let baseline = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: payload,
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("failure"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
})
.expect("small v4 mutation response should encode");
let cases = [
TierMutationRpcResponseProofInput {
version: 2,
version: TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: payload,
success: true,
state: TierMutationPeerState::Prepared as i32,
applied: true,
error_info: None,
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("failure"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Commit,
mutation_id,
canonical_payload: payload,
success: true,
state: TierMutationPeerState::Prepared as i32,
applied: true,
error_info: None,
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("failure"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id: uuid!("22345678-1234-5678-9abc-def012345678"),
canonical_payload: payload,
success: true,
state: TierMutationPeerState::Prepared as i32,
applied: true,
error_info: None,
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("failure"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: b"tampered-intent-record",
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("failure"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: payload,
success: true,
state: TierMutationPeerState::Prepared as i32,
applied: true,
error_info: None,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("failure"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
@@ -2263,39 +2350,43 @@ mod tier_mutation_rpc_tests {
mutation_id,
canonical_payload: payload,
success: false,
state: TierMutationPeerState::Prepared as i32,
applied: true,
error_info: None,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: payload,
success: true,
state: TierMutationPeerState::Committed as i32,
applied: true,
error_info: None,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: payload,
success: true,
state: TierMutationPeerState::Prepared as i32,
applied: false,
error_info: None,
error_info: Some("failure"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: payload,
success: true,
state: TierMutationPeerState::Prepared as i32,
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: true,
error_info: Some("error"),
error_info: Some("failure"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: payload,
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("other failure"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: payload,
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("failure"),
failure_class: TierMutationFailureClass::PreDispatchRejected as i32,
},
];
for case in cases {
+7
View File
@@ -1068,12 +1068,19 @@ enum TierMutationPeerState {
TIER_MUTATION_PEER_STATE_ABORTED = 3;
}
enum TierMutationFailureClass {
TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED = 0;
TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED = 1;
TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS = 2;
}
message TierMutationControlResponse {
bool success = 1;
TierMutationPeerState state = 2;
bool applied = 3;
optional string error_info = 4;
bytes response_proof = 5;
TierMutationFailureClass failure_class = 6;
}
message GetLiveEventsRequest {
@@ -46,7 +46,7 @@ use storage_api::lifecycle::{
STORAGE_FORMAT_FILE, TRANSITION_PENDING, TransitionCleanupStoreBarrier, TransitionOptions, assert_transition_meta_consistent,
enqueue_transition_for_existing_objects, expire_transitioned_object, free_version_count, get_bucket_metadata,
get_global_tier_config_mgr, init_background_expiry, init_bucket_metadata_sys, init_local_disks, is_err_object_not_found,
is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_tier_delete_journal_entries,
is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_transition_transaction_records,
register_mock_tier_util, update_bucket_metadata, wait_for_free_version_absence,
};
@@ -562,6 +562,29 @@ where
}
}
// Deep transition futures can overflow libtest's default stack before their
// first assertion, so the serial ILM cases use one dedicated test thread.
fn run_large_stack_async_test<F, Fut>(thread_name: &'static str, test_fn: F)
where
F: FnOnce() -> Fut + Send + 'static,
Fut: std::future::Future<Output = ()> + 'static,
{
let handle = std::thread::Builder::new()
.name(thread_name.to_string())
.stack_size(32 * 1024 * 1024)
.spawn(move || {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("large-stack scanner test runtime should build");
runtime.block_on(test_fn());
})
.expect("large-stack scanner test thread should spawn");
if let Err(payload) = handle.join() {
std::panic::resume_unwind(payload);
}
}
mod serial_tests {
use super::*;
@@ -737,10 +760,17 @@ mod serial_tests {
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[test]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
async fn rejected_transition_candidate_is_recovered_from_persisted_delete_journal() {
fn rejected_transition_candidate_is_recovered_from_persisted_transaction() {
run_large_stack_async_test(
"scanner-rejected-transition-transaction",
rejected_transition_candidate_is_recovered_from_persisted_transaction_case,
);
}
async fn rejected_transition_candidate_is_recovered_from_persisted_transaction_case() {
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
@@ -790,21 +820,20 @@ mod serial_tests {
"no cleanup path may delete the candidate while remove failures are enabled"
);
let retained = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let retained = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("tier delete journal recovery should scan the persisted candidate");
assert_eq!(retained.scanned, 1);
assert_eq!(retained.deleted, 0);
assert_eq!(retained.failed, 1);
.expect("transition transaction recovery should scan the persisted candidate");
assert_eq!((retained.scanned, retained.recovered, retained.retained, retained.failed), (1, 0, 0, 1));
assert_eq!(backend.object_count().await, 1, "failed recovery must retain the remote candidate");
backend.set_remove_failure(false);
let recovered = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("tier delete journal recovery should delete the retained candidate");
assert_eq!(recovered.scanned, 1);
assert_eq!(recovered.deleted, 1);
assert_eq!(recovered.failed, 0);
.expect("transition transaction recovery should delete the retained candidate");
assert_eq!(
(recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
(1, 1, 0, 0)
);
let removed_versions = backend.remove_versions().await;
assert!(!removed_versions.is_empty(), "recovery must issue at least one successful delete");
assert!(
@@ -813,10 +842,10 @@ mod serial_tests {
);
assert_eq!(backend.object_count().await, 0, "recovery should remove the rejected remote candidate");
let empty = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("a removed tier delete journal entry should no longer be listed");
assert_eq!(empty.scanned, 0, "successful recovery must remove the persisted journal entry");
.expect("a removed transition transaction should no longer be listed");
assert_eq!(empty.scanned, 0, "successful recovery must remove the persisted transaction");
assert_eq!(
read_object_fully(&ecstore, bucket_name.as_str(), object_name).await,
payload,
@@ -824,10 +853,17 @@ mod serial_tests {
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[test]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
async fn cancelled_before_cleanup_store_resolution_persists_journal() {
fn cancelled_before_cleanup_store_resolution_persists_transaction() {
run_large_stack_async_test(
"scanner-cancelled-transition-transaction",
cancelled_before_cleanup_store_resolution_persists_transaction_case,
);
}
async fn cancelled_before_cleanup_store_resolution_persists_transaction_case() {
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&tier_name).await;
@@ -876,9 +912,9 @@ mod serial_tests {
let retained = tokio::time::timeout(Duration::from_secs(30), async {
loop {
let recovery = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let recovery = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("the cancelled transition journal should be readable");
.expect("the cancelled transition transaction should be readable");
if recovery.scanned > 0 {
break recovery;
}
@@ -887,29 +923,36 @@ mod serial_tests {
})
.await
.expect("Drop should persist the rejected candidate through the saved instance context");
assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 0, 1));
assert_eq!((retained.scanned, retained.recovered, retained.retained, retained.failed), (1, 0, 0, 1));
tokio::time::timeout(Duration::from_secs(5), async {
while backend.exact_remove_count() < 2 {
tokio::task::yield_now().await;
}
})
.await
.expect("Drop cleanup and failed journal recovery must both preserve the exact version constraint");
.expect("Drop cleanup and failed transaction recovery must both preserve the exact version constraint");
let failed_exact_attempts = backend.exact_remove_count();
assert_eq!(
failed_exact_attempts, 2,
"the cancelled task and the first failed recovery must each preserve the exact delete constraint"
);
assert_eq!(backend.object_count().await, 1);
assert!(backend.remove_versions().await.is_empty());
backend.set_remove_failure(false);
let recovered = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("recovery should delete the candidate retained by the cancelled transition");
assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (1, 1, 0));
assert_eq!(
(recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
(1, 1, 0, 0)
);
assert_eq!(backend.remove_versions().await, backend.put_versions().await);
assert_eq!(backend.exact_remove_count(), failed_exact_attempts + 1);
assert_eq!(backend.object_count().await, 0);
let empty = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("successful recovery should remove the cancellation journal");
.expect("successful recovery should remove the cancellation transaction");
assert_eq!(empty.scanned, 0);
assert_eq!(
read_object_fully(&ecstore, bucket_name.as_str(), object_name).await,
@@ -918,10 +961,14 @@ mod serial_tests {
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[test]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
async fn rejected_transition_cleanup_durability_matrix() {
fn rejected_transition_cleanup_durability_matrix() {
run_large_stack_async_test("scanner-transition-cleanup-matrix", rejected_transition_cleanup_durability_matrix_case);
}
async fn rejected_transition_cleanup_durability_matrix_case() {
#[derive(Clone, Copy)]
enum CleanupCase {
Persisted,
@@ -1010,44 +1057,67 @@ mod serial_tests {
CleanupCase::Persisted | CleanupCase::DeleteFallback => {
assert_eq!(backend.remove_versions().await, backend.put_versions().await);
assert_eq!(backend.object_count().await, 0, "cleanup must remove the exact candidate");
let recovery = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let recovery = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("successful cleanup must not retain a journal entry");
assert_eq!(recovery.scanned, 0);
.expect("successful cleanup must leave no failed transition transaction");
assert_eq!(recovery.failed, 0);
assert_eq!(recovery.retained, 0);
assert_eq!(recovery.recovered, recovery.scanned);
let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("successful reconciliation must remove every transition transaction");
assert_eq!(empty.scanned, 0);
}
CleanupCase::RetryPersisted => {
assert!(
!err.to_string().contains("journal retry error"),
"a successful journal retry must preserve the original version-constraint error"
"the transaction path must not surface the removed journal fallback error"
);
assert_eq!(backend.object_count().await, 1);
let retained = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let retained = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("the retried journal should be recoverable");
assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 0, 1));
.expect("the retained transaction should be recoverable");
assert_eq!((retained.scanned, retained.recovered, retained.retained, retained.failed), (1, 0, 0, 1));
backend.set_remove_failure(false);
let recovered = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("recovery should delete the exact retried candidate");
assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (1, 1, 0));
.expect("recovery should delete the exact transaction candidate");
assert_eq!(
(recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
(1, 1, 0, 0)
);
assert_eq!(backend.remove_versions().await, backend.put_versions().await);
assert_eq!(backend.object_count().await, 0);
let empty = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("successful recovery must remove the retried journal");
.expect("successful recovery must remove the retained transaction");
assert_eq!(empty.scanned, 0);
}
CleanupCase::FullyFailed => {
let message = err.to_string();
assert!(message.contains("initial journal error"), "{message}");
assert!(message.contains("cleanup error"), "{message}");
assert!(message.contains("journal retry error"), "{message}");
assert_eq!(backend.object_count().await, 1, "both failed safeguards must leave the candidate visible");
assert!(backend.remove_versions().await.is_empty());
let recovery = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let retained = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("failed journal writes must not create partial recovery entries");
assert_eq!(recovery.scanned, 0);
.expect("the pre-upload transaction must retain ownership after cleanup failure");
assert_eq!(retained.scanned, 1, "the failed cleanup must keep one durable transaction owner");
assert_eq!(retained.recovered, 0);
assert_eq!(retained.retained + retained.failed, 1);
backend.set_remove_failure(false);
let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("recovery should delete the candidate after the backend becomes available");
assert_eq!(
(recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
(1, 1, 0, 0)
);
assert_eq!(backend.remove_versions().await, backend.put_versions().await);
assert_eq!(backend.object_count().await, 0);
let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("successful recovery must remove the failed cleanup transaction");
assert_eq!(empty.scanned, 0);
}
}
assert_eq!(
@@ -1062,20 +1132,7 @@ mod serial_tests {
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
fn test_transition_and_restore_flows() {
std::thread::Builder::new()
.name("scanner-transition-restore-flows".to_string())
.stack_size(32 * 1024 * 1024)
.spawn(|| {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("transition and restore test runtime should build");
runtime.block_on(test_transition_and_restore_flows_inner());
})
.expect("transition and restore test thread should spawn")
.join()
.expect("transition and restore test thread should finish");
run_large_stack_async_test("scanner-transition-restore-flows", test_transition_and_restore_flows_inner);
}
async fn test_transition_and_restore_flows_inner() {
+2 -2
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::tier_delete_journal::recover_tier_delete_journal_entries;
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::transition_transaction::recover_transition_transaction_records;
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::{
bucket_lifecycle_audit::LcEventSrc,
bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, expire_transitioned_object, init_background_expiry},
@@ -50,7 +50,7 @@ pub(crate) mod lifecycle {
TransitionCleanupStoreBarrier, TransitionOptions, assert_transition_meta_consistent,
enqueue_transition_for_existing_objects, expire_transitioned_object, free_version_count, get_bucket_metadata,
get_global_tier_config_mgr, init_background_expiry, init_bucket_metadata_sys, init_local_disks, is_err_object_not_found,
is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_tier_delete_journal_entries,
is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_transition_transaction_records,
register_mock_tier_util, update_bucket_metadata, wait_for_free_version_absence,
};
}
@@ -39,6 +39,8 @@ for later deletion.
- `rustfs-5063` pre-beta.9 Local KMS recovery: persisted Local KMS configs from beta.8 and earlier predate the explicit insecure-development flag, and encrypted key files use the legacy SHA-256 KDF. Remove the config fallback after supported upgrades have rewritten or explicitly resaved all pre-beta.9 configs with the development-default field, and remove the legacy KDF after supported upgrades have rewritten all pre-beta.9 Local KMS key files with explicit at-rest protection.
- `sse-local-dek-json-v1` legacy local SSE DEK decoding: releases before the JSON envelope wrote wrapped DEKs as `base64(nonce):base64(ciphertext)`, so readers retain that decoder while all new writes use the versioned JSON envelope. Remove the colon decoder after the minimum supported direct-upgrade release writes JSON envelopes and migration tooling has rewritten every retained legacy object.
- `not-initialized-error-code-v1` typed control-plane not-initialized wire code: control-plane RPC responses historically signaled an uninitialized peer only through the literal error_info string "errServerNotInitialized" (one drift site says "storage layer not initialized"). Responses now dual-carry a typed ControlPlaneErrorCode beside the legacy string, and clients prefer the code; the string stays populated and the client substring fallback (is_err_not_initialized, control_plane_failure) stays in place so mixed-version clusters keep classifying older peers' responses. Remove the substring fallback (and stop populating error_info for this case) after the minimum supported RustFS peer version always sends error_code.
- `backlog-2097-tier-mutation-v4-error-text` tier-mutation Prepare rejection classification: a v3 server rejects a v4 request before store/runtime dispatch with the FailedPrecondition status and an authenticated, byte-exact unsupported-version message. A v4 coordinator recognizes only that exact code/message/requested-version tuple as definitely not persisted and fails the mutation without sending that peer an incompatible Abort; Unimplemented, near-text, missing/unknown failure classes, timeouts, and every other transport outcome remain ambiguous and stay in identity-bound Abort fanout. There is no automatic v3 retry. New servers retain v3 request/proof decoding for older coordinators, while operators must pause tier edit/remove/clear during a mixed v3/v4 rollout. Remove the text classifier after the minimum supported RustFS peer version returns the signed v4 PreDispatchRejected failure class.
- `backlog-2097-tier-delete-journal-v6` tier-delete sole-owner recovery: v6 distinguishes transactions that may replace the xl.meta free-version owner and therefore require the all-pool live-source proof. v5-and-older readers reject and retain v6 records during rolling upgrades instead of performing an unsafe remote delete. Keep v1-v5 readers for upgrade recovery and keep the downgrade prohibition while any v6 record exists; retire the fence only after every supported rollback release understands and enforces v6 proof semantics.
- `multipart-compression-default-off-window` staged multipart disk-compression rollout: releases before the resumable legacy decompressor fail transient reads of compressed objects under mid-payload suspension, so multipart uploads advertise the compression marker only when RUSTFS_COMPRESSION_MULTIPART_ENABLED is set in addition to RUSTFS_COMPRESSION_ENABLED, keeping rolling upgrades from creating new compressed multipart objects while pre-fix nodes may still serve reads. Flip the default to enabled (and retire the extra switch) after the minimum supported direct-upgrade release ships the resumable decompressor.
## Review Checklist
+110 -16
View File
@@ -165,16 +165,33 @@ and all listing/GET paths exclude them.
### Lifecycle And Consumers
Creation: any local delete that removes a version whose transition status is
`complete` appends the record via `MetaObject::delete_version`
`init_free_version` (skipped only when `skip_tier_free_version` is set, as on
data-movement copies). The same deletes also persist a durable tier-journal
entry on every user-facing path: S3 single deletes (`execute_delete_object`
`delete_object_with_tier_delete_journal`), S3 batch deletes, lifecycle expiry,
and lifecycle delete-all all prepare and commit a journal entry around the
delete. A journal entry is omitted when the removed version's transition state
decodes as `TransitionVersionState::Unknown`, or on internal journal-less
delete paths that never touch transitioned user objects.
Creation: a local delete that removes a version whose transition status is
`complete` normally appends the record via `MetaObject::delete_version`
`init_free_version`. User-facing single and batch deletes always retain that
historical owner when they actually remove a transitioned source; they do not
create a tier journal, probe a fleet capability, or issue a peer mutation RPC.
`TransitionVersionState::Unknown` and incomplete destination identities remain
on the same conservative free-version path. Delete-marker creation on an Enabled
bucket remains unchanged and does not schedule remote deletion.
Recursive prefix/delete-all cannot preserve per-object markers across its
physical directory purge, so it requires a v6 recoverable journal for every
transitioned visible source plus a durable dispatch manifest for the complete
operation. It fails closed before mutation on legacy metadata or any existing
hidden tier free-version under the prefix. Its internal streaming walk
discovers logical keys, then exact-loads every key from its authoritative set in
every pool, including free versions; the S3 listing merge is never treated as a
complete physical-owner inventory. Tier-operation leases remain held from that
preflight through journal prepare and physical deletion. Once physical deletion
starts, any error is mutation-ambiguous: authorized/dispatched journals remain
for recovery to commit owners only after all physical sets prove both the source
and exact free-version identity absent; uncertain owners are retained.
If a retry discovers a later transitioned source after the manifest reached
`DispatchAuthorized`, it replays only the manifest's immutable predecessor set,
completes that operation, and leaves the newcomer for a successor dispatch.
Operators may retry after the legacy free-version worker has durably completed
remote and local cleanup. Journal-less internal deletes and older nodes retain
their established marker behavior.
Consumption while the record exists: the background recovery loop started by
`init_background_expiry` (spawned by `spawn_tier_free_version_recovery_once`,
@@ -209,16 +226,93 @@ only when its free-version identity matches; a conflicting ordinary version or
different free record is an overwrite error. This makes retries idempotent and
prevents a free record from replacing a user-visible version.
### Remote-Tuple Publication Fence
Cross-pool capability v3 includes a commit-late publication contract for every
path that can copy an existing transition tuple to a new physical owner. This
capability version is independent of the tier-mutation RPC protocol version.
A mixed fleet whose minimum cross-pool capability is below v3 cannot authorize
journal-v6 remote deletion.
Data movement captures a non-cloneable, process-local source capability before
copying, but it does not hold a namespace write lock or tier-operation lease
while reading a large body or uploading multipart parts. `NewMultipartUpload`
and `UploadPart` are staging only. Immediately before single-PUT rename,
Multipart Complete, or a pure-remote/free-version metadata quorum write, the
final consumer acquires the exact tier generation (when a remote tuple exists),
then fixed/source/target write domains in stable order. The fixed domain is used
only for a real remote-tuple decommission publisher; an ordinary local object
keeps the lighter source/target commit scope.
While that owned scope is held, the publisher re-reads the exact source pool and
compares version, data directory, modification time, ETag, checksums, transition
tuple, transition-version state, and destination identity. A missing or changed
source, changed/revoked tier generation, bucket incarnation change, or lost lock
fails before target rename. The scope remains owned through rename quorum and
the existing rename-tail guard handoff. Consequently, recovery-first ordering
cannot delete the remote object and then have a stale restored-transitioned
rebalance recreate its tuple; publisher-first ordering makes recovery wait and
rescan the newly committed owner.
Full cross-key S3 Copy is not an ownership-sharing operation: it materializes
local data and strips transition, destination, transaction, and free-version
keys. Same-key metadata/version-only updates preserve the existing protected
state. Admin heal keeps the legacy `nolock` request field for wire compatibility
but ignores it as lock authority; final heal writes enter the normal locked
path. Restore similarly ignores ambient `ObjectOptions.no_lock`, acquires its
own commit-late PUT/Complete lock, validates the restore operation id, and keeps
an exact tier generation lease through the local commit.
### Reference-Audit Result
After migration, user-facing GET/list/transition/replication/restore paths still
exclude the record. Recovery, usage scanning, lifecycle tier cleanup, and heal
continue to see it when they request free versions, so an unresolved remote
delete remains actionable on the target pool. The committed tier journal remains
an independent retry source where one exists; it is not used as a reason to drop
the xl.meta record. In particular, `Unknown` transition state records are
migrated unchanged rather than discarded: the lifecycle worker retains them if
remote identity validation cannot make a delete request.
continue to see a legacy/fallback record when they request free versions, so an
unresolved remote delete remains actionable on the target pool. Only an
authorized recursive prefix/delete-all v6 transaction may instead use a
per-source journal as the sole retry source; ordinary single/batch deletes never
take that path. A journal discovered
alongside an older or fallback free-version does not authorize dropping the
record. In particular, `Unknown` transition state records are migrated unchanged
rather than discarded: the lifecycle worker retains them if remote identity
validation cannot make a delete request.
Tier edit/remove/clear reference proof uses the internal walk with
`include_free_versions = true`, in addition to persisted journal and transition
transaction checks. Protocol v3 peer Prepare blocks new reference creators and
drains existing tier-operation leases before this proof; protocol v4 preserves
that state machine and adds a signed failure classification. Abort carries the
canonical Prepare intent, so a peer can create an identity-bound `Aborted`
tombstone even when Abort overtakes Prepare. A delayed matching Prepare then
converges on `Aborted` instead of reinstalling the block; a conflicting intent
with the same mutation id fails closed. The tombstone remains durable until the
intent expiry plus the configured clock-skew allowance, including across reload
and coordinator-record cleanup. After
expiry, a missing-record replay of the original signed Prepare is rejected and
cannot recreate a peer-only runtime fence. Abort checks an existing same-identity
terminal record before consulting mutable current-config proof, and recovery
reconstructs the original Prepared revision for Abort fanout.
A new server accepts both v3 and v4 requests and selects the matching canonical
response proof. During a mixed rollout, an older v3 server rejects a v4 request
with an authenticated, byte-exact unsupported-version status before dispatch;
the v4 coordinator treats only that exact rejection as definitely not installed,
fails the admin mutation, and does not send the peer an incompatible Abort.
There is deliberately no automatic v3 retry. `Unimplemented`, near-text,
timeouts, missing/unknown failure classes, and other ambiguous outcomes still
receive Abort and retain the coordinator retry record if Abort cannot be proven.
Operators must pause and drain tier edit/remove/clear operations before starting
the rolling upgrade, leave them disabled while any v3-only peer remains, and
resume only after every topology member advertises the v4-capable release.
Ordinary object I/O and free-version cleanup remain available; xl.meta is
unchanged by the rejected mutation.
Sole-owner transactions use journal v6: v5-and-older readers reject and retain
those records, so an old recovery worker cannot bypass the all-pool proof. Older
nodes may continue to create fallback free-versions until the rollout is
homogeneous. A deployment must not downgrade every v6-aware recovery worker
while any v6 record remains; drain the journal first or keep at least one v6-aware
worker until cleanup converges.
Each migrated record emits `state = "free_version_migrated"` with reason
`tier_free_version_migrated`. A record consumed before migration emits
+7 -4
View File
@@ -1109,7 +1109,10 @@ fn build_heal_channel_request(hip: &HealInitParams) -> HealChannelRequest {
heal_request.update_parity = Some(hip.hs.update_parity);
heal_request.recursive = Some(recursive || root_cluster_target);
heal_request.dry_run = Some(hip.hs.dry_run);
heal_request.no_lock = Some(hip.hs.no_lock);
// `nolock` is retained in the admin wire shape for compatibility, but it
// is not transferable lock authority. Admin heals must enter the storage
// layer through its normal object-lock path.
heal_request.no_lock = Some(false);
heal_request.source = HealRequestSource::Admin;
heal_request
}
@@ -2588,7 +2591,7 @@ mod tests {
}
#[test]
fn test_build_heal_channel_request_preserves_client_options() {
fn test_build_heal_channel_request_preserves_safe_client_options() {
let hip = HealInitParams {
bucket: "bucket-a".to_string(),
obj_prefix: "prefix-a".to_string(),
@@ -2623,7 +2626,7 @@ mod tests {
assert_eq!(request.update_parity, Some(false));
assert_eq!(request.recursive, Some(true));
assert_eq!(request.dry_run, Some(true));
assert_eq!(request.no_lock, Some(true));
assert_eq!(request.no_lock, Some(false));
let envelope = rustfs_protos::heal_control::Envelope::start(
request,
@@ -2639,7 +2642,7 @@ mod tests {
else {
panic!("expected admin heal start command");
};
assert_eq!(request.no_lock, Some(true));
assert_eq!(request.no_lock, Some(false));
}
#[test]
@@ -29,7 +29,7 @@ use super::storage_api::test::object_utils::to_s3s_etag;
use super::storage_api::test::runtime::{MockWarmBackend, MockWarmOp, register_mock_tier as register_mock_tier_util};
use super::storage_api::test::{
ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, StorageObjectInfo as ObjectInfo,
StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader,
StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader, install_all_v6_fleet_capability_proof,
};
use super::{multipart_usecase::DefaultMultipartUsecase, object_usecase::DefaultObjectUsecase};
use crate::app::bucket_usecase::DefaultBucketUsecase;
@@ -79,6 +79,12 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
return (paths.clone(), ecstore.clone());
}
// Production startup installs this proof through the notification-system
// fleet probe. This isolated single-node harness constructs ECStore
// directly, so model that completed all-v6 probe before any journal permit
// or background expiry worker can exist.
install_all_v6_fleet_capability_proof();
let test_base_dir = format!("/tmp/rustfs_app_lifecycle_test_{}", Uuid::new_v4());
let temp_dir = PathBuf::from(&test_base_dir);
if temp_dir.exists() {
+1
View File
@@ -1270,6 +1270,7 @@ pub(crate) mod test {
pub(crate) mod data_usage {
pub(crate) use super::super::data_usage::*;
}
pub(crate) use crate::storage::storage_api::ecstore_bucket::install_all_v6_fleet_capability_proof;
pub(crate) use crate::storage::storage_api::test_consumer::{get_global_bucket_metadata_sys, set_bucket_metadata};
pub(crate) use crate::storage::storage_api::{
ECStore, Endpoint, Endpoints, PoolEndpoints, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader,
+169 -21
View File
@@ -83,6 +83,9 @@ const TIER_MUTATION_PEER_STATE_UNSPECIFIED_WIRE: i32 = 0;
const TIER_MUTATION_PEER_STATE_PREPARED_WIRE: i32 = 1;
const TIER_MUTATION_PEER_STATE_COMMITTED_WIRE: i32 = 2;
const TIER_MUTATION_PEER_STATE_ABORTED_WIRE: i32 = 3;
const TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED_WIRE: i32 = 0;
const TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED_WIRE: i32 = 1;
const TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS_WIRE: i32 = 2;
fn signal_service_response(success: bool, error_info: Option<String>) -> Response<SignalServiceResponse> {
Response::new(SignalServiceResponse {
@@ -178,7 +181,10 @@ fn remove_heal_control_replay(
static HEAL_CONTROL_REPLAY_CACHE: OnceLock<tokio::sync::Mutex<HashMap<String, Arc<HealControlReplayEntry>>>> = OnceLock::new();
static NODE_CAPABILITY_SERVER_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
// v3 additionally promises the v6 tier-delete dispatch-manifest policy. The
// existing periodic topology probe carries both capabilities so normal object
// operations do not add another peer RPC.
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 3;
fn admit_heal_control_replay(
replay_cache: &mut HashMap<String, Arc<HealControlReplayEntry>>,
@@ -723,9 +729,28 @@ impl TierMutationControlRpcService {
.map_err(|_| Status::invalid_argument("tier mutation request length cannot be represented"))?;
verify_tonic_canonical_body_digest(request, &body)
.map_err(|err| Status::permission_denied(format!("tier mutation authentication failed: {err}")))?;
let store = self
.resolve_object_store()
.ok_or_else(|| Status::failed_precondition("tier mutation object store is not initialized"))?;
if !tier_mutation_protocol_version_is_supported(version) {
return Err(Status::failed_precondition(format!(
"unsupported tier mutation peer protocol version: {version}"
)));
}
let store = match self.resolve_object_store() {
Some(store) => store,
None if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION => {
return tier_mutation_control_response(TierMutationControlResponseInput {
version,
phase,
mutation_id,
canonical_payload,
success: false,
state: TIER_MUTATION_PEER_STATE_UNSPECIFIED_WIRE,
applied: false,
error_info: Some("tier mutation object store is not initialized".to_string()),
failure_class: TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED_WIRE,
});
}
None => return Err(Status::failed_precondition("tier mutation object store is not initialized")),
};
match tier_mutation_peer::handle_tier_mutation_peer_request(store, version, phase, mutation_id, canonical_payload).await {
Ok(outcome) => tier_mutation_control_response(TierMutationControlResponseInput {
@@ -737,6 +762,7 @@ impl TierMutationControlRpcService {
state: tier_mutation_peer_state_to_proto_wire(outcome.state),
applied: outcome.applied,
error_info: None,
failure_class: TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED_WIRE,
}),
Err(err) => tier_mutation_control_response(TierMutationControlResponseInput {
version,
@@ -746,7 +772,12 @@ impl TierMutationControlRpcService {
success: false,
state: TIER_MUTATION_PEER_STATE_UNSPECIFIED_WIRE,
applied: false,
error_info: Some(err.to_string()),
error_info: Some(bounded_tier_mutation_error_info(err.to_string())),
failure_class: if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION {
TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS_WIRE
} else {
TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED_WIRE
},
}),
}
}
@@ -811,15 +842,35 @@ fn parse_tier_mutation_id(mutation_id: &str) -> Result<Uuid, Status> {
Ok(parsed)
}
fn tier_mutation_protocol_version_is_supported(version: u32) -> bool {
matches!(
version,
rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION | rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
)
}
fn bounded_tier_mutation_error_info(mut error_info: String) -> String {
let limit = rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE;
if error_info.len() <= limit {
return error_info;
}
let mut boundary = limit;
while !error_info.is_char_boundary(boundary) {
boundary -= 1;
}
error_info.truncate(boundary);
error_info
}
fn validate_tier_mutation_payload_size(phase: rustfs_protos::TierMutationRpcPhase, payload_len: usize) -> Result<(), Status> {
let limit = match phase {
rustfs_protos::TierMutationRpcPhase::Prepare => rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE,
rustfs_protos::TierMutationRpcPhase::Commit => rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE,
rustfs_protos::TierMutationRpcPhase::Abort => {
if payload_len != 0 {
return Err(Status::invalid_argument("tier mutation abort payload must be empty"));
if payload_len == 0 {
return Err(Status::invalid_argument("tier mutation abort payload is empty"));
}
return Ok(());
rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE
}
_ => return Err(Status::invalid_argument("tier mutation rpc phase is unsupported")),
};
@@ -838,6 +889,7 @@ struct TierMutationControlResponseInput<'a> {
state: i32,
applied: bool,
error_info: Option<String>,
failure_class: i32,
}
fn tier_mutation_control_response(
@@ -853,6 +905,7 @@ fn tier_mutation_control_response(
state: input.state,
applied: input.applied,
error_info: input.error_info.as_deref(),
failure_class: input.failure_class,
})
.map_err(|_| Status::internal("tier mutation response length cannot be represented"))?;
let response_proof = sign_tonic_rpc_response_proof(&canonical_response)
@@ -863,6 +916,7 @@ fn tier_mutation_control_response(
applied: input.applied,
error_info: input.error_info,
response_proof: response_proof.into(),
failure_class: input.failure_class,
}))
}
@@ -2469,10 +2523,10 @@ impl Node for NodeService {
#[allow(unused_imports)]
mod tests {
use super::{
CollectMetricsOpts, DiskStore, Error, HEAL_CONTROL_PAYLOAD_MAX_SIZE, KMS_SIGNAL_SUBSYSTEM, MetricType, Node as _,
NodeService, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_PUBLICATION_LEASE_TTL_MS, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS, admit_heal_control_replay,
CROSS_POOL_FENCE_SUPPORTED_VERSION, CollectMetricsOpts, DiskStore, Error, HEAL_CONTROL_PAYLOAD_MAX_SIZE,
KMS_SIGNAL_SUBSYSTEM, MetricType, Node as _, NodeService, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_PUBLICATION_LEASE_TTL_MS,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS, admit_heal_control_replay,
background_rebalance_start_error_message, execute_heal_control_envelope_with_manager,
initialize_heal_topology_fingerprint, initialize_heal_topology_fingerprint_with_probe, legacy_scanner_activity_response,
make_heal_control_server, make_heal_control_server_with_cache, make_server, make_server_for_context,
@@ -2514,9 +2568,9 @@ mod tests {
RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerPublicationLeaseReleaseRequest,
ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest,
SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest,
StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationPeerState, TierMutationPrepareRequest,
UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
WriteRequest,
StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationFailureClass,
TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest,
VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, WriteRequest,
heal_control_service_client::HealControlServiceClient,
heal_control_service_server::{HealControlService as _, HealControlServiceServer},
node_service_client::NodeServiceClient,
@@ -3286,6 +3340,7 @@ mod tests {
#[tokio::test]
async fn tier_mutation_control_requires_body_bound_auth_before_store_lookup() {
let _ = rustfs_credentials::set_global_rpc_secret("tier-mutation-control-auth-test-secret".to_string());
let service = make_tier_mutation_control_server_for_context(None);
let mutation_id = uuid::Uuid::new_v4();
let unsigned = service
@@ -3317,8 +3372,26 @@ mod tests {
let unavailable = service
.prepare_tier_mutation(signed)
.await
.expect_err("authenticated request still requires initialized object store");
assert_eq!(unavailable.code(), tonic::Code::FailedPrecondition);
.expect("authenticated v4 pre-dispatch rejection should be signed")
.into_inner();
assert!(!unavailable.success);
assert_eq!(unavailable.failure_class, TierMutationFailureClass::PreDispatchRejected as i32);
assert_eq!(unavailable.error_info.as_deref(), Some("tier mutation object store is not initialized"));
let canonical =
rustfs_protos::canonical_tier_mutation_rpc_response_body(rustfs_protos::TierMutationRpcResponseProofInput {
version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: rustfs_protos::TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: b"intent",
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("tier mutation object store is not initialized"),
failure_class: TierMutationFailureClass::PreDispatchRejected as i32,
})
.expect("small response should encode");
crate::storage::storage_api::verify_tonic_rpc_response_proof(&canonical, &unavailable.response_proof)
.expect("v4 pre-dispatch rejection must authenticate its failure class");
}
#[tokio::test]
@@ -3337,12 +3410,13 @@ mod tests {
}
#[tokio::test]
async fn tier_mutation_control_rejects_old_protocol_version_before_store_lookup() {
async fn tier_mutation_control_rejects_unsupported_protocol_version_before_store_lookup() {
let service = make_tier_mutation_control_server_for_context(None);
let mutation_id = uuid::Uuid::new_v4();
let payload = Bytes::from_static(b"intent");
let unsupported_version = rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION - 1;
let mut request = Request::new(TierMutationPrepareRequest {
version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION - 1,
version: unsupported_version,
mutation_id: mutation_id.to_string(),
canonical_payload: payload,
});
@@ -3361,6 +3435,38 @@ mod tests {
.await
.expect_err("old tier mutation protocol version must fail closed");
assert_eq!(error.code(), tonic::Code::FailedPrecondition);
assert_eq!(
error.message(),
format!("unsupported tier mutation peer protocol version: {}", unsupported_version)
);
}
#[tokio::test]
async fn tier_mutation_control_accepts_v3_before_store_lookup() {
let service = make_tier_mutation_control_server_for_context(None);
let mutation_id = uuid::Uuid::new_v4();
let payload = Bytes::from_static(b"intent");
let mut request = Request::new(TierMutationPrepareRequest {
version: rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
mutation_id: mutation_id.to_string(),
canonical_payload: payload,
});
let body = rustfs_protos::canonical_tier_mutation_rpc_body(
request.get_ref().version,
rustfs_protos::TierMutationRpcPhase::Prepare,
mutation_id,
&request.get_ref().canonical_payload,
)
.expect("v3 request should encode");
set_tonic_canonical_body_digest(&mut request, &body).expect("digest metadata should encode");
mark_v2_authenticated(&mut request);
let error = service
.prepare_tier_mutation(request)
.await
.expect_err("accepted v3 request should reach the missing-store check");
assert_eq!(error.code(), tonic::Code::FailedPrecondition);
assert_eq!(error.message(), "tier mutation object store is not initialized");
}
#[tokio::test]
@@ -3379,6 +3485,32 @@ mod tests {
assert_eq!(error.code(), tonic::Code::InvalidArgument);
}
#[tokio::test]
async fn tier_mutation_control_rejects_invalid_abort_payload_before_auth_and_store_lookup() {
let service = make_tier_mutation_control_server_for_context(None);
let mutation_id = uuid::Uuid::new_v4();
let empty = service
.abort_tier_mutation(Request::new(TierMutationAbortRequest {
version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
mutation_id: mutation_id.to_string(),
canonical_payload: Bytes::new(),
}))
.await
.expect_err("empty abort must fail before digest construction");
assert_eq!(empty.code(), tonic::Code::InvalidArgument);
let oversized = Bytes::from(vec![0; rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE + 1]);
let error = service
.abort_tier_mutation(Request::new(TierMutationAbortRequest {
version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
mutation_id: mutation_id.to_string(),
canonical_payload: oversized,
}))
.await
.expect_err("oversized abort must fail before digest construction");
assert_eq!(error.code(), tonic::Code::InvalidArgument);
}
#[test]
fn tier_mutation_peer_state_wire_constants_match_generated_proto() {
assert_eq!(
@@ -3390,6 +3522,19 @@ mod tests {
assert_eq!(super::TIER_MUTATION_PEER_STATE_ABORTED_WIRE, TierMutationPeerState::Aborted as i32);
}
#[test]
fn tier_mutation_error_info_bound_preserves_utf8_and_byte_limit() {
let ascii = "a".repeat(rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE + 1);
let bounded = super::bounded_tier_mutation_error_info(ascii);
assert_eq!(bounded.len(), rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE);
let unicode = "".repeat(rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE);
let bounded = super::bounded_tier_mutation_error_info(unicode);
assert!(bounded.len() <= rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE);
assert!(bounded.is_char_boundary(bounded.len()));
assert!(bounded.chars().all(|character| character == '界'));
}
#[test]
fn tier_mutation_control_response_proof_binds_request_and_result() {
let _ = rustfs_credentials::set_global_rpc_secret("tier-mutation-control-response-proof-test-secret".to_string());
@@ -3404,6 +3549,7 @@ mod tests {
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("store failed".to_string()),
failure_class: TierMutationFailureClass::Ambiguous as i32,
})
.expect("response proof should be signed")
.into_inner();
@@ -3417,6 +3563,7 @@ mod tests {
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("store failed"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
})
.expect("small mutation response should encode");
crate::storage::storage_api::verify_tonic_rpc_response_proof(&canonical, &response.response_proof)
@@ -3432,6 +3579,7 @@ mod tests {
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("store failed"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
})
.expect("small mutation response should encode");
let error = crate::storage::storage_api::verify_tonic_rpc_response_proof(&tampered, &response.response_proof)
@@ -3599,7 +3747,7 @@ mod tests {
}
#[tokio::test]
async fn cross_pool_fence_probe_authenticates_supported_v2_state() {
async fn cross_pool_fence_probe_authenticates_supported_v3_state() {
let _ = rustfs_credentials::set_global_rpc_secret("cross-pool-fence-node-service-test-secret".to_string());
let endpoints = heal_control_test_endpoints_with_coordinator("node-0", true);
assert!(
@@ -3664,7 +3812,7 @@ mod tests {
assert!(response.success);
assert_eq!(response.error_info, None);
assert_eq!(&response.result[..4], &2_u32.to_be_bytes());
assert_eq!(&response.result[..4], &CROSS_POOL_FENCE_SUPPORTED_VERSION.to_be_bytes());
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(&response.result[4..])
.expect("capability identity should decode");
assert_eq!(topology_member, "node-a:9000");
+2
View File
@@ -404,6 +404,8 @@ pub(crate) mod ecstore_admin {
}
pub(crate) mod ecstore_bucket {
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::tier_delete_journal::test_util::install_all_v6_fleet_capability_proof;
pub(crate) use rustfs_ecstore::api::bucket::{
bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, migration, object_lock, policy_sys,
replication, tagging, target, utils,
+3 -3
View File
@@ -21,12 +21,12 @@
2|crates/ecstore/src/bucket/replication/replication_target_boundary.rs
3|crates/ecstore/src/cluster/rpc/http_auth.rs
2|crates/ecstore/src/cluster/rpc/internode_data_transport.rs
11|crates/ecstore/src/cluster/rpc/peer_rest_client.rs
10|crates/ecstore/src/cluster/rpc/peer_rest_client.rs
10|crates/ecstore/src/cluster/rpc/peer_s3_client.rs
10|crates/ecstore/src/cluster/rpc/remote_disk.rs
6|crates/ecstore/src/config/com.rs
14|crates/ecstore/src/config/storageclass.rs
183|crates/ecstore/src/core/pools.rs
182|crates/ecstore/src/core/pools.rs
8|crates/ecstore/src/data_movement/mod.rs
2|crates/ecstore/src/data_usage/local_snapshot.rs
12|crates/ecstore/src/data_usage/mod.rs
@@ -70,5 +70,5 @@
12|crates/ecstore/src/store/init.rs
2|crates/ecstore/src/store/init_format.rs
3|crates/ecstore/src/store/multipart.rs
8|crates/ecstore/src/store/object.rs
7|crates/ecstore/src/store/object.rs
5|crates/ecstore/src/store/rebalance/support.rs