mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-06 12:09:12 +00:00
fix(ecstore): make transitioned cleanup crash-safe (#6978)
* fix(ecstore): fence transitioned object cleanup * fix(ecstore): address ILM recovery review findings * fix(ecstore): complete crash-safe tier cleanup recovery * test(ecstore): avoid typo false positive * fix(ecstore): stabilize decommission error buckets * fix(ecstore): stabilize transition delete validation * fix(ecstore): resume authorized tier delete dispatch * fix(ecstore): satisfy feature clippy
This commit is contained in:
@@ -65,7 +65,7 @@ command = ['sh', '-c', 'echo RUST_MIN_STACK=33554432 >> "$NEXTEST_ENV"']
|
||||
|
||||
# --- default profile (local): serialize the flaky groups, never retry --------
|
||||
[[profile.default.scripts]]
|
||||
filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)'
|
||||
filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)'
|
||||
setup = 'ecstore-large-stack'
|
||||
|
||||
[[profile.default.scripts]]
|
||||
@@ -183,7 +183,7 @@ fail-fast = false
|
||||
path = "junit.xml"
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)'
|
||||
filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)'
|
||||
setup = 'ecstore-large-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
|
||||
@@ -75,6 +75,10 @@ pub mod bucket {
|
||||
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
|
||||
inspect_transition_transaction_for_operator,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::bucket::lifecycle::transition_transaction::{
|
||||
TransitionTransactionRecoveryStats, recover_transition_transaction_records,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod evaluator {
|
||||
@@ -99,6 +103,17 @@ pub mod bucket {
|
||||
pub use crate::bucket::lifecycle::tier_delete_journal::{
|
||||
persist_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
|
||||
};
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub mod test_util {
|
||||
/// Model a single-node, all-v6 fleet after its capability probe has completed.
|
||||
///
|
||||
/// Call this only once while constructing an isolated test store, before any
|
||||
/// tier-delete journal permit or background worker can be active.
|
||||
pub fn install_all_v6_fleet_capability_proof() {
|
||||
crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub mod tier_last_day_stats {
|
||||
|
||||
@@ -41,7 +41,7 @@ use crate::bucket::lifecycle::tier_free_version_recovery::{
|
||||
DEFAULT_FREE_VERSION_RECOVERY_LIMIT, FreeVersionRecoveryStats, recover_tier_free_versions_with_cancel,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
|
||||
use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent_with_manager_and_identity};
|
||||
use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_with_lease_idempotent};
|
||||
use crate::bucket::lifecycle::transition_transaction::run_transition_transaction_recovery_loop;
|
||||
use crate::bucket::object_lock::ObjectLockApi;
|
||||
use crate::bucket::versioning::VersioningApi as _;
|
||||
@@ -50,7 +50,10 @@ use crate::disk::error::DiskError;
|
||||
use crate::disk::{DeleteOptions, Disk, DiskAPI, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, STORAGE_FORMAT_FILE};
|
||||
use crate::error::Error;
|
||||
use crate::error::StorageError;
|
||||
use crate::error::{is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, is_network_or_host_down};
|
||||
use crate::error::{
|
||||
is_err_object_not_found, is_err_read_quorum, is_err_strict_volume_not_found, is_err_version_not_found,
|
||||
is_network_or_host_down,
|
||||
};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
|
||||
use crate::object_api::{ObjectEncryptionResolver, ReadPlan};
|
||||
use crate::services::tier::{
|
||||
@@ -586,25 +589,217 @@ impl ExpiryOp for FreeVersionTask {
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_free_version_remote_object(
|
||||
async fn acquire_free_version_tier_lease(
|
||||
oi: &ObjectInfo,
|
||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||
) -> Result<(), std::io::Error> {
|
||||
) -> Result<(TierOperationLease, bool), std::io::Error> {
|
||||
let version_id_exact = validate_transition_remote_version(oi)?;
|
||||
let identity = tier_destination_id_from_metadata(&oi.user_defined)?
|
||||
.ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?;
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
|
||||
let lease =
|
||||
TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, &oi.transitioned_object.tier, identity)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
Ok((lease, version_id_exact))
|
||||
}
|
||||
|
||||
async fn delete_free_version_remote_object_with_lease(
|
||||
oi: &ObjectInfo,
|
||||
lease: &TierOperationLease,
|
||||
version_id_exact: bool,
|
||||
) -> Result<(), std::io::Error> {
|
||||
delete_object_from_remote_tier_with_lease_idempotent(
|
||||
&oi.transitioned_object.name,
|
||||
&oi.transitioned_object.version_id,
|
||||
&oi.transitioned_object.tier,
|
||||
identity,
|
||||
tier_config_mgr,
|
||||
lease,
|
||||
version_id_exact,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn free_version_physical_topology_generation(api: &ECStore) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
for pool in &api.pools {
|
||||
hasher.update(pool.pool_idx.to_be_bytes());
|
||||
hasher.update(pool.disk_set.len().to_be_bytes());
|
||||
for set in &pool.disk_set {
|
||||
hasher.update(set.set_index.to_be_bytes());
|
||||
}
|
||||
}
|
||||
rustfs_utils::crypto::hex(hasher.finalize().as_slice())
|
||||
}
|
||||
|
||||
fn free_version_remote_tuple_matches(candidate: &ObjectInfo, expected: &ObjectInfo) -> std::io::Result<bool> {
|
||||
if candidate.transitioned_object.tier != expected.transitioned_object.tier
|
||||
|| candidate.transitioned_object.name != expected.transitioned_object.name
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let candidate_identity = tier_destination_id_from_metadata(&candidate.user_defined)?
|
||||
.ok_or_else(|| std::io::Error::other("tier free-version is missing its backend identity"))?;
|
||||
let expected_identity = tier_destination_id_from_metadata(&expected.user_defined)?
|
||||
.ok_or_else(|| std::io::Error::other("tier free-version task is missing its backend identity"))?;
|
||||
if candidate_identity != expected_identity {
|
||||
return Ok(false);
|
||||
}
|
||||
if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|
||||
|| expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|
||||
{
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier free-version remote version state is unknown",
|
||||
));
|
||||
}
|
||||
Ok(candidate.transition_version_state == expected.transition_version_state
|
||||
&& candidate.transitioned_object.version_id == expected.transitioned_object.version_id)
|
||||
}
|
||||
|
||||
async fn scan_exact_free_version_targets(
|
||||
api: &ECStore,
|
||||
oi: &ObjectInfo,
|
||||
local_object: &str,
|
||||
) -> std::io::Result<Vec<(Arc<SetDisks>, FileInfo)>> {
|
||||
let mut targets = Vec::new();
|
||||
for pool in &api.pools {
|
||||
for set in &pool.disk_set {
|
||||
let versions = match set.load_file_info_versions_exact(&oi.bucket, &oi.name).await {
|
||||
Ok(Some(versions)) => versions,
|
||||
Ok(None) => continue,
|
||||
Err(err) if is_err_strict_volume_not_found(&err) => continue,
|
||||
Err(err) => return Err(std::io::Error::other(err)),
|
||||
};
|
||||
for version in versions.versions.iter().chain(versions.free_versions.iter()) {
|
||||
let candidate = ObjectInfo::from_file_info(version, &oi.bucket, &oi.name, true);
|
||||
if free_version_remote_tuple_matches(&candidate, oi)? {
|
||||
if candidate.transitioned_object.free_version {
|
||||
// Data movement can leave the same remote tuple in
|
||||
// several physical pools. Ordinary deletion assigns a
|
||||
// fresh local free-version UUID to each copy, but all
|
||||
// of those markers own the same idempotent remote
|
||||
// DELETE. Consume them together while holding every
|
||||
// physical object lock; treating their local UUIDs as
|
||||
// conflicting would strand cleanup forever.
|
||||
let mut actual = version.clone();
|
||||
actual.name = local_object.to_string();
|
||||
targets.push((Arc::clone(set), actual));
|
||||
} else {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"a live transitioned source still references the free-version remote tuple",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(targets)
|
||||
}
|
||||
|
||||
fn free_version_cleanup_fences_current(
|
||||
topology_generation: &str,
|
||||
api: &ECStore,
|
||||
bucket_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
object_guards: &[crate::store::ObjectLockDiagGuard],
|
||||
lease: &TierOperationLease,
|
||||
cancel: &CancellationToken,
|
||||
deadline: tokio::time::Instant,
|
||||
) -> bool {
|
||||
!cancel.is_cancelled()
|
||||
&& tokio::time::Instant::now() < deadline
|
||||
&& !bucket_guard.is_lock_lost()
|
||||
&& object_guards.iter().all(|guard| !guard.is_lock_lost())
|
||||
&& lease.is_current_generation()
|
||||
&& free_version_physical_topology_generation(api) == topology_generation
|
||||
}
|
||||
|
||||
async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel: &CancellationToken) -> std::io::Result<bool> {
|
||||
const FREE_VERSION_REMOTE_DEADLINE: StdDuration = StdDuration::from_secs(30);
|
||||
|
||||
let topology_generation = free_version_physical_topology_generation(&api);
|
||||
let bucket_guard = api
|
||||
.acquire_bucket_lifecycle_read_lock(&oi.bucket)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
|
||||
let local_object = encode_dir_object(&oi.name);
|
||||
let object_guards = api
|
||||
.acquire_all_physical_object_write_locks("tier_free_version_cleanup", &oi.bucket, &local_object)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
let targets = scan_exact_free_version_targets(&api, oi, &local_object).await?;
|
||||
if targets.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let deadline = tokio::time::Instant::now() + FREE_VERSION_REMOTE_DEADLINE;
|
||||
if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier free-version cleanup fence is invalid before remote delete",
|
||||
));
|
||||
}
|
||||
tokio::select! {
|
||||
_ = cancel.cancelled() => {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled"));
|
||||
}
|
||||
result = tokio::time::timeout_at(
|
||||
deadline,
|
||||
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact),
|
||||
) => {
|
||||
result
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out"))??;
|
||||
}
|
||||
}
|
||||
if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
|
||||
// Remote DELETE is idempotent, but a changed fence makes the local
|
||||
// outcome ambiguous. Keep every marker for a fully fenced retry.
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier free-version cleanup fence changed after remote delete",
|
||||
));
|
||||
}
|
||||
|
||||
let mut first_error = None;
|
||||
for (set, actual) in &targets {
|
||||
let mut delete_request = FileInfo {
|
||||
name: local_object.clone(),
|
||||
version_id: actual.version_id,
|
||||
..Default::default()
|
||||
};
|
||||
delete_request.set_tier_free_version();
|
||||
if let Err(err) = set
|
||||
.delete_object_version(&oi.bucket, &local_object, &delete_request, false)
|
||||
.await
|
||||
&& first_error.is_none()
|
||||
{
|
||||
first_error = Some(std::io::Error::other(err));
|
||||
}
|
||||
}
|
||||
let remaining = scan_exact_free_version_targets(&api, oi, &local_object).await?;
|
||||
if !remaining.is_empty() {
|
||||
return Err(first_error.unwrap_or_else(|| {
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier free-version cleanup remained on at least one physical set",
|
||||
)
|
||||
}));
|
||||
}
|
||||
if let Some(err) = first_error {
|
||||
return Err(err);
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
async fn delete_free_version_remote_object(
|
||||
oi: &ObjectInfo,
|
||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||
) -> Result<(), std::io::Error> {
|
||||
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
|
||||
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
@@ -618,8 +813,11 @@ where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = T>,
|
||||
{
|
||||
delete_free_version_remote_object(oi, tier_config_mgr).await?;
|
||||
Ok(delete_local().await)
|
||||
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
|
||||
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await?;
|
||||
let result = delete_local().await;
|
||||
drop(lease);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
struct NewerNoncurrentTask {
|
||||
@@ -690,6 +888,10 @@ impl ExpiryState {
|
||||
usize::try_from(self.stats.pending_tasks().max(0)).unwrap_or(usize::MAX)
|
||||
}
|
||||
|
||||
pub fn active_tasks(&self) -> usize {
|
||||
usize::try_from(self.stats.active_tasks().max(0)).unwrap_or(usize::MAX)
|
||||
}
|
||||
|
||||
fn send_expiry_task(&self, wrkr: Sender<Option<ExpiryOpType>>, task: ExpiryOpType) -> bool {
|
||||
let queued = wrkr.try_send(Some(task)).is_ok();
|
||||
if queued {
|
||||
@@ -826,7 +1028,7 @@ impl ExpiryState {
|
||||
}
|
||||
|
||||
pub async fn resize_workers(n: usize, api: Arc<ECStore>) {
|
||||
let expiry_state = runtime_sources::expiry_state_handle();
|
||||
let expiry_state = api.ctx.expiry_state();
|
||||
if n == expiry_state.read().await.tasks_tx.len() || n < 1 {
|
||||
return;
|
||||
}
|
||||
@@ -867,7 +1069,7 @@ impl ExpiryState {
|
||||
stats: Arc<ExpiryStats>,
|
||||
recovery_notify: Arc<Notify>,
|
||||
) {
|
||||
let cancel_token = runtime_sources::background_services_cancel_token().unwrap_or_else(|| {
|
||||
let cancel_token = api.ctx.background_cancel_token().unwrap_or_else(|| {
|
||||
static FALLBACK: std::sync::OnceLock<tokio_util::sync::CancellationToken> = std::sync::OnceLock::new();
|
||||
FALLBACK.get_or_init(tokio_util::sync::CancellationToken::new).clone()
|
||||
});
|
||||
@@ -968,119 +1170,33 @@ impl ExpiryState {
|
||||
else if v.as_any().is::<FreeVersionTask>() {
|
||||
let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("FreeVersionTask downcast failed");
|
||||
let oi = v.0.clone();
|
||||
if let Err(err) = delete_free_version_remote_object(&oi, &api.tier_config_mgr()).await {
|
||||
recovery_notify.notify_one();
|
||||
debug!(
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
remote_object = %oi.transitioned_object.name,
|
||||
remote_version_id = %oi.transitioned_object.version_id,
|
||||
tier = %oi.transitioned_object.tier,
|
||||
error = ?err,
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
reason = "remote_tier_delete_failed",
|
||||
"Lifecycle worker skipped remote tier delete"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
let local_object = encode_dir_object(&oi.name);
|
||||
let mut fi = FileInfo {
|
||||
name: local_object.clone(),
|
||||
version_id: oi.version_id,
|
||||
..Default::default()
|
||||
};
|
||||
// This removes an existing internal cleanup marker. Keeping
|
||||
// `deleted` false makes duplicate tasks return not-found
|
||||
// instead of creating an ordinary delete marker.
|
||||
fi.set_tier_free_version();
|
||||
|
||||
let mut deleted_locally = false;
|
||||
for pool in &api.pools {
|
||||
let set = pool.get_disks_by_key(&local_object);
|
||||
let ns_lock = match set.new_ns_lock(&oi.bucket, &local_object).await {
|
||||
Ok(lock) => lock,
|
||||
Err(err) => {
|
||||
recovery_notify.notify_one();
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
pool_index = pool.pool_idx,
|
||||
set_index = set.set_index,
|
||||
error = ?err,
|
||||
reason = "local_free_version_lock_failed",
|
||||
"Lifecycle worker failed to create local free-version cleanup lock"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let _object_lock_guard =
|
||||
match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => {
|
||||
recovery_notify.notify_one();
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
pool_index = pool.pool_idx,
|
||||
set_index = set.set_index,
|
||||
error = ?err,
|
||||
reason = "local_free_version_lock_failed",
|
||||
"Lifecycle worker failed to acquire local free-version cleanup lock"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
match set
|
||||
.delete_object_version(&oi.bucket, &local_object, &fi, false)
|
||||
.await
|
||||
{
|
||||
Ok(()) => {
|
||||
deleted_locally = true;
|
||||
break;
|
||||
}
|
||||
Err(err) if is_err_version_not_found(&err) || is_err_object_not_found(&err) => continue,
|
||||
Err(err) => {
|
||||
recovery_notify.notify_one();
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
remote_object = %oi.transitioned_object.name,
|
||||
remote_version_id = %oi.transitioned_object.version_id,
|
||||
tier = %oi.transitioned_object.tier,
|
||||
error = ?err,
|
||||
reason = "local_free_version_delete_failed",
|
||||
"Lifecycle worker failed local free-version cleanup"
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !deleted_locally {
|
||||
debug!(
|
||||
match cleanup_free_version_exact(api.clone(), &oi, &cancel_token).await {
|
||||
Ok(true) => {}
|
||||
Ok(false) => debug!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
remote_object = %oi.transitioned_object.name,
|
||||
remote_version_id = %oi.transitioned_object.version_id,
|
||||
tier = %oi.transitioned_object.tier,
|
||||
reason = "local_free_version_missing",
|
||||
"Lifecycle worker could not find transitioned free version locally"
|
||||
);
|
||||
"Lifecycle worker found that the exact free-version was already absent"
|
||||
),
|
||||
Err(err) => {
|
||||
recovery_notify.notify_one();
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
remote_object = %oi.transitioned_object.name,
|
||||
remote_version_id = %oi.transitioned_object.version_id,
|
||||
tier = %oi.transitioned_object.tier,
|
||||
error = ?err,
|
||||
reason = "free_version_exact_cleanup_deferred",
|
||||
"Lifecycle worker retained the exact free-version for a fenced retry"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
@@ -1152,8 +1268,8 @@ fn set_recovered_free_version_enqueue_observer(
|
||||
RecoveredFreeVersionEnqueueObserverGuard
|
||||
}
|
||||
|
||||
pub async fn enqueue_recovered_free_version(oi: ObjectInfo) -> bool {
|
||||
let expiry_state = runtime_sources::expiry_state_handle();
|
||||
pub async fn enqueue_recovered_free_version(api: &ECStore, oi: ObjectInfo) -> bool {
|
||||
let expiry_state = api.ctx.expiry_state();
|
||||
let queued = enqueue_recovered_free_version_with_state(&expiry_state, oi).await;
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -2580,8 +2696,8 @@ fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<(
|
||||
}
|
||||
|
||||
Some(tokio::spawn(async move {
|
||||
let cancel_token = runtime_sources::background_services_cancel_token().unwrap_or_default();
|
||||
let expiry_state = runtime_sources::expiry_state_handle();
|
||||
let cancel_token = api.ctx.background_cancel_token().unwrap_or_default();
|
||||
let expiry_state = api.ctx.expiry_state();
|
||||
run_tier_free_version_recovery_loop(
|
||||
cancel_token,
|
||||
expiry_state,
|
||||
@@ -6229,9 +6345,18 @@ mod tests {
|
||||
rustfs_utils::crypto::hex(old_identity),
|
||||
);
|
||||
oi.user_defined = Arc::new(metadata.clone());
|
||||
let lease_observed_during_local_delete = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
delete_free_version_remote_object_then(&oi, &manager, {
|
||||
let local_delete_calls = Arc::clone(&local_delete_calls);
|
||||
let lease_observed_during_local_delete = Arc::clone(&lease_observed_during_local_delete);
|
||||
let manager = manager.clone();
|
||||
move || async move {
|
||||
assert_eq!(
|
||||
crate::services::tier::tier::TierConfigMgr::active_operation_lease_count(&manager, "WARM").await,
|
||||
1,
|
||||
"the identity-bound tier lease must span the exact local marker delete"
|
||||
);
|
||||
lease_observed_during_local_delete.store(true, Ordering::Relaxed);
|
||||
local_delete_calls.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
})
|
||||
@@ -6239,6 +6364,12 @@ mod tests {
|
||||
.expect("matching destination identity should allow idempotent remote cleanup");
|
||||
assert_eq!(old_backend.remove_count().await, 1);
|
||||
assert_eq!(local_delete_calls.load(Ordering::Relaxed), 1);
|
||||
assert!(lease_observed_during_local_delete.load(Ordering::Relaxed));
|
||||
assert_eq!(
|
||||
crate::services::tier::tier::TierConfigMgr::active_operation_lease_count(&manager, "WARM").await,
|
||||
0,
|
||||
"the tier lease should be released after the local marker delete completes"
|
||||
);
|
||||
|
||||
let mut single_prefix_metadata = HashMap::new();
|
||||
single_prefix_metadata.insert(
|
||||
@@ -6472,6 +6603,7 @@ mod tests {
|
||||
let state = ExpiryState::new();
|
||||
let mut state = state.write().await;
|
||||
let je = Jentry {
|
||||
persisted_version: 0,
|
||||
obj_name: "remote/object".to_string(),
|
||||
version_id: "remote-version".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
@@ -6480,6 +6612,7 @@ mod tests {
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
dispatch: None,
|
||||
};
|
||||
|
||||
let err = state
|
||||
@@ -6620,6 +6753,7 @@ mod tests {
|
||||
let state = ExpiryState::new_with_unconsumed_worker_channel(1);
|
||||
let mut state = state.write().await;
|
||||
let je = Jentry {
|
||||
persisted_version: 0,
|
||||
obj_name: "remote/object".to_string(),
|
||||
version_id: "remote-version".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
@@ -6628,6 +6762,7 @@ mod tests {
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
dispatch: None,
|
||||
};
|
||||
|
||||
state
|
||||
@@ -6759,7 +6894,7 @@ mod tests {
|
||||
};
|
||||
|
||||
assert!(
|
||||
super::enqueue_recovered_free_version(oi).await,
|
||||
super::enqueue_recovered_free_version(&ecstore, oi).await,
|
||||
"the resized production worker queue should accept the task"
|
||||
);
|
||||
stop_tx.send(None).await.expect("worker stop signal should be delivered");
|
||||
@@ -6875,12 +7010,12 @@ mod tests {
|
||||
.await
|
||||
.expect("free-version task should reach the worker");
|
||||
tokio::time::timeout(StdDuration::from_secs(30), async {
|
||||
while remote_backend.remove_count().await == 0 {
|
||||
while stats.active_tasks() == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("worker should complete remote cleanup before taking the local lock");
|
||||
.expect("worker should mark the cleanup task active before the lock assertion");
|
||||
let completed_while_locked = tokio::time::timeout(StdDuration::from_millis(100), async {
|
||||
while stats.active_tasks() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
@@ -6889,7 +7024,12 @@ mod tests {
|
||||
.await;
|
||||
assert!(
|
||||
completed_while_locked.is_err(),
|
||||
"local cleanup must wait while a competing object writer owns the namespace lock"
|
||||
"the cleanup task must wait while a competing object writer owns the namespace lock"
|
||||
);
|
||||
assert_eq!(
|
||||
remote_backend.remove_count().await,
|
||||
0,
|
||||
"the remote tuple must not be deleted before the all-physical namespace fence is acquired"
|
||||
);
|
||||
for disk_path in &disk_paths {
|
||||
assert!(
|
||||
@@ -6900,6 +7040,13 @@ mod tests {
|
||||
}
|
||||
|
||||
drop(object_lock_guard);
|
||||
tokio::time::timeout(StdDuration::from_secs(30), async {
|
||||
while remote_backend.remove_count().await == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("worker should delete the remote tuple after acquiring the released namespace fence");
|
||||
tx.send(None).await.expect("worker stop signal should be delivered");
|
||||
worker.await.expect("free-version worker should stop cleanly");
|
||||
|
||||
@@ -6995,6 +7142,7 @@ mod tests {
|
||||
.next()
|
||||
.expect("seeded free version should be recoverable");
|
||||
let stale_version_id = oi.version_id.expect("free version should have a concrete UUID");
|
||||
let ordinary_marker_mod_time = OffsetDateTime::now_utc();
|
||||
|
||||
for disk_path in &disk_paths {
|
||||
let metadata_path = disk_path.join(&bucket).join(object).join(STORAGE_FORMAT_FILE);
|
||||
@@ -7017,7 +7165,7 @@ mod tests {
|
||||
name: object.to_string(),
|
||||
version_id: Some(stale_version_id),
|
||||
deleted: true,
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
mod_time: Some(ordinary_marker_mod_time),
|
||||
..Default::default()
|
||||
})
|
||||
.expect("same-ID ordinary marker should replace the stale free version");
|
||||
@@ -7031,6 +7179,13 @@ mod tests {
|
||||
.expect("same-ID ordinary marker metadata should be written");
|
||||
}
|
||||
|
||||
assert!(
|
||||
!super::cleanup_free_version_exact(Arc::clone(&ecstore), &oi, &CancellationToken::new())
|
||||
.await
|
||||
.expect("a stale task whose local UUID now names an ordinary marker should be an idempotent no-op"),
|
||||
"the stale free-version task must not report local cleanup"
|
||||
);
|
||||
|
||||
let state = ExpiryState::new();
|
||||
let (stats, recovery_notify) = {
|
||||
let state = state.read().await;
|
||||
@@ -11522,7 +11677,7 @@ mod tests {
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn journal_replay_rejects_unknown_version_state_before_backend_io() {
|
||||
async fn journal_replay_quarantines_legacy_unknown_version_state_before_backend_io() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let (backend, _) = register_recovery_mock_tier(&ecstore).await;
|
||||
let identity = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
|
||||
@@ -11530,6 +11685,7 @@ mod tests {
|
||||
.expect("mock tier lease should be available")
|
||||
.backend_identity();
|
||||
let je = Jentry {
|
||||
persisted_version: 0,
|
||||
obj_name: "remote/object".to_string(),
|
||||
version_id: "legacy-version".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
@@ -11538,25 +11694,28 @@ mod tests {
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
dispatch: None,
|
||||
};
|
||||
|
||||
crate::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(ecstore.clone(), &je)
|
||||
.await
|
||||
.expect("legacy unknown journal should remain byte-compatible and persistable");
|
||||
let err = crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
|
||||
.await
|
||||
.expect_err("unknown journal state must fail before backend IO");
|
||||
.expect_err("legacy unknown journal must be quarantined before backend IO");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn journal_replay_deletes_confirmed_exact_provider_token() {
|
||||
async fn rejected_upload_cleanup_retries_confirmed_exact_provider_token_without_legacy_journal() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let (backend, _) = register_recovery_mock_tier(&ecstore).await;
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
|
||||
.await
|
||||
.expect("mock tier lease should be available");
|
||||
let identity = lease.backend_identity();
|
||||
backend
|
||||
.set_put_remote_version(Some("provider-version-token".to_string()))
|
||||
.await;
|
||||
@@ -11570,34 +11729,30 @@ mod tests {
|
||||
.expect("confirmed remote candidate should be seeded");
|
||||
backend.set_remove_failure(true);
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
let je = Jentry {
|
||||
obj_name: "remote/object".to_string(),
|
||||
version_id: "provider-version-token".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some(identity),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
};
|
||||
|
||||
crate::set_disk::cleanup_rejected_transition_upload_durably(
|
||||
let err = crate::set_disk::cleanup_rejected_transition_upload_durably(
|
||||
&lease,
|
||||
&je.obj_name,
|
||||
&je.version_id,
|
||||
"remote/object",
|
||||
"provider-version-token",
|
||||
true,
|
||||
Some(ecstore.clone()),
|
||||
)
|
||||
.await
|
||||
.expect("failed immediate cleanup should remain durable in the journal");
|
||||
assert!(backend.contains(&je.obj_name).await);
|
||||
.expect_err("a failed immediate cleanup must remain owned by the caller's transition transaction");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::Other);
|
||||
assert!(backend.contains("remote/object").await);
|
||||
|
||||
backend.set_remove_failure(false);
|
||||
crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
|
||||
.await
|
||||
.expect("identity-bound exact journal must retry confirmed candidate cleanup");
|
||||
crate::set_disk::cleanup_rejected_transition_upload_durably(
|
||||
&lease,
|
||||
"remote/object",
|
||||
"provider-version-token",
|
||||
true,
|
||||
Some(ecstore),
|
||||
)
|
||||
.await
|
||||
.expect("the transaction retry must delete the same confirmed candidate");
|
||||
|
||||
assert!(!backend.contains(&je.obj_name).await);
|
||||
assert!(!backend.contains("remote/object").await);
|
||||
assert_eq!(backend.exact_remove_count(), 2);
|
||||
assert_eq!(
|
||||
backend.remove_versions().await,
|
||||
@@ -11760,11 +11915,14 @@ mod tests {
|
||||
};
|
||||
let mut recovery_rx = recovery_rx.lock().await;
|
||||
assert!(
|
||||
super::enqueue_recovered_free_version(ObjectInfo {
|
||||
bucket: "prefill".to_string(),
|
||||
name: "prefill".to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
super::enqueue_recovered_free_version(
|
||||
&ecstore,
|
||||
ObjectInfo {
|
||||
bucket: "prefill".to_string(),
|
||||
name: "prefill".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await,
|
||||
"the production recovery queue should accept its first task"
|
||||
);
|
||||
|
||||
@@ -33,6 +33,7 @@ const MANUAL_TRANSITION_CURSOR_MARKER_PROOF_MAX_SIZE: usize = 1024;
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum DurableIlmRecordKind {
|
||||
TierDeleteJournal,
|
||||
TierDeleteDispatchManifest,
|
||||
TransitionTransaction,
|
||||
ManualTransitionJob,
|
||||
ManualTransitionScope,
|
||||
@@ -54,6 +55,18 @@ pub(crate) const TIER_DELETE_JOURNAL_NAMESPACE: DurableIlmNamespace = DurableIlm
|
||||
max_record_size: 64 * 1024,
|
||||
kind: DurableIlmRecordKind::TierDeleteJournal,
|
||||
};
|
||||
pub(crate) const TIER_DELETE_JOURNAL_V6_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
|
||||
name: "tier-delete-journal-v6",
|
||||
prefix: "ilm/tier-delete-journal-v6/",
|
||||
max_record_size: 64 * 1024,
|
||||
kind: DurableIlmRecordKind::TierDeleteJournal,
|
||||
};
|
||||
pub(crate) const TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
|
||||
name: "tier-delete-dispatch-manifest",
|
||||
prefix: tier_delete_journal::TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
|
||||
max_record_size: tier_delete_journal::MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE,
|
||||
kind: DurableIlmRecordKind::TierDeleteDispatchManifest,
|
||||
};
|
||||
pub(crate) const TRANSITION_TRANSACTION_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
|
||||
name: "transition-transaction",
|
||||
prefix: "ilm/transition-transactions/records",
|
||||
@@ -85,8 +98,10 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace
|
||||
kind: DurableIlmRecordKind::ManualTransitionWorkerResult,
|
||||
};
|
||||
|
||||
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 6] = [
|
||||
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 8] = [
|
||||
TIER_DELETE_JOURNAL_NAMESPACE,
|
||||
TIER_DELETE_JOURNAL_V6_NAMESPACE,
|
||||
TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
|
||||
TRANSITION_TRANSACTION_NAMESPACE,
|
||||
MANUAL_TRANSITION_JOB_NAMESPACE,
|
||||
MANUAL_TRANSITION_SCOPE_NAMESPACE,
|
||||
@@ -157,6 +172,15 @@ pub(crate) enum DurableIlmRecordCheckpoint {
|
||||
content_sha256: String,
|
||||
identity_sha256: String,
|
||||
committed: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
dispatch_identity_sha256: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
state: Option<super::tier_sweeper::TierDeleteJournalState>,
|
||||
},
|
||||
TierDeleteDispatchManifest {
|
||||
content_sha256: String,
|
||||
identity_sha256: String,
|
||||
state: tier_delete_journal::TierDeleteDispatchManifestState,
|
||||
},
|
||||
TransitionTransaction {
|
||||
content_sha256: String,
|
||||
@@ -195,6 +219,7 @@ impl DurableIlmRecordCheckpoint {
|
||||
pub(crate) fn content_sha256(&self) -> &str {
|
||||
match self {
|
||||
Self::TierDeleteJournal { content_sha256, .. }
|
||||
| Self::TierDeleteDispatchManifest { content_sha256, .. }
|
||||
| Self::TransitionTransaction { content_sha256, .. }
|
||||
| Self::ManualTransitionJob { content_sha256, .. }
|
||||
| Self::ManualTransitionScope { content_sha256, .. }
|
||||
@@ -228,6 +253,19 @@ impl DurableIlmRecordCheckpoint {
|
||||
}
|
||||
|
||||
pub(crate) fn validate_successor(&self, next: &Self) -> Result<()> {
|
||||
for checkpoint in [self, next] {
|
||||
if let Self::TierDeleteJournal {
|
||||
committed,
|
||||
dispatch_identity_sha256,
|
||||
state,
|
||||
..
|
||||
} = checkpoint
|
||||
&& (state.is_some() != dispatch_identity_sha256.is_some()
|
||||
|| state.is_some_and(|state| *committed != (state == super::tier_sweeper::TierDeleteJournalState::Committed)))
|
||||
{
|
||||
return Err(Error::other("durable ILM tier delete journal checkpoint is invalid"));
|
||||
}
|
||||
}
|
||||
if self == next {
|
||||
if let Self::ManualTransitionJob {
|
||||
progress,
|
||||
@@ -244,18 +282,64 @@ impl DurableIlmRecordCheckpoint {
|
||||
let valid = match (self, next) {
|
||||
(
|
||||
Self::TierDeleteJournal {
|
||||
content_sha256: previous_content,
|
||||
identity_sha256: previous_identity,
|
||||
committed: previous_committed,
|
||||
dispatch_identity_sha256: previous_dispatch_identity,
|
||||
state: previous_state,
|
||||
..
|
||||
},
|
||||
Self::TierDeleteJournal {
|
||||
content_sha256: next_content,
|
||||
identity_sha256: next_identity,
|
||||
committed: next_committed,
|
||||
dispatch_identity_sha256: next_dispatch_identity,
|
||||
state: next_state,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
use super::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared};
|
||||
|
||||
let dispatch_identity_is_monotonic = match (previous_dispatch_identity, next_dispatch_identity) {
|
||||
(Some(previous), Some(next)) => previous == next,
|
||||
(None, None) => true,
|
||||
// Old receipts did not record the v6 dispatch binding. A
|
||||
// byte-identical observation may adopt the stronger proof,
|
||||
// but an in-flight mutation must fail closed instead of
|
||||
// guessing which operation owned the journal.
|
||||
(None, Some(_)) => previous_content == next_content,
|
||||
(Some(_), None) => false,
|
||||
};
|
||||
let state_is_monotonic = match (previous_state, next_state) {
|
||||
(Some(previous), Some(next)) => {
|
||||
previous == next || matches!((previous, next), (Prepared, Dispatched) | (Dispatched, Committed))
|
||||
}
|
||||
(None, None) => previous_committed == next_committed || (!previous_committed && *next_committed),
|
||||
(None, Some(_)) => previous_content == next_content,
|
||||
(Some(_), None) => false,
|
||||
};
|
||||
previous_identity == next_identity && dispatch_identity_is_monotonic && state_is_monotonic
|
||||
}
|
||||
(
|
||||
Self::TierDeleteDispatchManifest {
|
||||
identity_sha256: previous_identity,
|
||||
state: previous_state,
|
||||
..
|
||||
},
|
||||
Self::TierDeleteDispatchManifest {
|
||||
identity_sha256: next_identity,
|
||||
state: next_state,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
use tier_delete_journal::TierDeleteDispatchManifestState::{
|
||||
Aborted, Aborting, Completed, DispatchAuthorized, Preparing,
|
||||
};
|
||||
previous_identity == next_identity
|
||||
&& (previous_committed == next_committed || (!previous_committed && *next_committed))
|
||||
&& matches!(
|
||||
(previous_state, next_state),
|
||||
(Preparing, DispatchAuthorized | Aborting) | (Aborting, Aborted) | (DispatchAuthorized, Completed)
|
||||
)
|
||||
}
|
||||
(
|
||||
Self::TransitionTransaction {
|
||||
@@ -351,6 +435,49 @@ impl DurableIlmRecordCheckpoint {
|
||||
Err(Error::other("durable ILM record generation is not a monotonic successor"))
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `self` is an older generation of the same immutable record
|
||||
/// that can reach `terminal` through one or more valid state transitions.
|
||||
/// This is deliberately broader than `validate_successor`, which remains
|
||||
/// adjacent-only for receipt advancement. Terminal cleanup uses this only
|
||||
/// after the exact terminal ETag and terminal receipt were committed, to
|
||||
/// purge older object versions exposed by that deletion.
|
||||
pub(crate) fn is_predecessor_of_terminal(&self, terminal: &Self) -> bool {
|
||||
if self == terminal || self.validate_successor(terminal).is_ok() {
|
||||
return true;
|
||||
}
|
||||
match (self, terminal) {
|
||||
(
|
||||
Self::TierDeleteJournal {
|
||||
identity_sha256: previous_identity,
|
||||
dispatch_identity_sha256: previous_dispatch,
|
||||
state: Some(super::tier_sweeper::TierDeleteJournalState::Prepared),
|
||||
..
|
||||
},
|
||||
Self::TierDeleteJournal {
|
||||
identity_sha256: terminal_identity,
|
||||
dispatch_identity_sha256: terminal_dispatch,
|
||||
state: Some(super::tier_sweeper::TierDeleteJournalState::Committed),
|
||||
..
|
||||
},
|
||||
) => previous_identity == terminal_identity && previous_dispatch == terminal_dispatch,
|
||||
(
|
||||
Self::TierDeleteDispatchManifest {
|
||||
identity_sha256: previous_identity,
|
||||
state: tier_delete_journal::TierDeleteDispatchManifestState::Preparing,
|
||||
..
|
||||
},
|
||||
Self::TierDeleteDispatchManifest {
|
||||
identity_sha256: terminal_identity,
|
||||
state:
|
||||
tier_delete_journal::TierDeleteDispatchManifestState::Aborted
|
||||
| tier_delete_journal::TierDeleteDispatchManifestState::Completed,
|
||||
..
|
||||
},
|
||||
) => previous_identity == terminal_identity,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn transition_state_distance(
|
||||
@@ -750,10 +877,19 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
|
||||
if tier_delete_journal::tier_delete_journal_object_name(&entry) != path {
|
||||
return Err(Error::other("tier delete journal content does not match its path"));
|
||||
}
|
||||
let operation_id = path
|
||||
let legacy_operation_id = path
|
||||
.strip_prefix(namespace.prefix)
|
||||
.and_then(|suffix| suffix.strip_suffix(".json"))
|
||||
.ok_or_else(|| Error::other("tier delete journal path is invalid"))?;
|
||||
// Legacy v1-v5 paths already expose a 64-hex operation id and
|
||||
// must remain receipt-compatible. V6 uses an operation-scoped
|
||||
// nested path, so derive a fixed, path-unique receipt id instead
|
||||
// of embedding slashes in the receipt locator.
|
||||
let operation_id = if entry.persisted_version == 6 {
|
||||
hex_sha256(path.as_bytes(), ToOwned::to_owned)
|
||||
} else {
|
||||
legacy_operation_id.to_string()
|
||||
};
|
||||
let identity_sha256 = checkpoint_hash(&(
|
||||
&entry.obj_name,
|
||||
&entry.version_id,
|
||||
@@ -763,13 +899,29 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
|
||||
entry.version_state,
|
||||
&entry.source,
|
||||
))?;
|
||||
let dispatch_identity_sha256 = entry.dispatch.as_ref().map(checkpoint_hash).transpose()?;
|
||||
(
|
||||
"operation_id",
|
||||
operation_id.to_string(),
|
||||
operation_id,
|
||||
DurableIlmRecordCheckpoint::TierDeleteJournal {
|
||||
content_sha256,
|
||||
identity_sha256,
|
||||
committed: entry.state == super::tier_sweeper::TierDeleteJournalState::Committed,
|
||||
dispatch_identity_sha256,
|
||||
state: (entry.persisted_version == 6).then_some(entry.state),
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::TierDeleteDispatchManifest => {
|
||||
let (operation_id, identity_sha256, state) =
|
||||
tier_delete_journal::validate_tier_delete_dispatch_manifest_record(path, data)?;
|
||||
(
|
||||
"operation_id",
|
||||
hex_sha256(operation_id.as_bytes(), ToOwned::to_owned),
|
||||
DurableIlmRecordCheckpoint::TierDeleteDispatchManifest {
|
||||
content_sha256,
|
||||
identity_sha256,
|
||||
state,
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -956,6 +1108,87 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_dispatch_manifest_namespace_validates_monotonic_branches() {
|
||||
use tier_delete_journal::TierDeleteDispatchManifestState::{Aborted, Aborting, Completed, DispatchAuthorized, Preparing};
|
||||
|
||||
let operation_id = Uuid::new_v4();
|
||||
let checkpoint = |state| {
|
||||
let (path, data) = tier_delete_journal::test_tier_delete_dispatch_manifest_record(operation_id, state);
|
||||
let namespace = classify_durable_ilm_record(&path)
|
||||
.expect("dispatch manifest namespace should classify")
|
||||
.expect("dispatch manifest should be durable");
|
||||
assert_eq!(namespace, &TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE);
|
||||
validate_durable_ilm_record(&path, &data)
|
||||
.expect("dispatch manifest should validate")
|
||||
.checkpoint
|
||||
};
|
||||
|
||||
let preparing = checkpoint(Preparing);
|
||||
let authorized = checkpoint(DispatchAuthorized);
|
||||
let completed = checkpoint(Completed);
|
||||
let aborting = checkpoint(Aborting);
|
||||
let aborted = checkpoint(Aborted);
|
||||
|
||||
preparing
|
||||
.validate_successor(&authorized)
|
||||
.expect("Preparing may become DispatchAuthorized");
|
||||
authorized
|
||||
.validate_successor(&completed)
|
||||
.expect("DispatchAuthorized may become Completed");
|
||||
preparing.validate_successor(&aborting).expect("Preparing may enter rollback");
|
||||
aborting.validate_successor(&aborted).expect("Aborting may become Aborted");
|
||||
assert!(authorized.validate_successor(&aborting).is_err());
|
||||
assert!(completed.validate_successor(&authorized).is_err());
|
||||
assert!(aborted.validate_successor(&preparing).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_checkpoint_binds_dispatch_and_full_state_monotonically() {
|
||||
use crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared};
|
||||
|
||||
let checkpoint = |content: &str, dispatch: Option<&str>, state| DurableIlmRecordCheckpoint::TierDeleteJournal {
|
||||
content_sha256: content.repeat(64),
|
||||
identity_sha256: "i".repeat(64),
|
||||
committed: state == Some(Committed),
|
||||
dispatch_identity_sha256: dispatch.map(|value| value.repeat(64)),
|
||||
state,
|
||||
};
|
||||
let prepared = checkpoint("a", Some("d"), Some(Prepared));
|
||||
let dispatched = checkpoint("b", Some("d"), Some(Dispatched));
|
||||
let committed = checkpoint("c", Some("d"), Some(Committed));
|
||||
prepared
|
||||
.validate_successor(&dispatched)
|
||||
.expect("Prepared may advance to Dispatched");
|
||||
dispatched
|
||||
.validate_successor(&committed)
|
||||
.expect("Dispatched may advance to Committed");
|
||||
assert!(prepared.validate_successor(&committed).is_err());
|
||||
assert!(dispatched.validate_successor(&prepared).is_err());
|
||||
|
||||
let rebound = checkpoint("b", Some("e"), Some(Dispatched));
|
||||
assert!(dispatched.validate_successor(&rebound).is_err());
|
||||
|
||||
let legacy: DurableIlmRecordCheckpoint = serde_json::from_value(serde_json::json!({
|
||||
"kind": "tier_delete_journal",
|
||||
"content_sha256": "a".repeat(64),
|
||||
"identity_sha256": "i".repeat(64),
|
||||
"committed": false
|
||||
}))
|
||||
.expect("legacy tier-delete checkpoint should remain decodable");
|
||||
legacy
|
||||
.validate_successor(&prepared)
|
||||
.expect("byte-identical legacy receipt may adopt the stronger v6 proof");
|
||||
let changed_legacy = DurableIlmRecordCheckpoint::TierDeleteJournal {
|
||||
content_sha256: "z".repeat(64),
|
||||
identity_sha256: "i".repeat(64),
|
||||
committed: false,
|
||||
dispatch_identity_sha256: None,
|
||||
state: None,
|
||||
};
|
||||
assert!(changed_legacy.validate_successor(&prepared).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_job_checkpoint_compacts_legacy_progress_compatibly() {
|
||||
let options = super::super::bucket_lifecycle_ops::ManualTransitionRunOptions::default();
|
||||
|
||||
@@ -34,6 +34,6 @@ pub mod tier_sweeper;
|
||||
pub mod transition_transaction;
|
||||
|
||||
pub(crate) use durable_namespace::{
|
||||
DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record,
|
||||
validate_durable_ilm_record,
|
||||
DurableIlmRecordCheckpoint, ILM_META_PREFIX, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE, ValidatedDurableIlmRecord,
|
||||
classify_durable_ilm_record, validate_durable_ilm_record,
|
||||
};
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -172,7 +172,8 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
|
||||
return Err(std::io::Error::other("free-version recovery limit must be greater than zero").into());
|
||||
}
|
||||
|
||||
let page = list_tier_free_versions(api, limit, bucket_marker.clone(), object_marker.clone(), cancel_token.clone()).await?;
|
||||
let page =
|
||||
list_tier_free_versions(api.clone(), limit, bucket_marker.clone(), object_marker.clone(), cancel_token.clone()).await?;
|
||||
let mut stats = FreeVersionRecoveryStats {
|
||||
scanned: 0,
|
||||
enqueued: 0,
|
||||
@@ -190,7 +191,7 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
|
||||
return Err(tier_free_version_recovery_cancelled());
|
||||
}
|
||||
retry_cursor.visit(&oi);
|
||||
if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(oi).await) {
|
||||
if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(&api, oi).await) {
|
||||
let (bucket_marker, object_marker) = retry_cursor.retry_markers();
|
||||
stats.truncated = true;
|
||||
stats.next_bucket_marker = bucket_marker;
|
||||
|
||||
@@ -255,6 +255,7 @@ impl ObjSweeper {
|
||||
}
|
||||
if del_tier {
|
||||
return Some(Jentry {
|
||||
persisted_version: 0,
|
||||
obj_name: self.remote_object.clone(),
|
||||
version_id: self.transition_version_id.clone(),
|
||||
tier_name: self.transition_tier.clone(),
|
||||
@@ -266,6 +267,7 @@ impl ObjSweeper {
|
||||
version_state: self.transition_version_state,
|
||||
state: TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
dispatch: None,
|
||||
});
|
||||
}
|
||||
None
|
||||
@@ -298,9 +300,19 @@ impl ObjSweeper {
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub(crate) enum TierDeleteJournalState {
|
||||
Prepared,
|
||||
Dispatched,
|
||||
Committed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct TierDeleteDispatchBinding {
|
||||
pub(crate) operation_id: Uuid,
|
||||
pub(crate) manifest_object: String,
|
||||
pub(crate) journal_set_sha256: String,
|
||||
pub(crate) topology_generation: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct TierDeleteSourceIdentity {
|
||||
@@ -342,6 +354,10 @@ impl TierDeleteSourceIdentity {
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(unused_assignments)]
|
||||
pub struct Jentry {
|
||||
/// On-disk format version when decoded. Newly constructed entries use 0;
|
||||
/// the encoder chooses their format from the durable ownership fields.
|
||||
/// Recovery uses this value to quarantine v1-v5 without rewriting them.
|
||||
pub(crate) persisted_version: u8,
|
||||
pub(crate) obj_name: String,
|
||||
pub(crate) version_id: String,
|
||||
pub(crate) tier_name: String,
|
||||
@@ -350,6 +366,23 @@ pub struct Jentry {
|
||||
pub(crate) version_state: rustfs_filemeta::TransitionVersionState,
|
||||
pub(crate) state: TierDeleteJournalState,
|
||||
pub(crate) source: Option<TierDeleteSourceIdentity>,
|
||||
pub(crate) dispatch: Option<TierDeleteDispatchBinding>,
|
||||
}
|
||||
|
||||
impl Jentry {
|
||||
/// Whether this prepared transaction is eligible to become the sole
|
||||
/// cleanup owner for its transitioned source. The caller may use this to
|
||||
/// decide whether to persist it, but must not set `skip_free_version`
|
||||
/// until persistence succeeds.
|
||||
pub(crate) fn can_replace_tier_free_version(&self) -> bool {
|
||||
self.state == TierDeleteJournalState::Prepared
|
||||
&& self.backend_identity.is_some()
|
||||
&& self.version_state != rustfs_filemeta::TransitionVersionState::Unknown
|
||||
&& self
|
||||
.source
|
||||
.as_ref()
|
||||
.is_some_and(TierDeleteSourceIdentity::has_stable_identity)
|
||||
}
|
||||
}
|
||||
|
||||
impl ExpiryOp for Jentry {
|
||||
@@ -617,6 +650,7 @@ pub fn transitioned_force_delete_journal_entry(
|
||||
}
|
||||
|
||||
Some(Jentry {
|
||||
persisted_version: 0,
|
||||
obj_name: transitioned.name.clone(),
|
||||
version_id: transitioned.version_id.clone(),
|
||||
tier_name: transitioned.tier.clone(),
|
||||
@@ -628,6 +662,7 @@ pub fn transitioned_force_delete_journal_entry(
|
||||
version_state: transition_version_state,
|
||||
state: TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
dispatch: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -673,17 +708,73 @@ mod test {
|
||||
use rustfs_s3_client::signer_error::invalid_utf8_header_error;
|
||||
|
||||
use super::{
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED,
|
||||
RemoteDeleteBreaker, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
|
||||
should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, Jentry,
|
||||
RemoteDeleteBreaker, RemoteTierDeleteOutcome, TierDeleteJournalState, TierDeleteSourceIdentity,
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity, delete_object_from_remote_tier_idempotent,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error,
|
||||
is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, should_record_remote_delete_failure,
|
||||
transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
|
||||
};
|
||||
use crate::storage_api_contracts::lifecycle::TransitionedObject;
|
||||
use rustfs_filemeta::TransitionVersionState;
|
||||
use std::io::{Error, ErrorKind};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
fn stable_prepared_journal() -> Jentry {
|
||||
Jentry {
|
||||
persisted_version: 0,
|
||||
obj_name: "remote/object".to_string(),
|
||||
version_id: "remote-version".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some([7; 32]),
|
||||
version_id_exact: true,
|
||||
version_state: TransitionVersionState::Exact,
|
||||
state: TierDeleteJournalState::Prepared,
|
||||
source: Some(TierDeleteSourceIdentity {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4().to_string()),
|
||||
versioned: true,
|
||||
version_suspended: false,
|
||||
data_dir: None,
|
||||
etag: None,
|
||||
mod_time: None,
|
||||
}),
|
||||
dispatch: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_stable_prepared_journal_can_replace_tier_free_version() {
|
||||
let stable = stable_prepared_journal();
|
||||
assert!(stable.can_replace_tier_free_version());
|
||||
|
||||
let mut committed = stable.clone();
|
||||
committed.state = TierDeleteJournalState::Committed;
|
||||
assert!(!committed.can_replace_tier_free_version());
|
||||
|
||||
let mut unbound = stable.clone();
|
||||
unbound.backend_identity = None;
|
||||
assert!(!unbound.can_replace_tier_free_version());
|
||||
|
||||
let mut unknown = stable.clone();
|
||||
unknown.version_state = TransitionVersionState::Unknown;
|
||||
assert!(!unknown.can_replace_tier_free_version());
|
||||
|
||||
let mut unstable = stable;
|
||||
unstable.source = Some(TierDeleteSourceIdentity {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
version_id: None,
|
||||
versioned: false,
|
||||
version_suspended: false,
|
||||
data_dir: None,
|
||||
etag: Some("etag-only".to_string()),
|
||||
mod_time: None,
|
||||
});
|
||||
assert!(!unstable.can_replace_tier_free_version());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signer_header_error_detection_matches_utf8_failures() {
|
||||
let err = Error::new(
|
||||
|
||||
@@ -49,8 +49,8 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
|
||||
ScannerPublicationLeaseResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
|
||||
StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
|
||||
TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
TierMutationControlResponse, TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest,
|
||||
node_service_client::NodeServiceClient, tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
};
|
||||
pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
|
||||
use rustfs_protos::{TierMutationRpcPhase, evict_failed_connection};
|
||||
@@ -462,6 +462,31 @@ pub struct PeerTierMutationOutcome {
|
||||
pub applied: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("{message}")]
|
||||
struct TierMutationDefinitelyRejected {
|
||||
message: String,
|
||||
}
|
||||
|
||||
fn tier_mutation_definitely_rejected_error(message: String) -> Error {
|
||||
Error::other(TierMutationDefinitelyRejected { message })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn test_tier_mutation_definitely_rejected_error(message: &str) -> Error {
|
||||
tier_mutation_definitely_rejected_error(message.to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn tier_mutation_error_is_definitely_rejected(error: &Error) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
Error::Io(io_error)
|
||||
if io_error
|
||||
.get_ref()
|
||||
.is_some_and(|source| source.downcast_ref::<TierMutationDefinitelyRejected>().is_some())
|
||||
)
|
||||
}
|
||||
|
||||
fn validate_tier_mutation_response_proof(
|
||||
version: u32,
|
||||
phase: TierMutationRpcPhase,
|
||||
@@ -469,6 +494,16 @@ fn validate_tier_mutation_response_proof(
|
||||
canonical_payload: &[u8],
|
||||
response: &TierMutationControlResponse,
|
||||
) -> Result<()> {
|
||||
if response.response_proof.len() > rustfs_protos::TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE {
|
||||
return Err(Error::other("peer tier mutation response proof exceeds size limit"));
|
||||
}
|
||||
if response
|
||||
.error_info
|
||||
.as_ref()
|
||||
.is_some_and(|error| error.len() > rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE)
|
||||
{
|
||||
return Err(Error::other("peer tier mutation error response exceeds size limit"));
|
||||
}
|
||||
let canonical_response =
|
||||
rustfs_protos::canonical_tier_mutation_rpc_response_body(rustfs_protos::TierMutationRpcResponseProofInput {
|
||||
version,
|
||||
@@ -479,6 +514,7 @@ fn validate_tier_mutation_response_proof(
|
||||
state: response.state,
|
||||
applied: response.applied,
|
||||
error_info: response.error_info.as_deref(),
|
||||
failure_class: response.failure_class,
|
||||
})
|
||||
.map_err(|_| Error::other("tier mutation response length cannot be represented"))?;
|
||||
verify_tonic_rpc_response_proof(&canonical_response, &response.response_proof)
|
||||
@@ -500,9 +536,9 @@ fn validate_tier_mutation_payload_len(phase: TierMutationRpcPhase, payload_len:
|
||||
TierMutationRpcPhase::Commit => rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE,
|
||||
TierMutationRpcPhase::Abort => {
|
||||
if payload_len == 0 {
|
||||
return Ok(());
|
||||
return Err(Error::other("tier mutation abort payload is empty"));
|
||||
}
|
||||
return Err(Error::other("tier mutation abort payload must be empty"));
|
||||
rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE
|
||||
}
|
||||
_ => return Err(Error::other("tier mutation rpc phase is unsupported")),
|
||||
};
|
||||
@@ -521,8 +557,29 @@ fn tier_mutation_phase_label(phase: TierMutationRpcPhase) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
fn tier_mutation_control_status_error(phase: TierMutationRpcPhase, status: tonic::Status) -> Error {
|
||||
Error::other(format!("peer tier mutation {} RPC failed: {status}", tier_mutation_phase_label(phase)))
|
||||
fn tier_mutation_control_status_error(phase: TierMutationRpcPhase, requested_version: u32, status: tonic::Status) -> Error {
|
||||
let message = format!("peer tier mutation {} RPC failed: {status}", tier_mutation_phase_label(phase));
|
||||
let legacy_rejection = format!("unsupported tier mutation peer protocol version: {requested_version}");
|
||||
// RUSTFS_COMPAT_TODO(backlog-2097-tier-mutation-v4-error-text): retain this exact v3-server rejection classifier for mixed-version peers. Remove after every supported peer returns the signed v4 failure class.
|
||||
if requested_version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
|
||||
&& status.code() == tonic::Code::FailedPrecondition
|
||||
&& status.message().as_bytes() == legacy_rejection.as_bytes()
|
||||
{
|
||||
return tier_mutation_definitely_rejected_error(message);
|
||||
}
|
||||
Error::other(message)
|
||||
}
|
||||
|
||||
fn tier_mutation_failed_response_error(version: u32, failure_class: i32, error_info: Option<String>) -> Error {
|
||||
let message = error_info.unwrap_or_else(|| "peer tier mutation failed without an error".to_string());
|
||||
if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
|
||||
&& TierMutationFailureClass::try_from(failure_class).ok() == Some(TierMutationFailureClass::PreDispatchRejected)
|
||||
{
|
||||
return tier_mutation_definitely_rejected_error(message);
|
||||
}
|
||||
// Missing/zero, unknown, and explicit Ambiguous are deliberately the same
|
||||
// fail-closed result: the coordinator must include this peer in Abort.
|
||||
Error::other(message)
|
||||
}
|
||||
|
||||
impl PeerRestClient {
|
||||
@@ -1315,8 +1372,12 @@ impl PeerRestClient {
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn abort_tier_mutation(&self, mutation_id: Uuid) -> Result<PeerTierMutationOutcome> {
|
||||
self.tier_mutation_control(TierMutationRpcPhase::Abort, mutation_id, Bytes::new())
|
||||
pub async fn abort_tier_mutation(
|
||||
&self,
|
||||
mutation_id: Uuid,
|
||||
canonical_prepare_payload: Bytes,
|
||||
) -> Result<PeerTierMutationOutcome> {
|
||||
self.tier_mutation_control(TierMutationRpcPhase::Abort, mutation_id, canonical_prepare_payload)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1350,7 +1411,7 @@ impl PeerRestClient {
|
||||
client
|
||||
.prepare_tier_mutation(request)
|
||||
.await
|
||||
.map_err(|status| tier_mutation_control_status_error(phase, status))?
|
||||
.map_err(|status| tier_mutation_control_status_error(phase, version, status))?
|
||||
.into_inner()
|
||||
}
|
||||
TierMutationRpcPhase::Commit => {
|
||||
@@ -1363,7 +1424,7 @@ impl PeerRestClient {
|
||||
client
|
||||
.commit_tier_mutation(request)
|
||||
.await
|
||||
.map_err(|status| tier_mutation_control_status_error(phase, status))?
|
||||
.map_err(|status| tier_mutation_control_status_error(phase, version, status))?
|
||||
.into_inner()
|
||||
}
|
||||
TierMutationRpcPhase::Abort => {
|
||||
@@ -1376,18 +1437,19 @@ impl PeerRestClient {
|
||||
client
|
||||
.abort_tier_mutation(request)
|
||||
.await
|
||||
.map_err(|status| tier_mutation_control_status_error(phase, status))?
|
||||
.map_err(|status| tier_mutation_control_status_error(phase, version, status))?
|
||||
.into_inner()
|
||||
}
|
||||
_ => return Err(Error::other("tier mutation rpc phase is unsupported")),
|
||||
};
|
||||
validate_tier_mutation_response_proof(version, phase, mutation_id, &canonical_payload, &response)?;
|
||||
if !response.success {
|
||||
return Err(Error::other(
|
||||
response
|
||||
.error_info
|
||||
.unwrap_or_else(|| "peer tier mutation failed without an error".to_string()),
|
||||
));
|
||||
return Err(tier_mutation_failed_response_error(version, response.failure_class, response.error_info));
|
||||
}
|
||||
if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
|
||||
&& response.failure_class != TierMutationFailureClass::Unspecified as i32
|
||||
{
|
||||
return Err(Error::other("successful peer tier mutation response carried a failure class"));
|
||||
}
|
||||
let state = decode_tier_mutation_peer_state(response.state)?;
|
||||
Ok(PeerTierMutationOutcome {
|
||||
@@ -3332,6 +3394,7 @@ mod tests {
|
||||
state: i32,
|
||||
applied: bool,
|
||||
error_info: Option<&'a str>,
|
||||
failure_class: i32,
|
||||
}
|
||||
|
||||
fn signed_tier_mutation_response(input: TierMutationResponseFixture<'_>) -> TierMutationControlResponse {
|
||||
@@ -3345,6 +3408,7 @@ mod tests {
|
||||
state: input.state,
|
||||
applied: input.applied,
|
||||
error_info: input.error_info,
|
||||
failure_class: input.failure_class,
|
||||
})
|
||||
.expect("small tier mutation response should encode");
|
||||
let response_proof =
|
||||
@@ -3355,6 +3419,7 @@ mod tests {
|
||||
applied: input.applied,
|
||||
error_info: input.error_info.map(str::to_string),
|
||||
response_proof: response_proof.into(),
|
||||
failure_class: input.failure_class,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3372,6 +3437,7 @@ mod tests {
|
||||
state: TierMutationPeerState::Prepared as i32,
|
||||
applied: true,
|
||||
error_info: None,
|
||||
failure_class: TierMutationFailureClass::Unspecified as i32,
|
||||
});
|
||||
validate_tier_mutation_response_proof(
|
||||
rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
@@ -3396,6 +3462,10 @@ mod tests {
|
||||
applied: false,
|
||||
..response.clone()
|
||||
},
|
||||
TierMutationControlResponse {
|
||||
failure_class: TierMutationFailureClass::Ambiguous as i32,
|
||||
..response.clone()
|
||||
},
|
||||
] {
|
||||
let err = validate_tier_mutation_response_proof(
|
||||
rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
@@ -3419,6 +3489,44 @@ mod tests {
|
||||
assert!(err.to_string().contains("invalid tier mutation response proof"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_mutation_response_rejects_oversized_proof_and_error_before_verification() {
|
||||
let mutation_id = Uuid::new_v4();
|
||||
let payload = b"tier-mutation-prepare";
|
||||
let oversized_proof = TierMutationControlResponse {
|
||||
success: false,
|
||||
state: TierMutationPeerState::Unspecified as i32,
|
||||
applied: false,
|
||||
error_info: None,
|
||||
response_proof: vec![0; rustfs_protos::TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE + 1].into(),
|
||||
failure_class: TierMutationFailureClass::Ambiguous as i32,
|
||||
};
|
||||
let err = validate_tier_mutation_response_proof(
|
||||
rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
payload,
|
||||
&oversized_proof,
|
||||
)
|
||||
.expect_err("oversized proof must fail before cryptographic verification");
|
||||
assert!(err.to_string().contains("response proof exceeds size limit"));
|
||||
|
||||
let oversized_error = TierMutationControlResponse {
|
||||
response_proof: Bytes::new(),
|
||||
error_info: Some("e".repeat(rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE + 1)),
|
||||
..oversized_proof
|
||||
};
|
||||
let err = validate_tier_mutation_response_proof(
|
||||
rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
payload,
|
||||
&oversized_error,
|
||||
)
|
||||
.expect_err("oversized error detail must fail before proof construction");
|
||||
assert!(err.to_string().contains("error response exceeds size limit"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_mutation_peer_state_decode_fails_closed() {
|
||||
assert_eq!(
|
||||
@@ -3463,8 +3571,17 @@ mod tests {
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 0).expect("empty abort payload should fit");
|
||||
assert!(validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 1).is_err());
|
||||
assert!(validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 0).is_err());
|
||||
validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 1).expect("non-empty abort payload should fit");
|
||||
validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE)
|
||||
.expect("max abort payload should fit");
|
||||
assert!(
|
||||
validate_tier_mutation_payload_len(
|
||||
TierMutationRpcPhase::Abort,
|
||||
rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE + 1,
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3479,7 +3596,7 @@ mod tests {
|
||||
tonic::Status::deadline_exceeded("peer tier mutation control timed out"),
|
||||
tonic::Status::unavailable("peer tier mutation control unavailable"),
|
||||
] {
|
||||
let err = tier_mutation_control_status_error(phase, status);
|
||||
let err = tier_mutation_control_status_error(phase, rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION, status);
|
||||
let rendered = err.to_string();
|
||||
assert!(rendered.contains(&format!("peer tier mutation {label} RPC failed")), "{rendered}");
|
||||
assert!(
|
||||
@@ -3493,6 +3610,61 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_mutation_v4_to_v3_rejection_classification_requires_exact_status_and_message() {
|
||||
let version = rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION;
|
||||
let exact = format!("unsupported tier mutation peer protocol version: {version}");
|
||||
let rejected = tier_mutation_control_status_error(
|
||||
TierMutationRpcPhase::Prepare,
|
||||
version,
|
||||
tonic::Status::failed_precondition(exact.clone()),
|
||||
);
|
||||
assert!(tier_mutation_error_is_definitely_rejected(&rejected));
|
||||
|
||||
for status in [
|
||||
tonic::Status::failed_precondition(format!("{exact}.")),
|
||||
tonic::Status::failed_precondition(format!("unsupported tier mutation peer protocol version: {}", version - 1)),
|
||||
tonic::Status::invalid_argument(exact.clone()),
|
||||
tonic::Status::unimplemented(exact),
|
||||
] {
|
||||
let ambiguous = tier_mutation_control_status_error(TierMutationRpcPhase::Prepare, version, status);
|
||||
assert!(
|
||||
!tier_mutation_error_is_definitely_rejected(&ambiguous),
|
||||
"near-text, wrong-code, and Unimplemented failures must remain ambiguous"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_mutation_v4_failure_class_is_typed_and_fails_closed() {
|
||||
let version = rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION;
|
||||
let rejected = tier_mutation_failed_response_error(
|
||||
version,
|
||||
TierMutationFailureClass::PreDispatchRejected as i32,
|
||||
Some("rejected".to_string()),
|
||||
);
|
||||
assert!(tier_mutation_error_is_definitely_rejected(&rejected));
|
||||
|
||||
for failure_class in [
|
||||
TierMutationFailureClass::Unspecified as i32,
|
||||
TierMutationFailureClass::Ambiguous as i32,
|
||||
99,
|
||||
] {
|
||||
let ambiguous = tier_mutation_failed_response_error(version, failure_class, None);
|
||||
assert!(
|
||||
!tier_mutation_error_is_definitely_rejected(&ambiguous),
|
||||
"missing, unknown, and explicit ambiguous classes must trigger Abort fanout"
|
||||
);
|
||||
}
|
||||
|
||||
let v3_ignores_v4_class = tier_mutation_failed_response_error(
|
||||
rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
|
||||
TierMutationFailureClass::PreDispatchRejected as i32,
|
||||
Some("legacy failure".to_string()),
|
||||
);
|
||||
assert!(!tier_mutation_error_is_definitely_rejected(&v3_ignores_v4_class));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_rest_client_rejects_oversized_tier_prepare_before_dialing() {
|
||||
let client = test_peer_client();
|
||||
|
||||
@@ -18,7 +18,8 @@ use crate::bucket::utils::is_meta_bucketname;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::bucket::{
|
||||
lifecycle::{
|
||||
DurableIlmRecordCheckpoint, ILM_META_PREFIX, LifecycleExpiryConfigs, ValidatedDurableIlmRecord,
|
||||
DurableIlmRecordCheckpoint, ILM_META_PREFIX, LifecycleExpiryConfigs, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
|
||||
ValidatedDurableIlmRecord,
|
||||
bucket_lifecycle_audit::LcEventSrc,
|
||||
bucket_lifecycle_ops::{
|
||||
LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_for_data_movement, apply_expiry_rule_in,
|
||||
@@ -26,6 +27,7 @@ use crate::bucket::{
|
||||
},
|
||||
classify_durable_ilm_record, get_expiry_configs,
|
||||
lifecycle::IlmAction,
|
||||
tier_delete_journal::durable_ilm_v6_topology_generation,
|
||||
validate_durable_ilm_record,
|
||||
},
|
||||
metadata_sys,
|
||||
@@ -49,6 +51,9 @@ use crate::error::{
|
||||
use crate::layout::endpoints::EndpointServerPools;
|
||||
use crate::object_api::{DecommissionCapacityOptions, GetObjectReader, ObjectOptions};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::services::notification_sys::{
|
||||
acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches, tier_delete_journal_topology_generation,
|
||||
};
|
||||
use crate::services::rebalance::{REBAL_META_NAME, RebalanceMeta, is_rebalance_conflicting_with_decommission};
|
||||
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
|
||||
use crate::storage_api_contracts::{
|
||||
@@ -2384,10 +2389,12 @@ struct DecommissionDurableIlmReceipt {
|
||||
id: String,
|
||||
checkpoint: DurableIlmRecordCheckpoint,
|
||||
terminal_checkpoint: Option<DurableIlmRecordCheckpoint>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
fleet_topology_generation: Option<String>,
|
||||
}
|
||||
|
||||
impl DecommissionDurableIlmReceipt {
|
||||
fn new(path: &str, record: &ValidatedDurableIlmRecord) -> Self {
|
||||
fn new(path: &str, record: &ValidatedDurableIlmRecord, fleet_topology_generation: Option<String>) -> Self {
|
||||
Self {
|
||||
source_path: path.to_string(),
|
||||
namespace: record.namespace.to_string(),
|
||||
@@ -2395,6 +2402,7 @@ impl DecommissionDurableIlmReceipt {
|
||||
id: record.id.clone(),
|
||||
checkpoint: record.checkpoint.clone(),
|
||||
terminal_checkpoint: None,
|
||||
fleet_topology_generation,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2433,6 +2441,16 @@ impl DecommissionDurableIlmReceipt {
|
||||
))
|
||||
})?;
|
||||
}
|
||||
if self
|
||||
.fleet_topology_generation
|
||||
.as_deref()
|
||||
.is_some_and(|generation| !is_sha256_checksum(generation))
|
||||
{
|
||||
return Err(Error::other_with_context(
|
||||
"receipt fleet topology generation is invalid",
|
||||
format!("source path `{}` {}", self.source_path, self.context()),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -2541,6 +2559,21 @@ fn merge_decommission_durable_ilm_receipts(
|
||||
id: existing.id.clone(),
|
||||
checkpoint,
|
||||
terminal_checkpoint,
|
||||
fleet_topology_generation: match (
|
||||
existing.fleet_topology_generation.as_ref(),
|
||||
incoming.fleet_topology_generation.as_ref(),
|
||||
) {
|
||||
(Some(existing_generation), Some(incoming_generation)) if existing_generation == incoming_generation => {
|
||||
Some(existing_generation.clone())
|
||||
}
|
||||
(None, None) => None,
|
||||
_ => {
|
||||
return Err(Error::other_with_context(
|
||||
"durable ILM receipt fleet topology conflict",
|
||||
format!("source path `{}` {}", existing.source_path, existing.context()),
|
||||
));
|
||||
}
|
||||
},
|
||||
};
|
||||
merged.validate()?;
|
||||
Ok(merged)
|
||||
@@ -7050,6 +7083,15 @@ pub(crate) struct DecommissionCapacityOwner {
|
||||
pub(crate) mutation_id: Option<uuid::Uuid>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct DecommissionDurableIlmCheckpointTarget {
|
||||
pub(crate) source_pool_index: usize,
|
||||
pub(crate) target_pool_index: usize,
|
||||
pub(crate) capacity_owner: DecommissionCapacityOwner,
|
||||
pub(crate) already_committed: bool,
|
||||
pub(crate) target_etag: Option<String>,
|
||||
}
|
||||
|
||||
impl DecommissionCapacityOwner {
|
||||
pub(crate) fn apply_to(self, opts: &mut ObjectOptions) {
|
||||
opts.src_pool_idx = self.source_pool_index;
|
||||
@@ -7457,6 +7499,7 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
Self { state }
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) async fn wait_until_owner_paused(&self) {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), self.state.owner_arrived.notified())
|
||||
.await
|
||||
@@ -7469,6 +7512,7 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
.expect("external mutation should release capacity before waiting for the object namespace");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) async fn wait_until_external_object_capacity_probe_acquired(&self) {
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
@@ -7478,6 +7522,7 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
.expect("external object mutation should acquire its no-active capacity probe");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) async fn wait_until_external_object_commit_phase_started(&self) {
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
@@ -7502,24 +7547,29 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
.expect("external heal should attempt the target namespace lock after capacity admission");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) fn release_owner(&self) {
|
||||
self.state.owner_release.notify_one();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) fn pause_external_object_commit_phase(&self) {
|
||||
self.state.external_object_commit_phase_paused.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) fn release_external_object_commit_phase(&self) {
|
||||
self.state.external_object_commit_phase_release.notify_one();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) fn pause_external_object_capacity_probe(&self) {
|
||||
self.state
|
||||
.external_object_capacity_probe_paused
|
||||
.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) fn release_external_object_capacity_probe(&self) {
|
||||
self.state.external_object_capacity_probe_release.notify_one();
|
||||
}
|
||||
@@ -8315,6 +8365,122 @@ impl ECStore {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Finish the capacity transaction for an identity-preserving temporary
|
||||
/// replacement whose target bytes are already durably present. This is
|
||||
/// the crash-recovery half of `run_decommission_capacity_temporary_mutation`:
|
||||
/// it never writes the target again and only resolves a pending intent
|
||||
/// owned by the exact deterministic mutation id.
|
||||
pub(crate) async fn reconcile_decommission_capacity_after_equivalent_temporary_target(
|
||||
&self,
|
||||
owner: DecommissionCapacityOwner,
|
||||
target_pool_index: usize,
|
||||
expected_data_bytes: usize,
|
||||
) -> Result<()> {
|
||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let (pool_meta_guard, mut snapshot) = self
|
||||
.acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent temporary target reconciliation failed")
|
||||
.await?;
|
||||
let source_pool_index = owner.source_pool_index;
|
||||
let mutation_id = owner
|
||||
.mutation_id
|
||||
.ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target mutation identity is missing"))?;
|
||||
let (target_layout, pending_physical_bytes, pending_mutation_id, already_reconciled) = {
|
||||
let reservation = snapshot
|
||||
.pools
|
||||
.get(source_pool_index)
|
||||
.and_then(|pool| pool.decommission.as_ref())
|
||||
.and_then(|info| info.capacity_reservation.as_ref())
|
||||
.filter(|reservation| reservation.admits_cleanup_owner(owner))
|
||||
.ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target owner is stale"))?;
|
||||
let target = reservation
|
||||
.targets
|
||||
.iter()
|
||||
.find(|target| target.pool_index == target_pool_index)
|
||||
.ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target allocation is missing"))?;
|
||||
(
|
||||
target.layout,
|
||||
target.pending_physical_bytes,
|
||||
target.pending_mutation_id,
|
||||
target
|
||||
.temporary_mutations
|
||||
.iter()
|
||||
.any(|mutation| mutation.mutation_id == mutation_id),
|
||||
)
|
||||
};
|
||||
if pending_physical_bytes == 0 {
|
||||
// Either the successful attempt already saved its progress, or a
|
||||
// byte-neutral replacement had no inflight delta to record.
|
||||
ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation fence failed")?;
|
||||
return Ok(());
|
||||
}
|
||||
if pending_mutation_id != Some(mutation_id) {
|
||||
return Err(decommission_capacity_blocked_error(
|
||||
"equivalent temporary target pending intent belongs to another mutation",
|
||||
));
|
||||
}
|
||||
if already_reconciled {
|
||||
return Err(decommission_capacity_blocked_error(
|
||||
"equivalent temporary target has both pending and reconciled state",
|
||||
));
|
||||
}
|
||||
let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?;
|
||||
if pending_physical_bytes < expected_target_physical_bytes {
|
||||
return Err(decommission_capacity_blocked_error(
|
||||
"equivalent temporary target pending capacity is smaller than the committed checkpoint",
|
||||
));
|
||||
}
|
||||
resolve_decommission_target_pending(
|
||||
&mut snapshot,
|
||||
source_pool_index,
|
||||
target_pool_index,
|
||||
expected_target_physical_bytes,
|
||||
mutation_id,
|
||||
)?;
|
||||
// The replacement is byte-non-growing and its exact bytes were read
|
||||
// before this call, so no new physical delta is inferred on replay.
|
||||
// A prior successful progress save would have taken the idempotent
|
||||
// pending==0 return above.
|
||||
record_decommission_target_inflight(
|
||||
&mut snapshot,
|
||||
source_pool_index,
|
||||
target_pool_index,
|
||||
0,
|
||||
mutation_id,
|
||||
OffsetDateTime::now_utc(),
|
||||
)?;
|
||||
let outcome = snapshot
|
||||
.save_no_lock_armed(
|
||||
self.pools.clone(),
|
||||
&mut save_guard,
|
||||
pool_meta_guard.lock_lost_signal(),
|
||||
&[source_pool_index],
|
||||
)
|
||||
.await?;
|
||||
ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?;
|
||||
let persisted_info = outcome
|
||||
.committed
|
||||
.pools
|
||||
.get(source_pool_index)
|
||||
.and_then(|pool| pool.decommission.as_ref())
|
||||
.cloned()
|
||||
.ok_or_else(|| decommission_metadata_not_initialized_error("publish equivalent temporary target reconciliation"))?;
|
||||
{
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
let pool_count = pool_meta.pools.len();
|
||||
pool_meta.version = pool_meta.version.max(outcome.committed.version);
|
||||
let info = pool_meta
|
||||
.pools
|
||||
.get_mut(source_pool_index)
|
||||
.and_then(|pool| pool.decommission.as_mut())
|
||||
.ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
|
||||
info.capacity_reservation = persisted_info.capacity_reservation;
|
||||
info.capacity_blocked_reason = persisted_info.capacity_blocked_reason;
|
||||
}
|
||||
ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?;
|
||||
outcome.disarm();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn has_decommission_capacity_temporary_mutation_state(
|
||||
&self,
|
||||
target_pool_index: usize,
|
||||
@@ -12847,6 +13013,7 @@ impl ECStore {
|
||||
let receipt_path = decommission_durable_ilm_receipt_path(run_token, path, source_record.id_kind, &source_record.id);
|
||||
let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
|
||||
let mut proof = None::<DecommissionDurableIlmReceipt>;
|
||||
let mut nonterminal_receipt_found = false;
|
||||
for pool_idx in 0..self.pools.len() {
|
||||
if pool_idx == source_pool_idx {
|
||||
continue;
|
||||
@@ -12890,23 +13057,35 @@ impl ECStore {
|
||||
receipt.context()
|
||||
)));
|
||||
}
|
||||
source_record
|
||||
.checkpoint
|
||||
.validate_successor(&receipt.checkpoint)
|
||||
.map_err(|err| {
|
||||
Error::other(format!(
|
||||
"terminal durable ILM decommission receipt does not cover source at path `{path}` {}: {err}",
|
||||
source_record.context()
|
||||
))
|
||||
})?;
|
||||
if receipt.terminal_checkpoint.is_some() {
|
||||
if let Some(terminal_checkpoint) = receipt.terminal_checkpoint.as_ref() {
|
||||
if !source_record.checkpoint.is_predecessor_of_terminal(terminal_checkpoint) {
|
||||
return Err(Error::other_with_context(
|
||||
"terminal durable ILM decommission receipt does not cover source",
|
||||
format!("path `{path}` {}", source_record.context()),
|
||||
));
|
||||
}
|
||||
proof = Some(match proof {
|
||||
Some(existing) => merge_decommission_durable_ilm_receipts(&existing, &receipt)?,
|
||||
None => receipt,
|
||||
});
|
||||
} else {
|
||||
source_record
|
||||
.checkpoint
|
||||
.validate_successor(&receipt.checkpoint)
|
||||
.map_err(|err| {
|
||||
Error::other_with_context(
|
||||
"durable ILM decommission receipt does not cover source",
|
||||
format!("path `{path}` {}: {err}", source_record.context()),
|
||||
)
|
||||
})?;
|
||||
nonterminal_receipt_found = true;
|
||||
}
|
||||
}
|
||||
Ok(proof)
|
||||
// A terminal receipt on one target must not hide another target copy
|
||||
// whose receipt was installed later and has not reached terminal yet.
|
||||
// Returning no proof makes recovery advance every outstanding copy
|
||||
// before source cleanup can treat the operation as complete.
|
||||
if nonterminal_receipt_found { Ok(None) } else { Ok(proof) }
|
||||
}
|
||||
|
||||
async fn verify_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> {
|
||||
@@ -13050,6 +13229,7 @@ impl ECStore {
|
||||
pool_idx: usize,
|
||||
receipt_path: &str,
|
||||
record: &ValidatedDurableIlmRecord,
|
||||
fleet_topology_generation: Option<&str>,
|
||||
terminal: bool,
|
||||
) -> Result<bool> {
|
||||
let stage = if terminal { "terminal" } else { "progress" };
|
||||
@@ -13085,6 +13265,12 @@ impl ECStore {
|
||||
))
|
||||
})?;
|
||||
Self::validate_decommission_durable_ilm_receipt_locator(receipt_path, &locator, &receipt)?;
|
||||
if receipt.fleet_topology_generation.as_deref() != fleet_topology_generation {
|
||||
return Err(Error::other_with_context(
|
||||
"durable ILM decommission receipt fleet topology mismatch",
|
||||
format!("path `{}` {}", receipt.source_path, receipt.context()),
|
||||
));
|
||||
}
|
||||
receipt.checkpoint.validate_successor(&record.checkpoint).map_err(|err| {
|
||||
Error::other(format!(
|
||||
"{stage} durable ILM record generation mismatch at path `{}` {}: {err}",
|
||||
@@ -13197,6 +13383,7 @@ impl ECStore {
|
||||
let stage = if terminal { "terminal" } else { "progress" };
|
||||
let record = validate_durable_ilm_record(path, data)
|
||||
.map_err(|err| Error::other(format!("{stage} durable ILM record is invalid at path `{path}`: {err}")))?;
|
||||
let fleet_topology_generation = durable_ilm_v6_topology_generation(path, data)?;
|
||||
let active_source_pool_indices = active_runs.iter().map(|(pool_idx, _)| *pool_idx).collect::<Vec<_>>();
|
||||
let mut terminal_target_pool_indices = Vec::new();
|
||||
for (source_pool_idx, run_token) in active_runs {
|
||||
@@ -13205,7 +13392,13 @@ impl ECStore {
|
||||
for pool_idx in 0..self.pools.len() {
|
||||
if pool_idx != source_pool_idx {
|
||||
let found = self
|
||||
.advance_durable_ilm_decommission_receipt(pool_idx, &receipt_path, &record, terminal)
|
||||
.advance_durable_ilm_decommission_receipt(
|
||||
pool_idx,
|
||||
&receipt_path,
|
||||
&record,
|
||||
fleet_topology_generation.as_deref(),
|
||||
terminal,
|
||||
)
|
||||
.await?;
|
||||
receipt_found |= found;
|
||||
if terminal
|
||||
@@ -13227,6 +13420,240 @@ impl ECStore {
|
||||
Ok(Some(terminal_target_pool_indices))
|
||||
}
|
||||
|
||||
/// Resolve the exact receipt-bearing target copies on which a v6 dispatch
|
||||
/// manifest may advance while a decommission reservation is active.
|
||||
///
|
||||
/// This is intentionally not a general capacity bypass. The target copy
|
||||
/// must still be covered by the active source reservation and its durable
|
||||
/// receipt, the ETag must be the caller's exact CAS generation, and the
|
||||
/// replacement must be a byte-non-growing adjacent manifest checkpoint.
|
||||
pub(crate) async fn decommission_durable_ilm_checkpoint_targets(
|
||||
&self,
|
||||
path: &str,
|
||||
next_data: &[u8],
|
||||
expected_etag: &str,
|
||||
) -> Result<Option<Vec<DecommissionDurableIlmCheckpointTarget>>> {
|
||||
let active_runs = {
|
||||
let pool_meta = self.pool_meta.read().await;
|
||||
let mut active_runs = Vec::new();
|
||||
for (source_pool_index, pool) in pool_meta.pools.iter().enumerate() {
|
||||
let Some(info) = pool
|
||||
.decommission
|
||||
.as_ref()
|
||||
.filter(|info| info.has_decommission_state() && !info.complete)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let Some(start_time) = info.start_time else {
|
||||
continue;
|
||||
};
|
||||
let reservation = info.capacity_reservation.clone().ok_or_else(|| {
|
||||
decommission_capacity_blocked_error(format!(
|
||||
"active decommission source pool {source_pool_index} has no reservation for durable ILM checkpoint"
|
||||
))
|
||||
})?;
|
||||
if !reservation.active() {
|
||||
return Err(decommission_capacity_blocked_error(format!(
|
||||
"active decommission source pool {source_pool_index} has a released reservation for durable ILM checkpoint"
|
||||
)));
|
||||
}
|
||||
active_runs.push((
|
||||
source_pool_index,
|
||||
decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time),
|
||||
reservation,
|
||||
));
|
||||
}
|
||||
active_runs
|
||||
};
|
||||
if active_runs.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let next_record = validate_durable_ilm_record(path, next_data)?;
|
||||
if next_record.namespace != TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE.name {
|
||||
return Ok(None);
|
||||
}
|
||||
let next_fleet_topology_generation = durable_ilm_v6_topology_generation(path, next_data)?;
|
||||
|
||||
let mut targets = Vec::<DecommissionDurableIlmCheckpointTarget>::new();
|
||||
let mut missing_source_receipts = 0usize;
|
||||
let mut stale_target_etag_mismatch = false;
|
||||
for (source_pool_index, run_token, reservation) in active_runs {
|
||||
let receipt_path = decommission_durable_ilm_receipt_path(&run_token, path, next_record.id_kind, &next_record.id);
|
||||
let mut source_receipt_found = false;
|
||||
for allocation in &reservation.targets {
|
||||
let receipt_data = match read_config_limited_preserve_empty(
|
||||
self.pools[allocation.pool_index].clone(),
|
||||
&receipt_path,
|
||||
DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(data) => data,
|
||||
Err(err)
|
||||
if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
|
||||
|| is_err_object_not_found(&err)
|
||||
|| is_err_version_not_found(&err) =>
|
||||
{
|
||||
continue;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let receipt = DecommissionDurableIlmReceipt::decode(&receipt_data)?;
|
||||
let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
|
||||
Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &receipt)?;
|
||||
if receipt.source_path != path
|
||||
|| receipt.namespace != next_record.namespace
|
||||
|| receipt.id_kind != next_record.id_kind
|
||||
|| receipt.id != next_record.id
|
||||
{
|
||||
return Err(Error::other_with_context(
|
||||
"durable ILM checkpoint receipt identity does not authorize source",
|
||||
format!("path `{path}` {}", next_record.context()),
|
||||
));
|
||||
}
|
||||
if receipt.fleet_topology_generation != next_fleet_topology_generation {
|
||||
return Err(Error::other_with_context(
|
||||
"durable ILM checkpoint receipt fleet topology does not authorize source",
|
||||
format!("path `{path}` {}", next_record.context()),
|
||||
));
|
||||
}
|
||||
receipt
|
||||
.checkpoint
|
||||
.validate_successor(&next_record.checkpoint)
|
||||
.map_err(|err| {
|
||||
Error::other_with_context(
|
||||
"durable ILM checkpoint receipt is not a predecessor of the requested generation",
|
||||
format!(
|
||||
"target pool {}, path `{path}` {}; receipt checkpoint {:?}, requested checkpoint {:?}: {err}",
|
||||
allocation.pool_index,
|
||||
next_record.context(),
|
||||
receipt.checkpoint,
|
||||
next_record.checkpoint
|
||||
),
|
||||
)
|
||||
})?;
|
||||
|
||||
let (target_data, metadata) = read_config_limited_preserve_empty_with_metadata(
|
||||
self.pools[allocation.pool_index].clone(),
|
||||
path,
|
||||
TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE.max_record_size,
|
||||
)
|
||||
.await?;
|
||||
let target_record = validate_durable_ilm_record(path, &target_data)?;
|
||||
if target_record.namespace != next_record.namespace
|
||||
|| target_record.id_kind != next_record.id_kind
|
||||
|| target_record.id != next_record.id
|
||||
{
|
||||
return Err(Error::other_with_context(
|
||||
"durable ILM checkpoint target identity does not match source",
|
||||
format!("path `{path}` {}", next_record.context()),
|
||||
));
|
||||
}
|
||||
let already_committed = target_data.as_slice() == next_data;
|
||||
let target_etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| {
|
||||
Error::other_with_context("durable ILM checkpoint target is missing an ETag", format!("path `{path}`"))
|
||||
})?;
|
||||
if already_committed {
|
||||
receipt
|
||||
.checkpoint
|
||||
.validate_successor(&target_record.checkpoint)
|
||||
.map_err(|err| {
|
||||
Error::other_with_context(
|
||||
"durable ILM checkpoint receipt is not a predecessor of the committed target generation",
|
||||
format!("path `{path}` {}: {err}", next_record.context()),
|
||||
)
|
||||
})?;
|
||||
} else {
|
||||
if target_record.checkpoint != receipt.checkpoint {
|
||||
return Err(Error::other_with_context(
|
||||
"durable ILM checkpoint target is not the receipt generation",
|
||||
format!("path `{path}` {}", next_record.context()),
|
||||
));
|
||||
}
|
||||
target_record
|
||||
.checkpoint
|
||||
.validate_successor(&next_record.checkpoint)
|
||||
.map_err(|err| {
|
||||
Error::other_with_context(
|
||||
"durable ILM checkpoint target is not a predecessor of the requested generation",
|
||||
format!("path `{path}` {}: {err}", next_record.context()),
|
||||
)
|
||||
})?;
|
||||
if next_data.len() > target_data.len() {
|
||||
return Err(decommission_capacity_blocked_error(format!(
|
||||
"durable ILM checkpoint update at `{path}` grows from {} to {} bytes",
|
||||
target_data.len(),
|
||||
next_data.len()
|
||||
)));
|
||||
}
|
||||
stale_target_etag_mismatch |= target_etag != expected_etag;
|
||||
}
|
||||
|
||||
source_receipt_found = true;
|
||||
if let Some(existing) = targets
|
||||
.iter_mut()
|
||||
.find(|target| target.target_pool_index == allocation.pool_index)
|
||||
{
|
||||
if existing.already_committed != already_committed {
|
||||
return Err(Error::other_with_context(
|
||||
"durable ILM checkpoint target state changed during authorization",
|
||||
format!("path `{path}`"),
|
||||
));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let base_owner = DecommissionCapacityOwner {
|
||||
source_pool_index,
|
||||
operation_id: reservation.operation_id,
|
||||
generation: reservation.generation,
|
||||
owner_nonce: reservation.owner_nonce,
|
||||
mutation_id: None,
|
||||
};
|
||||
let mutation_id = decommission_capacity_mutation_id(
|
||||
base_owner,
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
Some(next_record.checkpoint.content_sha256()),
|
||||
false,
|
||||
None,
|
||||
);
|
||||
targets.push(DecommissionDurableIlmCheckpointTarget {
|
||||
source_pool_index,
|
||||
target_pool_index: allocation.pool_index,
|
||||
capacity_owner: base_owner.with_mutation_id(mutation_id),
|
||||
already_committed,
|
||||
target_etag: Some(target_etag),
|
||||
});
|
||||
}
|
||||
if !source_receipt_found {
|
||||
missing_source_receipts = missing_source_receipts.saturating_add(1);
|
||||
}
|
||||
}
|
||||
|
||||
if missing_source_receipts > 0 {
|
||||
return Err(decommission_capacity_blocked_error(format!(
|
||||
"durable ILM checkpoint at `{path}` is missing receipt coverage for {missing_source_receipts} active source(s)"
|
||||
)));
|
||||
}
|
||||
if targets.is_empty() {
|
||||
return Err(decommission_capacity_blocked_error(format!(
|
||||
"durable ILM checkpoint at `{path}` has no receipt-bearing reservation target"
|
||||
)));
|
||||
}
|
||||
if stale_target_etag_mismatch && !targets.iter().any(|target| target.already_committed) {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
// After a partial multi-target commit, an aggregate read may return
|
||||
// the ETag of the already-advanced target while another authorized
|
||||
// target still has the predecessor ETag. The exact committed bytes
|
||||
// plus every target's receipt/checkpoint proof authorize repairing
|
||||
// that predecessor with its own target-local CAS. Without an exact
|
||||
// committed target, retain the caller-ETag requirement above.
|
||||
targets.sort_unstable_by_key(|target| target.target_pool_index);
|
||||
Ok(Some(targets))
|
||||
}
|
||||
|
||||
pub(crate) async fn record_durable_ilm_decommission_progress(&self, path: &str, data: &[u8]) -> Result<()> {
|
||||
self.advance_durable_ilm_decommission_receipts(path, data, false)
|
||||
.await
|
||||
@@ -13248,6 +13675,66 @@ impl ECStore {
|
||||
self.advance_durable_ilm_decommission_receipts(path, data, true).await
|
||||
}
|
||||
|
||||
/// Return true only when `data` is the exact copy still owned by an active
|
||||
/// decommission source and a target-side terminal receipt authorizes that
|
||||
/// source's later verified cleanup. Lifecycle recovery may then regard the
|
||||
/// logical record as terminal without deleting the source checkpoint.
|
||||
pub(crate) async fn durable_ilm_terminal_receipt_covers_active_source(&self, path: &str, data: &[u8]) -> Result<bool> {
|
||||
let namespace = classify_durable_ilm_record(path)?
|
||||
.ok_or_else(|| Error::other_with_context("path is not a durable ILM record", format!("path `{path}`")))?;
|
||||
let source_record = validate_durable_ilm_record(path, data)?;
|
||||
let active_runs = {
|
||||
let pool_meta = self.pool_meta.read().await;
|
||||
pool_meta
|
||||
.pools
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(source_pool_index, pool)| {
|
||||
pool.decommission
|
||||
.as_ref()
|
||||
.filter(|info| info.has_decommission_state() && !info.complete)
|
||||
.and_then(|info| info.start_time)
|
||||
.map(|start_time| {
|
||||
(source_pool_index, decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time))
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let mut covered_active_source = false;
|
||||
for (source_pool_index, run_token) in active_runs {
|
||||
let source_data =
|
||||
match read_config_limited_preserve_empty(self.pools[source_pool_index].clone(), path, namespace.max_record_size)
|
||||
.await
|
||||
{
|
||||
Ok(source_data) => source_data,
|
||||
Err(err)
|
||||
if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
|
||||
|| is_err_object_not_found(&err)
|
||||
|| is_err_version_not_found(&err) =>
|
||||
{
|
||||
continue;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if source_data.as_slice() != data {
|
||||
continue;
|
||||
}
|
||||
if self
|
||||
.load_decommission_durable_ilm_terminal_receipt_for_run(source_pool_index, path, &source_record, &run_token)
|
||||
.await?
|
||||
.is_some()
|
||||
{
|
||||
covered_active_source = true;
|
||||
} else {
|
||||
// Every active source that still stores this exact generation
|
||||
// needs complete terminal receipt coverage. One covered source
|
||||
// cannot authorize lifecycle recovery to skip another.
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(covered_active_source)
|
||||
}
|
||||
|
||||
async fn cleanup_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> {
|
||||
for (pool_idx, receipt_path) in self.list_decommission_durable_ilm_receipts(source_pool_idx).await? {
|
||||
match delete_config(self.pools[pool_idx].clone(), &receipt_path).await {
|
||||
@@ -13316,12 +13803,20 @@ impl ECStore {
|
||||
.map_err(|err| Error::other(format!("failed to read source durable ILM record at path `{path}`: {err}")))?;
|
||||
let source_record = validate_durable_ilm_record(path, &source)
|
||||
.map_err(|err| Error::other(format!("source durable ILM record is invalid at path `{path}`: {err}")))?;
|
||||
let source_fleet_topology_generation = durable_ilm_v6_topology_generation(path, &source)?;
|
||||
let target = self
|
||||
.load_decommissioned_durable_ilm_target(source_pool_idx, path, namespace.max_record_size, &source_record.context())
|
||||
.await?;
|
||||
let manifest_receipt = if let Some((target_pool_idx, target)) = target {
|
||||
let target_record = validate_decommission_durable_ilm_copy(path, &source_record, &target)?;
|
||||
let receipt = DecommissionDurableIlmReceipt::new(path, &target_record);
|
||||
let target_fleet_topology_generation = durable_ilm_v6_topology_generation(path, &target)?;
|
||||
if target_fleet_topology_generation != source_fleet_topology_generation {
|
||||
return Err(Error::other_with_context(
|
||||
"target durable ILM fleet topology generation differs from source",
|
||||
format!("path `{path}` {}", source_record.context()),
|
||||
));
|
||||
}
|
||||
let receipt = DecommissionDurableIlmReceipt::new(path, &target_record, target_fleet_topology_generation);
|
||||
self.persist_decommission_durable_ilm_receipt_for_run(target_pool_idx, &receipt, run_token)
|
||||
.await?;
|
||||
receipt
|
||||
@@ -13335,9 +13830,34 @@ impl ECStore {
|
||||
))
|
||||
})?
|
||||
};
|
||||
if manifest_receipt.fleet_topology_generation != source_fleet_topology_generation {
|
||||
return Err(Error::other_with_context(
|
||||
"terminal durable ILM receipt fleet topology generation does not cover source",
|
||||
format!("path `{path}` {}", source_record.context()),
|
||||
));
|
||||
}
|
||||
let fleet_proof = if let Some(expected_generation) = source_fleet_topology_generation.as_deref() {
|
||||
let proof = acquire_tier_delete_journal_fleet_proof()
|
||||
.ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable for source cleanup"))?;
|
||||
if tier_delete_journal_topology_generation(&proof) != expected_generation
|
||||
|| !tier_delete_journal_fleet_proof_matches(&proof)
|
||||
{
|
||||
return Err(Error::other("tier delete journal v6 fleet generation changed before source cleanup"));
|
||||
}
|
||||
Some(proof)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
self.persist_decommission_durable_ilm_receipt_for_run(source_pool_idx, &manifest_receipt, run_token)
|
||||
.await?;
|
||||
|
||||
if fleet_proof
|
||||
.as_ref()
|
||||
.is_some_and(|proof| !tier_delete_journal_fleet_proof_matches(proof))
|
||||
{
|
||||
return Err(Error::other("tier delete journal v6 fleet proof changed before source cleanup"));
|
||||
}
|
||||
|
||||
let cleanup_result = data_movement::cleanup_source_entry_if_unchanged(
|
||||
source_set,
|
||||
RUSTFS_META_BUCKET,
|
||||
@@ -13354,7 +13874,16 @@ impl ECStore {
|
||||
source_record.context()
|
||||
))
|
||||
});
|
||||
resolve_decommission_entry_cleanup_delete_result(cleanup_result, RUSTFS_META_BUCKET, path)
|
||||
let cleanup_result = resolve_decommission_entry_cleanup_delete_result(cleanup_result, RUSTFS_META_BUCKET, path);
|
||||
if fleet_proof
|
||||
.as_ref()
|
||||
.is_some_and(|proof| !tier_delete_journal_fleet_proof_matches(proof))
|
||||
{
|
||||
return Err(Error::other(
|
||||
"tier delete journal v6 fleet proof changed during source cleanup; exact source outcome requires verification",
|
||||
));
|
||||
}
|
||||
cleanup_result
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
@@ -13390,7 +13919,26 @@ impl ECStore {
|
||||
record: &ValidatedDurableIlmRecord,
|
||||
terminal: bool,
|
||||
) -> Result<String> {
|
||||
let mut receipt = DecommissionDurableIlmReceipt::new(source_path, record);
|
||||
let fleet_topology_generation = match read_config_limited_preserve_empty(
|
||||
self.pools[target_pool_idx].clone(),
|
||||
source_path,
|
||||
classify_durable_ilm_record(source_path)?
|
||||
.ok_or_else(|| Error::other_with_context("path is not a durable ILM record", format!("path `{source_path}`")))?
|
||||
.max_record_size,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(target_data) => durable_ilm_v6_topology_generation(source_path, &target_data)?,
|
||||
Err(err)
|
||||
if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
|
||||
|| is_err_object_not_found(&err)
|
||||
|| is_err_version_not_found(&err) =>
|
||||
{
|
||||
None
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let mut receipt = DecommissionDurableIlmReceipt::new(source_path, record, fleet_topology_generation);
|
||||
if terminal {
|
||||
receipt.terminal_checkpoint = Some(record.checkpoint.clone());
|
||||
}
|
||||
@@ -17361,11 +17909,15 @@ mod pools_tests {
|
||||
content_sha256: "b".repeat(64),
|
||||
identity_sha256: "c".repeat(64),
|
||||
committed: false,
|
||||
dispatch_identity_sha256: None,
|
||||
state: None,
|
||||
};
|
||||
let terminal_checkpoint = DurableIlmRecordCheckpoint::TierDeleteJournal {
|
||||
content_sha256: "d".repeat(64),
|
||||
identity_sha256: "c".repeat(64),
|
||||
committed: true,
|
||||
dispatch_identity_sha256: None,
|
||||
state: None,
|
||||
};
|
||||
let incoming = DecommissionDurableIlmReceipt {
|
||||
source_path,
|
||||
@@ -17374,6 +17926,7 @@ mod pools_tests {
|
||||
id: operation_id,
|
||||
checkpoint: checkpoint.clone(),
|
||||
terminal_checkpoint: None,
|
||||
fleet_topology_generation: None,
|
||||
};
|
||||
let existing = DecommissionDurableIlmReceipt {
|
||||
terminal_checkpoint: Some(terminal_checkpoint.clone()),
|
||||
@@ -17384,7 +17937,24 @@ mod pools_tests {
|
||||
.expect("retry receipt must merge with a terminal receipt");
|
||||
|
||||
assert_eq!(merged.checkpoint, checkpoint);
|
||||
assert_eq!(merged.terminal_checkpoint, Some(terminal_checkpoint));
|
||||
assert_eq!(merged.terminal_checkpoint, Some(terminal_checkpoint.clone()));
|
||||
|
||||
let topology_bound = DecommissionDurableIlmReceipt {
|
||||
fleet_topology_generation: Some("e".repeat(64)),
|
||||
..incoming
|
||||
};
|
||||
let mixed_error = merge_decommission_durable_ilm_receipts(&existing, &topology_bound)
|
||||
.expect_err("a topology-bound v6 receipt must not mask an unbound receipt")
|
||||
.to_string();
|
||||
assert!(mixed_error.contains("fleet topology conflict"));
|
||||
|
||||
let topology_existing = DecommissionDurableIlmReceipt {
|
||||
terminal_checkpoint: Some(terminal_checkpoint),
|
||||
..topology_bound.clone()
|
||||
};
|
||||
let topology_merged = merge_decommission_durable_ilm_receipts(&topology_existing, &topology_bound)
|
||||
.expect("receipts bound to the same fleet topology should merge");
|
||||
assert_eq!(topology_merged.fleet_topology_generation, Some("e".repeat(64)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -17409,7 +17979,7 @@ mod pools_tests {
|
||||
assert!(job_bytes.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE);
|
||||
let record = validate_durable_ilm_record(&path, &job_bytes).expect("large manual job should validate");
|
||||
let expected_checkpoint = record.checkpoint.clone();
|
||||
let mut receipt = DecommissionDurableIlmReceipt::new(&path, &record);
|
||||
let mut receipt = DecommissionDurableIlmReceipt::new(&path, &record, None);
|
||||
receipt.terminal_checkpoint = Some(record.checkpoint);
|
||||
|
||||
let encoded = receipt.encode().expect("bounded progress proof should fit the receipt limit");
|
||||
|
||||
@@ -37,8 +37,9 @@ use crate::{
|
||||
runtime::instance::{InstanceContext, bootstrap_ctx},
|
||||
runtime::sources as runtime_sources,
|
||||
set_disk::{PreparedGetObjectMetadata, SetDisks},
|
||||
store::init_format::{
|
||||
check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum,
|
||||
store::{
|
||||
RemoteTuplePublicationFence,
|
||||
init_format::{check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum},
|
||||
},
|
||||
};
|
||||
use futures::{
|
||||
@@ -625,6 +626,19 @@ impl Sets {
|
||||
.put_object_with_old_current_size(bucket, object, data, opts)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_object_with_old_current_size_for_data_movement(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
publication_fence: RemoteTuplePublicationFence,
|
||||
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
|
||||
self.get_disks_by_key(object)
|
||||
.put_object_with_old_current_size_for_data_movement(bucket, object, data, opts, publication_fence)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
|
||||
@@ -27,7 +27,7 @@ use crate::storage_api_contracts::{
|
||||
namespace::NamespaceLocking as _,
|
||||
object::{HTTPPreconditions, ObjectOperations as _},
|
||||
};
|
||||
use crate::store::{ECStore, ObjectLockDiagGuard, SourceCleanupMutationFence};
|
||||
use crate::store::{DecommissionFixedReadAnchor, ECStore, SourceCleanupMutationFence};
|
||||
use bytes::Bytes;
|
||||
use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
|
||||
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
|
||||
@@ -161,7 +161,7 @@ pub fn mark_multipart_upload_completed(flag: &Arc<AtomicBool>) {
|
||||
flag.store(false, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
struct DataMovementMultipartAbortBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
@@ -169,17 +169,17 @@ struct DataMovementMultipartAbortBarrierState {
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) struct DataMovementMultipartAbortBarrier {
|
||||
state: Arc<DataMovementMultipartAbortBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
static DATA_MOVEMENT_MULTIPART_ABORT_BARRIER: std::sync::OnceLock<
|
||||
std::sync::Mutex<Option<Arc<DataMovementMultipartAbortBarrierState>>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
impl DataMovementMultipartAbortBarrier {
|
||||
pub(crate) fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(DataMovementMultipartAbortBarrierState {
|
||||
@@ -204,7 +204,7 @@ impl DataMovementMultipartAbortBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
impl Drop for DataMovementMultipartAbortBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
@@ -218,7 +218,7 @@ impl Drop for DataMovementMultipartAbortBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
async fn pause_data_movement_multipart_before_abort(bucket: &str, object: &str) {
|
||||
let barrier = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
@@ -1518,7 +1518,7 @@ pub(crate) async fn migrate_decommission_object(
|
||||
capacity_owner: Option<DecommissionCapacityOwner>,
|
||||
) -> Result<()> {
|
||||
let source = rd.object_info.clone();
|
||||
let _mutation_fence = store
|
||||
let mutation_fence = store
|
||||
.acquire_decommission_object_mutation_fence(&bucket, &source.name)
|
||||
.await?;
|
||||
let current = find_data_movement_target_info(store.as_ref(), pool_idx, &bucket, &source)
|
||||
@@ -1537,7 +1537,7 @@ pub(crate) async fn migrate_decommission_object(
|
||||
op_label,
|
||||
None,
|
||||
capacity_owner,
|
||||
Some(&_mutation_fence),
|
||||
Some(mutation_fence),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -1588,8 +1588,9 @@ async fn migrate_object_inner(
|
||||
op_label: &str,
|
||||
lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
|
||||
capacity_owner: Option<DecommissionCapacityOwner>,
|
||||
mutation_fence: Option<&ObjectLockDiagGuard>,
|
||||
mutation_fence: Option<DecommissionFixedReadAnchor>,
|
||||
) -> Result<()> {
|
||||
let mut mutation_fence = mutation_fence;
|
||||
let object_info = rd.object_info.clone();
|
||||
let capacity_owner = capacity_owner.map(|owner| {
|
||||
let version_id = object_info.version_id.map(|version_id| version_id.to_string());
|
||||
@@ -1605,6 +1606,13 @@ async fn migrate_object_inner(
|
||||
});
|
||||
owner.with_mutation_id(mutation_id)
|
||||
});
|
||||
// Capture the exact source/tier identity before any client-paced read, but
|
||||
// defer both the tier lease and source/target write locks to the final
|
||||
// publication. Decommission already owns main's fixed-domain mutation
|
||||
// fence, so reacquiring that domain as a write lock would self-deadlock.
|
||||
let remote_tuple_publication_fence = store
|
||||
.acquire_remote_tuple_publication_fence(&bucket, pool_idx, &object_info, false)
|
||||
.await?;
|
||||
let has_part_checksums = object_info
|
||||
.parts
|
||||
.iter()
|
||||
@@ -1656,8 +1664,8 @@ async fn migrate_object_inner(
|
||||
}
|
||||
let mut cleanup_opts =
|
||||
data_movement_abort_opts(pool_idx, source_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
|
||||
if let Some(fence) = mutation_fence {
|
||||
fence.add_namespace_lock_fence(&mut cleanup_opts);
|
||||
if let Some(anchor) = mutation_fence.as_ref() {
|
||||
anchor.guard().add_namespace_lock_fence(&mut cleanup_opts);
|
||||
}
|
||||
if let Some(fence) = multipart_mutation_fence.as_ref() {
|
||||
fence.add_namespace_lock_fence(&mut cleanup_opts);
|
||||
@@ -1684,7 +1692,12 @@ async fn migrate_object_inner(
|
||||
}
|
||||
}
|
||||
let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
|
||||
.handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
|
||||
.handle_new_multipart_upload_with_pool_idx(
|
||||
&bucket,
|
||||
&object_info.name,
|
||||
&new_multipart_opts,
|
||||
mutation_fence.as_ref().map(DecommissionFixedReadAnchor::guard),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(res) => res,
|
||||
@@ -1797,15 +1810,20 @@ async fn migrate_object_inner(
|
||||
if let Some(fence) = multipart_mutation_fence.as_ref() {
|
||||
fence.add_namespace_lock_fence(&mut complete_multipart_opts);
|
||||
}
|
||||
let remote_tuple_publication_fence = match mutation_fence.take() {
|
||||
Some(anchor) => remote_tuple_publication_fence.under_fixed_read_anchor(anchor)?,
|
||||
None => remote_tuple_publication_fence,
|
||||
};
|
||||
if let Err(err) = store
|
||||
.clone()
|
||||
.complete_multipart_upload_for_data_movement(
|
||||
(target_pool_idx, mutation_fence),
|
||||
.complete_multipart_upload_for_data_movement_with_publication_fence(
|
||||
target_pool_idx,
|
||||
&bucket,
|
||||
&object_info.name,
|
||||
&res.upload_id,
|
||||
parts,
|
||||
&complete_multipart_opts,
|
||||
remote_tuple_publication_fence,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -1849,8 +1867,8 @@ async fn migrate_object_inner(
|
||||
if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) {
|
||||
let mut abort_opts =
|
||||
data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
|
||||
if let Some(fence) = mutation_fence {
|
||||
fence.add_namespace_lock_fence(&mut abort_opts);
|
||||
if let Some(anchor) = mutation_fence.as_ref() {
|
||||
anchor.guard().add_namespace_lock_fence(&mut abort_opts);
|
||||
}
|
||||
if let Some(fence) = multipart_mutation_fence.as_ref() {
|
||||
fence.add_namespace_lock_fence(&mut abort_opts);
|
||||
@@ -1923,12 +1941,12 @@ async fn migrate_object_inner(
|
||||
|
||||
if let Err(primary_err) = multipart_result {
|
||||
if should_abort_multipart_upload(&abort_multipart_flag) {
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pause_data_movement_multipart_before_abort(&bucket, &object_info.name).await;
|
||||
let mut abort_opts =
|
||||
data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
|
||||
if let Some(fence) = mutation_fence {
|
||||
fence.add_namespace_lock_fence(&mut abort_opts);
|
||||
if let Some(anchor) = mutation_fence.as_ref() {
|
||||
anchor.guard().add_namespace_lock_fence(&mut abort_opts);
|
||||
}
|
||||
if let Some(fence) = multipart_mutation_fence.as_ref() {
|
||||
fence.add_namespace_lock_fence(&mut abort_opts);
|
||||
@@ -1982,8 +2000,18 @@ async fn migrate_object_inner(
|
||||
if let Some(signal) = lock_lost_signal {
|
||||
put_opts.add_namespace_lock_lost_signal(signal);
|
||||
}
|
||||
let remote_tuple_publication_fence = match mutation_fence.take() {
|
||||
Some(anchor) => remote_tuple_publication_fence.under_fixed_read_anchor(anchor)?,
|
||||
None => remote_tuple_publication_fence,
|
||||
};
|
||||
let (target_pool_idx, put_result) = store
|
||||
.put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence)
|
||||
.put_object_for_data_movement_with_publication_fence(
|
||||
&bucket,
|
||||
&object_info.name,
|
||||
&mut data,
|
||||
&put_opts,
|
||||
remote_tuple_publication_fence,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?;
|
||||
if let Err(err) = put_result {
|
||||
|
||||
@@ -278,3 +278,17 @@ fn reduce_errs_buckets_identical_other_messages_together() {
|
||||
assert_eq!(count, 3);
|
||||
assert_eq!(err, Some(DiskError::other("can not get client")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stable_io_context_buckets_by_cause_and_preserves_diagnostic_source() {
|
||||
let first = StorageError::other_with_context("tier mutation intent changed", "mutation-a");
|
||||
let second = StorageError::other_with_context("tier mutation intent changed", "mutation-b");
|
||||
|
||||
assert_eq!(first, second, "diagnostic identity must not split quorum buckets");
|
||||
let StorageError::Io(io_error) = first else {
|
||||
panic!("stable context must remain an io error");
|
||||
};
|
||||
assert_eq!(io_error.to_string(), "tier mutation intent changed");
|
||||
let context = io_error.get_ref().expect("stable context must remain downcastable");
|
||||
assert_eq!(context.source().expect("diagnostic source must be retained").to_string(), "mutation-a");
|
||||
}
|
||||
|
||||
@@ -23,6 +23,36 @@ use s3s::S3ErrorCode;
|
||||
pub type Error = StorageError;
|
||||
pub type Result<T> = core::result::Result<T, Error>;
|
||||
|
||||
/// Keeps high-cardinality diagnostic detail in the error source while making
|
||||
/// the rendered `io::Error` stable for quorum aggregation.
|
||||
#[derive(Debug)]
|
||||
struct StableIoContextError {
|
||||
message: &'static str,
|
||||
source: Box<dyn std::error::Error + Send + Sync>,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for StableIoContextError {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str(self.message)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for StableIoContextError {
|
||||
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
|
||||
Some(self.source.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn stable_io_error<E>(message: &'static str, source: E) -> std::io::Error
|
||||
where
|
||||
E: Into<Box<dyn std::error::Error + Send + Sync>>,
|
||||
{
|
||||
std::io::Error::other(StableIoContextError {
|
||||
message,
|
||||
source: source.into(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Storage layer error type covering disk, volume, bucket, object, multipart,
|
||||
/// erasure-coding, and operational error conditions.
|
||||
///
|
||||
@@ -264,6 +294,13 @@ impl StorageError {
|
||||
StorageError::Io(std::io::Error::other(error))
|
||||
}
|
||||
|
||||
pub(crate) fn other_with_context<E>(message: &'static str, source: E) -> Self
|
||||
where
|
||||
E: Into<Box<dyn std::error::Error + Send + Sync>>,
|
||||
{
|
||||
StorageError::Io(stable_io_error(message, source))
|
||||
}
|
||||
|
||||
pub fn is_not_found(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
|
||||
@@ -301,36 +301,23 @@ pub enum LifecycleDeleteAllPhase {
|
||||
#[doc(hidden)]
|
||||
#[derive(Default)]
|
||||
pub struct LifecycleDeleteAllJournalState {
|
||||
prepared: HashMap<String, crate::bucket::lifecycle::tier_sweeper::Jentry>,
|
||||
mutation_started: bool,
|
||||
}
|
||||
|
||||
impl Debug for LifecycleDeleteAllJournalState {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("LifecycleDeleteAllJournalState")
|
||||
.field("prepared_count", &self.prepared.len())
|
||||
.field("mutation_started", &self.mutation_started)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl LifecycleDeleteAllJournalState {
|
||||
pub(crate) fn contains(&self, name: &str) -> bool {
|
||||
self.prepared.contains_key(name)
|
||||
}
|
||||
|
||||
pub(crate) fn insert(&mut self, name: String, entry: crate::bucket::lifecycle::tier_sweeper::Jentry) {
|
||||
self.prepared.insert(name, entry);
|
||||
}
|
||||
|
||||
pub(crate) fn prepared_entries(&self) -> Vec<crate::bucket::lifecycle::tier_sweeper::Jentry> {
|
||||
self.prepared.values().cloned().collect()
|
||||
}
|
||||
|
||||
pub(crate) fn mark_mutation_started(&mut self) {
|
||||
self.mutation_started = true;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn mutation_started(&self) -> bool {
|
||||
self.mutation_started
|
||||
}
|
||||
@@ -706,6 +693,12 @@ pub struct ObjectOptions {
|
||||
pub lifecycle_delete_all: Option<LifecycleDeleteAllRequest>,
|
||||
#[doc(hidden)]
|
||||
pub lifecycle_delete_all_journal: Option<Arc<parking_lot::Mutex<LifecycleDeleteAllJournalState>>>,
|
||||
/// Whole-operation authorization created only by consuming a validated
|
||||
/// v6 dispatch-manifest permit. Clones share the authorization, not the
|
||||
/// one-shot permit itself.
|
||||
#[doc(hidden)]
|
||||
pub tier_delete_dispatch_authorization:
|
||||
Option<crate::bucket::lifecycle::tier_delete_journal::TierDeleteDispatchAuthorization>,
|
||||
/// RustFS-only compare-and-set condition checked under the object write lock.
|
||||
pub expected_current_version_id: Option<String>,
|
||||
/// Persisted bucket incarnation observed before authorization.
|
||||
@@ -847,6 +840,7 @@ impl std::fmt::Debug for ObjectOptions {
|
||||
.field("version_id", &self.version_id.is_some())
|
||||
.field("lifecycle_delete_all", &self.lifecycle_delete_all.is_some())
|
||||
.field("lifecycle_delete_all_journal", &self.lifecycle_delete_all_journal.is_some())
|
||||
.field("tier_delete_dispatch_authorization", &self.tier_delete_dispatch_authorization.is_some())
|
||||
.field("expected_current_version_id", &self.expected_current_version_id.is_some())
|
||||
.field("expected_bucket_incarnation_id", &self.expected_bucket_incarnation_id)
|
||||
.field("no_lock", &self.no_lock)
|
||||
@@ -972,7 +966,7 @@ impl ObjectOptions {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn add_namespace_lock_fence_for_test(&mut self, fence: &NamespaceLockFence) {
|
||||
pub(crate) fn add_namespace_lock_fence(&mut self, fence: &NamespaceLockFence) {
|
||||
self.namespace_lock_fence
|
||||
.get_or_insert_with(NamespaceLockFence::new)
|
||||
.extend(fence);
|
||||
|
||||
@@ -29,10 +29,14 @@ use rustfs_madmin::metrics::RealtimeMetrics;
|
||||
use rustfs_madmin::net::NetInfo;
|
||||
use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
|
||||
use rustfs_utils::XHost;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
|
||||
use std::future::Future;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
use std::sync::{
|
||||
Arc, Mutex, OnceLock,
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
use tokio::time::{sleep, timeout};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -52,6 +56,20 @@ const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
|
||||
const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
|
||||
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
|
||||
const TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION: u32 = 3;
|
||||
type CrossPoolFencePolicyResult = Result<BTreeMap<String, Uuid>>;
|
||||
|
||||
fn cross_pool_fence_policy_results(
|
||||
peer_epochs: BTreeMap<String, Uuid>,
|
||||
minimum_version: u32,
|
||||
) -> (CrossPoolFencePolicyResult, CrossPoolFencePolicyResult) {
|
||||
let journal_result = if minimum_version >= TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION {
|
||||
Ok(peer_epochs.clone())
|
||||
} else {
|
||||
Err(Error::other("tier delete journal v6 policy capability version is unsupported"))
|
||||
};
|
||||
(Ok(peer_epochs), journal_result)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ScannerPublicationLeaseGrant {
|
||||
@@ -107,15 +125,91 @@ struct FleetCapabilityProof {
|
||||
topology_fingerprint: String,
|
||||
peer_epochs: Arc<BTreeMap<String, Uuid>>,
|
||||
expires_at: Instant,
|
||||
generation: Arc<FleetCapabilityProofGeneration>,
|
||||
}
|
||||
|
||||
impl FleetCapabilityProof {
|
||||
fn new(topology_fingerprint: String, peer_epochs: Arc<BTreeMap<String, Uuid>>, expires_at: Instant) -> Self {
|
||||
Self {
|
||||
topology_fingerprint,
|
||||
peer_epochs,
|
||||
expires_at,
|
||||
generation: FleetCapabilityProofGeneration::fresh(),
|
||||
}
|
||||
}
|
||||
|
||||
fn token(&self) -> FleetCapabilityProofToken {
|
||||
FleetCapabilityProofToken {
|
||||
topology_fingerprint: self.topology_fingerprint.clone(),
|
||||
peer_epochs: self.peer_epochs.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
fn with_fresh_generation(&self) -> Self {
|
||||
Self::new(self.topology_fingerprint.clone(), Arc::clone(&self.peer_epochs), self.expires_at)
|
||||
}
|
||||
}
|
||||
|
||||
/// Admission generation for effects that must not straddle a fleet-proof
|
||||
/// replacement. Revocation is deliberately non-blocking: it closes admission
|
||||
/// immediately, while the proof slot withholds the successor generation until
|
||||
/// every admitted operation has drained.
|
||||
#[derive(Default)]
|
||||
struct FleetCapabilityProofGeneration {
|
||||
accepting: AtomicBool,
|
||||
active: AtomicUsize,
|
||||
}
|
||||
|
||||
impl FleetCapabilityProofGeneration {
|
||||
fn fresh() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
accepting: AtomicBool::new(true),
|
||||
active: AtomicUsize::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
fn try_acquire(self: &Arc<Self>) -> Option<FleetCapabilityProofPermit> {
|
||||
if !self.accepting.load(Ordering::Acquire) {
|
||||
return None;
|
||||
}
|
||||
self.active.fetch_add(1, Ordering::AcqRel);
|
||||
if self.accepting.load(Ordering::Acquire) {
|
||||
Some(FleetCapabilityProofPermit {
|
||||
generation: Arc::clone(self),
|
||||
})
|
||||
} else {
|
||||
self.release();
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn revoke(&self) {
|
||||
self.accepting.store(false, Ordering::Release);
|
||||
}
|
||||
|
||||
fn is_accepting(&self) -> bool {
|
||||
self.accepting.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
fn is_drained(&self) -> bool {
|
||||
self.active.load(Ordering::Acquire) == 0
|
||||
}
|
||||
|
||||
fn release(&self) {
|
||||
let previous = self.active.fetch_sub(1, Ordering::AcqRel);
|
||||
debug_assert!(previous > 0, "fleet capability permit count underflow");
|
||||
}
|
||||
}
|
||||
|
||||
struct FleetCapabilityProofPermit {
|
||||
generation: Arc<FleetCapabilityProofGeneration>,
|
||||
}
|
||||
|
||||
impl Drop for FleetCapabilityProofPermit {
|
||||
fn drop(&mut self) {
|
||||
self.generation.release();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
@@ -127,6 +221,7 @@ struct FleetCapabilityProofToken {
|
||||
#[derive(Default)]
|
||||
struct FleetCapabilityProofState {
|
||||
proof: Option<FleetCapabilityProof>,
|
||||
draining_generation: Option<Arc<FleetCapabilityProofGeneration>>,
|
||||
topology_conflict: bool,
|
||||
}
|
||||
|
||||
@@ -136,8 +231,17 @@ pub(crate) struct RemoteVersionStateFleetProofToken(FleetCapabilityProofToken);
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct CrossPoolFenceFleetProofToken(FleetCapabilityProofToken);
|
||||
|
||||
/// A point-in-time proof that every current storage member implements the v6
|
||||
/// dispatch-manifest policy. It intentionally has no `Clone` implementation:
|
||||
/// one acquisition authorizes one manifest construction attempt.
|
||||
pub(crate) struct TierDeleteJournalFleetProofToken {
|
||||
token: FleetCapabilityProofToken,
|
||||
_permit: FleetCapabilityProofPermit,
|
||||
}
|
||||
|
||||
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static TIER_DELETE_JOURNAL_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
|
||||
|
||||
fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
@@ -148,8 +252,35 @@ fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCa
|
||||
REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn replace_fleet_capability_proof(slot: &std::sync::RwLock<FleetCapabilityProofState>, proof: Option<FleetCapabilityProof>) {
|
||||
slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
|
||||
fn tier_delete_journal_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
TIER_DELETE_JOURNAL_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) {
|
||||
if let Some(proof) = state.proof.take() {
|
||||
proof.generation.revoke();
|
||||
if !proof.generation.is_drained() {
|
||||
state.draining_generation = Some(proof.generation);
|
||||
}
|
||||
}
|
||||
if state
|
||||
.draining_generation
|
||||
.as_ref()
|
||||
.is_some_and(|generation| generation.is_drained())
|
||||
{
|
||||
state.draining_generation = None;
|
||||
}
|
||||
}
|
||||
|
||||
fn revoke_fleet_capability_proof(slot: &std::sync::RwLock<FleetCapabilityProofState>) {
|
||||
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
revoke_fleet_capability_proof_state(&mut state);
|
||||
}
|
||||
|
||||
fn mark_fleet_capability_topology_conflict(slot: &std::sync::RwLock<FleetCapabilityProofState>) {
|
||||
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.topology_conflict = true;
|
||||
revoke_fleet_capability_proof_state(&mut state);
|
||||
}
|
||||
|
||||
fn publish_fleet_capability_probe_result(
|
||||
@@ -161,21 +292,42 @@ fn publish_fleet_capability_probe_result(
|
||||
match result {
|
||||
Ok(peer_epochs) => {
|
||||
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let peer_epochs = state
|
||||
if let Some(current) = state
|
||||
.proof
|
||||
.as_ref()
|
||||
.as_mut()
|
||||
.filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
|
||||
.map(|proof| Arc::clone(&proof.peer_epochs))
|
||||
.unwrap_or_else(|| Arc::new(peer_epochs));
|
||||
state.proof = Some(FleetCapabilityProof {
|
||||
topology_fingerprint: topology_fingerprint.to_string(),
|
||||
peer_epochs,
|
||||
expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
|
||||
});
|
||||
{
|
||||
current.expires_at = observed_at + REMOTE_VERSION_STATE_PROOF_TTL;
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(previous) = state.proof.take() {
|
||||
previous.generation.revoke();
|
||||
if !previous.generation.is_drained() {
|
||||
state.draining_generation = Some(previous.generation);
|
||||
}
|
||||
}
|
||||
if state
|
||||
.draining_generation
|
||||
.as_ref()
|
||||
.is_some_and(|generation| generation.is_drained())
|
||||
{
|
||||
state.draining_generation = None;
|
||||
}
|
||||
if state.draining_generation.is_some() {
|
||||
return Some(Error::other(
|
||||
"fleet capability proof successor waits for the previous generation to drain",
|
||||
));
|
||||
}
|
||||
state.proof = Some(FleetCapabilityProof::new(
|
||||
topology_fingerprint.to_string(),
|
||||
Arc::new(peer_epochs),
|
||||
observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
|
||||
));
|
||||
None
|
||||
}
|
||||
Err(err) => {
|
||||
replace_fleet_capability_proof(slot, None);
|
||||
revoke_fleet_capability_proof(slot);
|
||||
Some(err)
|
||||
}
|
||||
}
|
||||
@@ -216,7 +368,72 @@ pub fn cross_pool_fence_fleet_proof_matches(proof: &CrossPoolFenceFleetProofToke
|
||||
fleet_capability_proof_matches(cross_pool_fence_fleet_proof_slot(), &proof.0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn acquire_tier_delete_journal_fleet_proof() -> Option<TierDeleteJournalFleetProofToken> {
|
||||
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||
let state = tier_delete_journal_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
acquire_tier_delete_journal_fleet_proof_from(&state, expected_topology, Instant::now())
|
||||
}
|
||||
|
||||
fn acquire_tier_delete_journal_fleet_proof_from(
|
||||
state: &FleetCapabilityProofState,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> Option<TierDeleteJournalFleetProofToken> {
|
||||
let token = acquire_fleet_capability_proof_from(state, expected_topology, now)?;
|
||||
let permit = state.proof.as_ref()?.generation.try_acquire()?;
|
||||
Some(TierDeleteJournalFleetProofToken { token, _permit: permit })
|
||||
}
|
||||
|
||||
pub(crate) fn tier_delete_journal_fleet_proof_matches(proof: &TierDeleteJournalFleetProofToken) -> bool {
|
||||
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
|
||||
return false;
|
||||
};
|
||||
let state = tier_delete_journal_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
tier_delete_journal_fleet_proof_matches_at(&state, proof, expected_topology, Instant::now())
|
||||
}
|
||||
|
||||
fn tier_delete_journal_fleet_proof_matches_at(
|
||||
state: &FleetCapabilityProofState,
|
||||
proof: &TierDeleteJournalFleetProofToken,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
proof._permit.generation.is_accepting()
|
||||
&& fleet_capability_proof_matches_at(state, &proof.token, expected_topology, now)
|
||||
&& state
|
||||
.proof
|
||||
.as_ref()
|
||||
.is_some_and(|current| Arc::ptr_eq(¤t.generation, &proof._permit.generation))
|
||||
}
|
||||
|
||||
pub(crate) fn tier_delete_journal_topology_generation(proof: &TierDeleteJournalFleetProofToken) -> String {
|
||||
stable_tier_delete_journal_topology_generation(&proof.token.topology_fingerprint)
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) fn tier_delete_journal_fleet_proof_has_inflight_for_test() -> bool {
|
||||
let state = tier_delete_journal_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.proof.as_ref().is_some_and(|proof| !proof.generation.is_drained())
|
||||
|| state
|
||||
.draining_generation
|
||||
.as_ref()
|
||||
.is_some_and(|generation| !generation.is_drained())
|
||||
}
|
||||
|
||||
fn stable_tier_delete_journal_topology_generation(topology_fingerprint: &str) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(b"rustfs-tier-delete-journal-topology-v1\0");
|
||||
hasher.update(topology_fingerprint.as_bytes());
|
||||
rustfs_utils::crypto::hex(hasher.finalize().as_slice())
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
|
||||
let topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY
|
||||
.get()
|
||||
@@ -226,18 +443,39 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
|
||||
let mut state = cross_pool_fence_fleet_proof_slot()
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let now = Instant::now();
|
||||
let proof = if !state.topology_conflict && fleet_capability_proof_valid_at(state.proof.as_ref(), &topology, now) {
|
||||
state.proof.clone()
|
||||
} else {
|
||||
Some(FleetCapabilityProof::new(
|
||||
topology,
|
||||
Arc::new(BTreeMap::new()),
|
||||
now + Duration::from_secs(60 * 60),
|
||||
))
|
||||
};
|
||||
state.topology_conflict = false;
|
||||
state.proof = Some(FleetCapabilityProof {
|
||||
topology_fingerprint: topology,
|
||||
peer_epochs: Arc::new(BTreeMap::new()),
|
||||
expires_at: Instant::now() + Duration::from_secs(60 * 60),
|
||||
});
|
||||
state.proof = proof.clone();
|
||||
drop(state);
|
||||
let mut journal_state = tier_delete_journal_fleet_proof_slot()
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
debug_assert!(
|
||||
journal_state
|
||||
.proof
|
||||
.as_ref()
|
||||
.is_none_or(|current| current.generation.is_drained())
|
||||
);
|
||||
journal_state.topology_conflict = false;
|
||||
journal_state.draining_generation = None;
|
||||
journal_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct CrossPoolFenceFleetProofGuard {
|
||||
previous_proof: Option<FleetCapabilityProof>,
|
||||
previous_topology_conflict: bool,
|
||||
previous_journal_proof: Option<FleetCapabilityProof>,
|
||||
previous_journal_topology_conflict: bool,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -246,8 +484,24 @@ impl Drop for CrossPoolFenceFleetProofGuard {
|
||||
let mut state = cross_pool_fence_fleet_proof_slot()
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.proof = self.previous_proof.take();
|
||||
state.proof = self
|
||||
.previous_proof
|
||||
.take()
|
||||
.as_ref()
|
||||
.map(FleetCapabilityProof::with_fresh_generation);
|
||||
state.draining_generation = None;
|
||||
state.topology_conflict = self.previous_topology_conflict;
|
||||
drop(state);
|
||||
let mut journal_state = tier_delete_journal_fleet_proof_slot()
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
journal_state.proof = self
|
||||
.previous_journal_proof
|
||||
.take()
|
||||
.as_ref()
|
||||
.map(FleetCapabilityProof::with_fresh_generation);
|
||||
journal_state.draining_generation = None;
|
||||
journal_state.topology_conflict = self.previous_journal_topology_conflict;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -258,12 +512,29 @@ pub(crate) fn without_cross_pool_fence_fleet_proof_for_test() -> CrossPoolFenceF
|
||||
let mut state = cross_pool_fence_fleet_proof_slot()
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let mut journal_state = tier_delete_journal_fleet_proof_slot()
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let guard = CrossPoolFenceFleetProofGuard {
|
||||
previous_proof: state.proof.clone(),
|
||||
previous_topology_conflict: state.topology_conflict,
|
||||
previous_journal_proof: journal_state.proof.clone(),
|
||||
previous_journal_topology_conflict: journal_state.topology_conflict,
|
||||
};
|
||||
state.proof = None;
|
||||
if let Some(proof) = state.proof.take() {
|
||||
proof.generation.revoke();
|
||||
if !proof.generation.is_drained() {
|
||||
state.draining_generation = Some(proof.generation);
|
||||
}
|
||||
}
|
||||
state.topology_conflict = true;
|
||||
if let Some(proof) = journal_state.proof.take() {
|
||||
proof.generation.revoke();
|
||||
if !proof.generation.is_drained() {
|
||||
journal_state.draining_generation = Some(proof.generation);
|
||||
}
|
||||
}
|
||||
journal_state.topology_conflict = true;
|
||||
guard
|
||||
}
|
||||
|
||||
@@ -275,11 +546,33 @@ pub fn rotate_cross_pool_fence_fleet_proof_for_test() -> bool {
|
||||
let Some(current) = state.proof.as_ref() else {
|
||||
return false;
|
||||
};
|
||||
state.proof = Some(FleetCapabilityProof {
|
||||
topology_fingerprint: current.topology_fingerprint.clone(),
|
||||
peer_epochs: Arc::new(current.peer_epochs.as_ref().clone()),
|
||||
expires_at: current.expires_at,
|
||||
});
|
||||
let proof = FleetCapabilityProof::new(
|
||||
current.topology_fingerprint.clone(),
|
||||
Arc::new(current.peer_epochs.as_ref().clone()),
|
||||
current.expires_at,
|
||||
);
|
||||
state.proof = Some(proof.clone());
|
||||
drop(state);
|
||||
let mut journal_state = tier_delete_journal_fleet_proof_slot()
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
journal_state.topology_conflict = false;
|
||||
if let Some(previous) = journal_state.proof.take() {
|
||||
previous.generation.revoke();
|
||||
if !previous.generation.is_drained() {
|
||||
journal_state.draining_generation = Some(previous.generation);
|
||||
}
|
||||
}
|
||||
if journal_state
|
||||
.draining_generation
|
||||
.as_ref()
|
||||
.is_some_and(|generation| generation.is_drained())
|
||||
{
|
||||
journal_state.draining_generation = None;
|
||||
}
|
||||
if journal_state.draining_generation.is_none() {
|
||||
journal_state.proof = Some(proof.with_fresh_generation());
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
@@ -291,15 +584,22 @@ fn fleet_capability_proof_matches(
|
||||
return false;
|
||||
};
|
||||
let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if state.topology_conflict {
|
||||
return false;
|
||||
}
|
||||
state.proof.as_ref().is_some_and(|current| {
|
||||
current.topology_fingerprint == *expected_topology
|
||||
&& current.topology_fingerprint == proof.topology_fingerprint
|
||||
&& Arc::ptr_eq(¤t.peer_epochs, &proof.peer_epochs)
|
||||
&& Instant::now() < current.expires_at
|
||||
})
|
||||
fleet_capability_proof_matches_at(&state, proof, expected_topology, Instant::now())
|
||||
}
|
||||
|
||||
fn fleet_capability_proof_matches_at(
|
||||
state: &FleetCapabilityProofState,
|
||||
proof: &FleetCapabilityProofToken,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
!state.topology_conflict
|
||||
&& state.proof.as_ref().is_some_and(|current| {
|
||||
current.topology_fingerprint == expected_topology
|
||||
&& current.topology_fingerprint == proof.topology_fingerprint
|
||||
&& Arc::ptr_eq(¤t.peer_epochs, &proof.peer_epochs)
|
||||
&& now < current.expires_at
|
||||
})
|
||||
}
|
||||
|
||||
fn fleet_capability_proof_valid_at(proof: Option<&FleetCapabilityProof>, expected_topology: &str, now: Instant) -> bool {
|
||||
@@ -312,7 +612,7 @@ pub(crate) struct RemoteVersionStateFleetProofGuard;
|
||||
#[cfg(test)]
|
||||
impl Drop for RemoteVersionStateFleetProofGuard {
|
||||
fn drop(&mut self) {
|
||||
replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None);
|
||||
revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -348,10 +648,12 @@ fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, pe
|
||||
pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
|
||||
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
|
||||
for slot in [remote_version_state_fleet_proof_slot(), cross_pool_fence_fleet_proof_slot()] {
|
||||
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.topology_conflict = true;
|
||||
state.proof = None;
|
||||
for slot in [
|
||||
remote_version_state_fleet_proof_slot(),
|
||||
cross_pool_fence_fleet_proof_slot(),
|
||||
tier_delete_journal_fleet_proof_slot(),
|
||||
] {
|
||||
mark_fleet_capability_topology_conflict(slot);
|
||||
}
|
||||
}
|
||||
return;
|
||||
@@ -373,7 +675,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
}
|
||||
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
|
||||
};
|
||||
let fence_result = match get_global_notification_sys() {
|
||||
let fence_probe = match get_global_notification_sys() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
|
||||
@@ -382,13 +684,21 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
|
||||
None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
|
||||
};
|
||||
let (fence_result, journal_result) = match fence_probe {
|
||||
Ok((peer_epochs, minimum_version)) => cross_pool_fence_policy_results(peer_epochs, minimum_version),
|
||||
Err(err) => {
|
||||
let message = err.to_string();
|
||||
(Err(Error::other(message.clone())), Err(Error::other(message)))
|
||||
}
|
||||
};
|
||||
let topology_conflict = remote_version_state_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.topology_conflict;
|
||||
if topology_conflict {
|
||||
replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None);
|
||||
replace_fleet_capability_proof(cross_pool_fence_fleet_proof_slot(), None);
|
||||
revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(cross_pool_fence_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot());
|
||||
} else if let Some(err) = publish_fleet_capability_probe_result(
|
||||
remote_version_state_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
@@ -409,7 +719,25 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
capability = "cross_pool_fence_v2",
|
||||
capability = "cross_pool_fence",
|
||||
state = "failed_closed",
|
||||
error = %err,
|
||||
"notification capability probe"
|
||||
);
|
||||
}
|
||||
if !topology_conflict
|
||||
&& let Some(err) = publish_fleet_capability_probe_result(
|
||||
tier_delete_journal_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
journal_result,
|
||||
Instant::now(),
|
||||
)
|
||||
{
|
||||
debug!(
|
||||
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
capability = "tier_delete_journal_v6_policy",
|
||||
state = "failed_closed",
|
||||
error = %err,
|
||||
"notification capability probe"
|
||||
@@ -483,7 +811,7 @@ impl NotificationSys {
|
||||
Ok(peer_epochs)
|
||||
}
|
||||
|
||||
async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
|
||||
async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result<(BTreeMap<String, Uuid>, u32)> {
|
||||
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||
return Err(Error::other("cross-pool fence capability fleet membership is incomplete"));
|
||||
}
|
||||
@@ -494,14 +822,21 @@ impl NotificationSys {
|
||||
client.probe_cross_pool_fence(topology_fingerprint.to_string()).await
|
||||
});
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
let mut minimum_version = u32::MAX;
|
||||
for result in join_all(probes).await {
|
||||
let (peer, version, epoch) = result?;
|
||||
if version < CROSS_POOL_FENCE_SUPPORTED_VERSION {
|
||||
return Err(Error::other("cross-pool fence capability version is unsupported"));
|
||||
}
|
||||
minimum_version = minimum_version.min(version);
|
||||
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
|
||||
}
|
||||
Ok(peer_epochs)
|
||||
// A single-node deployment has no remote member to lower the local
|
||||
// policy version advertised by this binary.
|
||||
if minimum_version == u32::MAX {
|
||||
minimum_version = TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION;
|
||||
}
|
||||
Ok((peer_epochs, minimum_version))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1827,12 +2162,13 @@ impl NotificationSys {
|
||||
join_all(futures).await
|
||||
}
|
||||
|
||||
pub async fn abort_tier_mutation(&self, mutation_id: Uuid) -> Vec<NotificationPeerErr> {
|
||||
pub async fn abort_tier_mutation(&self, mutation_id: Uuid, canonical_prepare_payload: Bytes) -> Vec<NotificationPeerErr> {
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
for client in self.peer_clients.iter().cloned() {
|
||||
let payload = canonical_prepare_payload.clone();
|
||||
futures.push(async move {
|
||||
if let Some(client) = client {
|
||||
notification_peer_result(client.host.to_string(), client.abort_tier_mutation(mutation_id).await)
|
||||
notification_peer_result(client.host.to_string(), client.abort_tier_mutation(mutation_id, payload).await)
|
||||
} else {
|
||||
unreachable_notification_peer_err()
|
||||
}
|
||||
@@ -2467,16 +2803,24 @@ fn aggregate_scanner_dirty_usage_acknowledgement_results(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn cross_pool_v2_remains_generic_but_cannot_authorize_v6_journal() {
|
||||
let peers = BTreeMap::from([("node-b:9000".to_string(), Uuid::new_v4())]);
|
||||
let (generic_v2, journal_v2) = cross_pool_fence_policy_results(peers.clone(), 2);
|
||||
assert!(generic_v2.is_ok(), "v2 remains valid for existing cross-pool fencing");
|
||||
assert!(journal_v2.is_err(), "a mixed v2/v3 fleet must fail closed for journal-v6 deletion");
|
||||
|
||||
let (generic_v3, journal_v3) = cross_pool_fence_policy_results(peers, 3);
|
||||
assert!(generic_v3.is_ok());
|
||||
assert!(journal_v3.is_ok(), "an all-v3 fleet may authorize journal-v6 deletion");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_rejects_stale_or_mismatched_membership() {
|
||||
let now = Instant::now();
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
|
||||
let proof = FleetCapabilityProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(peer_epochs),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
};
|
||||
let proof = FleetCapabilityProof::new("topology-a".to_string(), Arc::new(peer_epochs), now + Duration::from_secs(1));
|
||||
|
||||
assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now));
|
||||
assert!(!fleet_capability_proof_valid_at(Some(&proof), "topology-b", now));
|
||||
@@ -2495,11 +2839,7 @@ mod tests {
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_accepts_single_node_membership() {
|
||||
let now = Instant::now();
|
||||
let proof = FleetCapabilityProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(BTreeMap::new()),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
};
|
||||
let proof = FleetCapabilityProof::new("topology-a".to_string(), Arc::new(BTreeMap::new()), now + Duration::from_secs(1));
|
||||
|
||||
assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now));
|
||||
}
|
||||
@@ -2507,21 +2847,97 @@ mod tests {
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
|
||||
let now = Instant::now();
|
||||
let proof = FleetCapabilityProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
};
|
||||
let proof = FleetCapabilityProof::new(
|
||||
"topology-a".to_string(),
|
||||
Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
|
||||
now + Duration::from_secs(1),
|
||||
);
|
||||
let captured = proof.token();
|
||||
let restarted = FleetCapabilityProof {
|
||||
topology_fingerprint: proof.topology_fingerprint.clone(),
|
||||
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
|
||||
expires_at: proof.expires_at,
|
||||
};
|
||||
let restarted = FleetCapabilityProof::new(
|
||||
proof.topology_fingerprint.clone(),
|
||||
Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
|
||||
proof.expires_at,
|
||||
);
|
||||
|
||||
assert!(captured != restarted.token());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_generation_is_stable_across_members_and_process_restarts() {
|
||||
let topology = "topology-a";
|
||||
let now = Instant::now();
|
||||
let node_a_view = FleetCapabilityProof::new(
|
||||
topology.to_string(),
|
||||
Arc::new(BTreeMap::from([("node-b".to_string(), Uuid::new_v4())])),
|
||||
now + Duration::from_secs(1),
|
||||
);
|
||||
let node_b_view = FleetCapabilityProof::new(
|
||||
topology.to_string(),
|
||||
Arc::new(BTreeMap::from([("node-a".to_string(), Uuid::new_v4())])),
|
||||
now + Duration::from_secs(1),
|
||||
);
|
||||
let restarted_node_a_view = FleetCapabilityProof::new(
|
||||
topology.to_string(),
|
||||
Arc::new(BTreeMap::from([("node-b".to_string(), Uuid::new_v4())])),
|
||||
now + Duration::from_secs(1),
|
||||
);
|
||||
|
||||
let generations = [&node_a_view, &node_b_view, &restarted_node_a_view]
|
||||
.map(|proof| stable_tier_delete_journal_topology_generation(&proof.token().topology_fingerprint));
|
||||
assert_eq!(generations[0], generations[1]);
|
||||
assert_eq!(generations[0], generations[2]);
|
||||
assert_ne!(
|
||||
generations[0],
|
||||
stable_tier_delete_journal_topology_generation("topology-b"),
|
||||
"a real topology change must produce a different durable generation"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_restart_revokes_old_token_but_fresh_token_recovers_same_generation() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let now = Instant::now();
|
||||
let original_peers = BTreeMap::from([("node-b".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original_peers), now).is_none());
|
||||
let original = slot
|
||||
.read()
|
||||
.expect("proof slot should not poison")
|
||||
.proof
|
||||
.as_ref()
|
||||
.expect("successful probe should publish proof")
|
||||
.token();
|
||||
let original_generation = stable_tier_delete_journal_topology_generation(&original.topology_fingerprint);
|
||||
|
||||
let restarted_peers = BTreeMap::from([("node-b".to_string(), Uuid::new_v4())]);
|
||||
assert!(
|
||||
publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted_peers), now + Duration::from_millis(1))
|
||||
.is_none()
|
||||
);
|
||||
let state = slot.read().expect("proof slot should not poison");
|
||||
let fresh = state
|
||||
.proof
|
||||
.as_ref()
|
||||
.expect("restart probe should publish a fresh proof")
|
||||
.token();
|
||||
|
||||
assert!(!fleet_capability_proof_matches_at(
|
||||
&state,
|
||||
&original,
|
||||
"topology-a",
|
||||
now + Duration::from_millis(2)
|
||||
));
|
||||
assert!(fleet_capability_proof_matches_at(
|
||||
&state,
|
||||
&fresh,
|
||||
"topology-a",
|
||||
now + Duration::from_millis(2)
|
||||
));
|
||||
assert_eq!(
|
||||
original_generation,
|
||||
stable_tier_delete_journal_topology_generation(&fresh.topology_fingerprint)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
@@ -2561,15 +2977,106 @@ mod tests {
|
||||
assert!(!Arc::ptr_eq(&original.peer_epochs, &replaced.peer_epochs));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_successor_waits_for_inflight_generation_to_drain() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let now = Instant::now();
|
||||
let original_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original_peers), now).is_none());
|
||||
|
||||
let admitted = {
|
||||
let state = slot.read().expect("proof slot should not poison");
|
||||
acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("a fresh proof should admit one journal operation")
|
||||
};
|
||||
{
|
||||
let state = slot.read().expect("proof slot should not poison");
|
||||
assert!(
|
||||
tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now),
|
||||
"a freshly admitted journal proof must remain current"
|
||||
);
|
||||
assert!(
|
||||
acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now + REMOTE_VERSION_STATE_PROOF_TTL,)
|
||||
.is_none(),
|
||||
"TTL expiry must stop new admission"
|
||||
);
|
||||
assert!(
|
||||
!tier_delete_journal_fleet_proof_matches_at(
|
||||
&state,
|
||||
&admitted,
|
||||
"topology-a",
|
||||
now + REMOTE_VERSION_STATE_PROOF_TTL,
|
||||
),
|
||||
"TTL expiry must also stop an admitted proof at its next durable fence"
|
||||
);
|
||||
assert!(!admitted._permit.generation.is_drained());
|
||||
}
|
||||
|
||||
let restarted_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
let blocked = publish_fleet_capability_probe_result(
|
||||
&slot,
|
||||
"topology-a",
|
||||
Ok(restarted_peers.clone()),
|
||||
now + Duration::from_millis(1),
|
||||
)
|
||||
.expect("a successor proof must wait for the admitted generation");
|
||||
assert!(blocked.to_string().contains("previous generation to drain"));
|
||||
{
|
||||
let state = slot.read().expect("proof slot should not poison");
|
||||
assert!(state.proof.is_none(), "new operations must remain closed while the predecessor drains");
|
||||
assert!(state.draining_generation.is_some());
|
||||
assert!(
|
||||
!tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now + Duration::from_millis(1),),
|
||||
"a restarted peer must revoke an admitted proof before its next durable fence"
|
||||
);
|
||||
}
|
||||
|
||||
drop(admitted);
|
||||
assert!(
|
||||
publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted_peers), now + Duration::from_millis(2),)
|
||||
.is_none(),
|
||||
"the successor may publish after the in-flight operation releases its permit"
|
||||
);
|
||||
let state = slot.read().expect("proof slot should not poison");
|
||||
assert!(state.proof.is_some());
|
||||
assert!(state.draining_generation.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_topology_conflict_revokes_admitted_generation() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let now = Instant::now();
|
||||
let peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("proof slot should not poison");
|
||||
acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("a fresh proof should admit one journal operation")
|
||||
};
|
||||
|
||||
mark_fleet_capability_topology_conflict(&slot);
|
||||
|
||||
let state = slot.read().expect("proof slot should not poison");
|
||||
assert!(state.topology_conflict);
|
||||
assert!(state.proof.is_none());
|
||||
assert!(state.draining_generation.is_some());
|
||||
assert!(!admitted._permit.generation.is_accepting());
|
||||
assert!(
|
||||
!tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now),
|
||||
"topology conflict must revoke an already admitted journal proof"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
|
||||
let now = Instant::now();
|
||||
let mut state = FleetCapabilityProofState {
|
||||
proof: Some(FleetCapabilityProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(BTreeMap::new()),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
}),
|
||||
proof: Some(FleetCapabilityProof::new(
|
||||
"topology-a".to_string(),
|
||||
Arc::new(BTreeMap::new()),
|
||||
now + Duration::from_secs(1),
|
||||
)),
|
||||
draining_generation: None,
|
||||
topology_conflict: false,
|
||||
};
|
||||
assert!(acquire_fleet_capability_proof_from(&state, "topology-a", now).is_some());
|
||||
@@ -3279,7 +3786,7 @@ mod tests {
|
||||
assert_eq!(commit.len(), 1);
|
||||
assert!(commit[0].err.is_some());
|
||||
|
||||
let abort = sys.abort_tier_mutation(mutation_id).await;
|
||||
let abort = sys.abort_tier_mutation(mutation_id, Bytes::from_static(b"prepare")).await;
|
||||
assert_eq!(abort.len(), 1);
|
||||
assert!(abort[0].err.is_some());
|
||||
}
|
||||
|
||||
@@ -356,7 +356,7 @@ impl ECStore {
|
||||
};
|
||||
run_guard.ensure_held("rebalance version migration")?;
|
||||
let result = migrate_entry_version(
|
||||
&RebalanceMigrationBackend::new(set.as_ref(), self.as_ref(), lock_lost_signal.clone()),
|
||||
&RebalanceMigrationBackend::new(set.as_ref(), self.clone(), lock_lost_signal.clone()),
|
||||
bucket.clone(),
|
||||
pool_index,
|
||||
version,
|
||||
@@ -1478,6 +1478,7 @@ mod tests {
|
||||
let (_temp_dirs, store, _unused_store) =
|
||||
crate::services::rebalance::test_two_pool_stores(Some(active_rebalance_meta(REBALANCE_ID))).await;
|
||||
prepare_rebalance_test_volumes(store.as_ref()).await;
|
||||
crate::services::tier::test_util::register_mock_tier(&store.tier_config_mgr(), "WARM").await;
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
let target_set = store.pools[1].get_disks_by_key(object);
|
||||
let version_id = uuid::Uuid::new_v4();
|
||||
@@ -1520,14 +1521,17 @@ mod tests {
|
||||
let entry = metacache_entry_from_source(source_set.as_ref(), bucket, object).await;
|
||||
let run_signal_fence = RebalanceRunSignalTestFence::install(REBALANCE_ID);
|
||||
let barrier = TieredMetadataCommitBarrier::install(bucket, object);
|
||||
let task = spawn_real_rebalance_entry(
|
||||
let mut task = spawn_real_rebalance_entry(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&source_set),
|
||||
entry,
|
||||
REBALANCE_ID,
|
||||
Arc::new(RebalanceBucketConfigs::default()),
|
||||
);
|
||||
barrier.wait_until_paused().await;
|
||||
tokio::select! {
|
||||
_ = barrier.wait_until_paused() => {}
|
||||
result = &mut task => panic!("rebalance exited before the tiered commit barrier: {result:?}"),
|
||||
}
|
||||
run_signal_fence.mark_lost();
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
|
||||
@@ -101,14 +101,14 @@ pub(crate) trait MigrationBackend: Send + Sync {
|
||||
|
||||
pub(crate) struct RebalanceMigrationBackend<'a> {
|
||||
source: &'a SetDisks,
|
||||
store: &'a ECStore,
|
||||
store: std::sync::Arc<ECStore>,
|
||||
lock_lost_signal: Option<std::sync::Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
|
||||
}
|
||||
|
||||
impl<'a> RebalanceMigrationBackend<'a> {
|
||||
pub(crate) fn new(
|
||||
source: &'a SetDisks,
|
||||
store: &'a ECStore,
|
||||
store: std::sync::Arc<ECStore>,
|
||||
lock_lost_signal: Option<std::sync::Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
|
||||
@@ -134,7 +134,7 @@ pub(crate) async fn test_three_pool_stores_with_isolated_node_contexts(
|
||||
test_pool_stores_with_contexts(rebalance_meta, true, 3, 2).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) async fn test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(
|
||||
rebalance_meta: Option<RebalanceMeta>,
|
||||
) -> (
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub mod test_util;
|
||||
pub mod tier;
|
||||
pub mod tier_admin;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
|
||||
use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -32,9 +32,34 @@ pub(crate) const TIER_MUTATION_INTENT_SCHEMA: &str = "rustfs-tier-mutation-inten
|
||||
pub(crate) const MAX_TIER_MUTATION_INTENT_SIZE: usize = rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE;
|
||||
pub(crate) const TIER_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/records";
|
||||
pub(crate) const TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/coordinators";
|
||||
pub(crate) const TIER_MUTATION_MUTEX_SHARDS: usize = 64;
|
||||
const TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS: usize = 3;
|
||||
pub(crate) type TierMutationDigest = [u8; 32];
|
||||
|
||||
static TIER_MUTATION_MUTEXES: LazyLock<[tokio::sync::Mutex<()>; TIER_MUTATION_MUTEX_SHARDS]> =
|
||||
LazyLock::new(|| std::array::from_fn(|_| tokio::sync::Mutex::new(())));
|
||||
|
||||
/// Serializes every local phase and recovery action for one mutation id while
|
||||
/// retaining bounded parallelism for unrelated mutations.
|
||||
pub(crate) async fn acquire_tier_mutation_mutex(mutation_id: Uuid) -> tokio::sync::MutexGuard<'static, ()> {
|
||||
TIER_MUTATION_MUTEXES[tier_mutation_mutex_shard_index(mutation_id)]
|
||||
.lock()
|
||||
.await
|
||||
}
|
||||
|
||||
fn tier_mutation_mutex_shard_index(mutation_id: Uuid) -> usize {
|
||||
let raw = mutation_id.as_u128();
|
||||
let mut mixed = (raw as u64) ^ ((raw >> 64) as u64);
|
||||
// MurmurHash3's 64-bit finalizer gives stable diffusion without allocating
|
||||
// or relying on RandomState, whose seed differs between processes.
|
||||
mixed ^= mixed >> 33;
|
||||
mixed = mixed.wrapping_mul(0xff51_afd7_ed55_8ccd);
|
||||
mixed ^= mixed >> 33;
|
||||
mixed = mixed.wrapping_mul(0xc4ce_b9fe_1a85_ec53);
|
||||
mixed ^= mixed >> 33;
|
||||
(mixed as usize) & (TIER_MUTATION_MUTEX_SHARDS - 1)
|
||||
}
|
||||
|
||||
pub(crate) type Result<T> = std::result::Result<T, TierMutationIntentError>;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
@@ -265,6 +290,30 @@ impl TierMutationIntent {
|
||||
&& self.expires_at_unix_nanos == other.expires_at_unix_nanos
|
||||
}
|
||||
|
||||
/// Reconstruct the exact Prepared record that originally produced this
|
||||
/// intent. Abort RPCs are identity-bound to that payload; serializing an
|
||||
/// Aborted terminal record would both violate the wire contract and use a
|
||||
/// different revision if a missing peer has to persist a tombstone.
|
||||
pub(crate) fn original_prepared(&self) -> Result<Self> {
|
||||
if self.state == TierMutationIntentState::Prepared {
|
||||
self.validate()?;
|
||||
return Ok(self.clone());
|
||||
}
|
||||
let mut prepared = self.clone();
|
||||
prepared.revision =
|
||||
prepared
|
||||
.revision
|
||||
.checked_sub(1)
|
||||
.filter(|revision| *revision != 0)
|
||||
.ok_or(TierMutationIntentError::Corrupt(
|
||||
"terminal intent cannot reconstruct its prepared revision",
|
||||
))?;
|
||||
prepared.state = TierMutationIntentState::Prepared;
|
||||
prepared.committed_config_etag = None;
|
||||
prepared.validate()?;
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
pub(crate) fn encode(&self) -> Result<Vec<u8>> {
|
||||
self.validate()?;
|
||||
let intent_bytes = serde_json::to_vec(self)?;
|
||||
@@ -439,6 +488,16 @@ where
|
||||
load_tier_mutation_intent_record_with_etag_at_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
|
||||
}
|
||||
|
||||
pub(crate) async fn load_tier_coordinator_mutation_intent_record_with_etag<S>(
|
||||
api: Arc<S>,
|
||||
mutation_id: Uuid,
|
||||
) -> EcstoreResult<(TierMutationIntent, String)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
load_tier_mutation_intent_record_with_etag_at_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
|
||||
}
|
||||
|
||||
async fn load_tier_mutation_intent_record_with_etag_at_prefix<S>(
|
||||
api: Arc<S>,
|
||||
prefix: &str,
|
||||
@@ -507,6 +566,7 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn delete_tier_mutation_intent_record<S>(api: Arc<S>, mutation_id: Uuid) -> EcstoreResult<()>
|
||||
where
|
||||
S: EcstoreObjectOperations,
|
||||
@@ -514,13 +574,36 @@ where
|
||||
delete_tier_mutation_intent_record_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_tier_coordinator_mutation_intent_record<S>(api: Arc<S>, mutation_id: Uuid) -> EcstoreResult<()>
|
||||
pub(crate) async fn delete_tier_mutation_intent_record_if_current<S>(
|
||||
api: Arc<S>,
|
||||
mutation_id: Uuid,
|
||||
current_etag: &str,
|
||||
) -> EcstoreResult<()>
|
||||
where
|
||||
S: EcstoreObjectOperations,
|
||||
{
|
||||
delete_tier_mutation_intent_record_with_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
|
||||
delete_tier_mutation_intent_record_if_current_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id, current_etag)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_tier_coordinator_mutation_intent_record_if_current<S>(
|
||||
api: Arc<S>,
|
||||
mutation_id: Uuid,
|
||||
current_etag: &str,
|
||||
) -> EcstoreResult<()>
|
||||
where
|
||||
S: EcstoreObjectOperations,
|
||||
{
|
||||
delete_tier_mutation_intent_record_if_current_with_prefix(
|
||||
api,
|
||||
TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX,
|
||||
mutation_id,
|
||||
current_etag,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn delete_tier_mutation_intent_record_with_prefix<S>(api: Arc<S>, prefix: &str, mutation_id: Uuid) -> EcstoreResult<()>
|
||||
where
|
||||
S: EcstoreObjectOperations,
|
||||
@@ -533,6 +616,40 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_tier_mutation_intent_record_if_current_with_prefix<S>(
|
||||
api: Arc<S>,
|
||||
prefix: &str,
|
||||
mutation_id: Uuid,
|
||||
current_etag: &str,
|
||||
) -> EcstoreResult<()>
|
||||
where
|
||||
S: EcstoreObjectOperations,
|
||||
{
|
||||
if current_etag.trim().is_empty() {
|
||||
return Err(Error::other("tier mutation intent current ETag is empty"));
|
||||
}
|
||||
let object =
|
||||
tier_mutation_intent_record_object_name_with_prefix(prefix, mutation_id).map_err(tier_mutation_intent_store_error)?;
|
||||
match api
|
||||
.delete_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
&object,
|
||||
ObjectOptions {
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(current_etag.to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) => Err(Error::ConfigNotFound),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn advance_tier_mutation_intent_record_idempotent<S>(
|
||||
api: Arc<S>,
|
||||
mutation_id: Uuid,
|
||||
@@ -713,6 +830,7 @@ fn digest_is_empty(digest: &TierMutationDigest) -> bool {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::Duration;
|
||||
|
||||
const OLD_IDENTITY: TierDestinationId = [1; 32];
|
||||
const NEW_IDENTITY: TierDestinationId = [2; 32];
|
||||
@@ -736,6 +854,61 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mutation_mutex_uses_exactly_64_stable_shards() {
|
||||
assert_eq!(TIER_MUTATION_MUTEX_SHARDS, 64);
|
||||
assert_eq!(TIER_MUTATION_MUTEXES.len(), TIER_MUTATION_MUTEX_SHARDS);
|
||||
|
||||
let mutation_id = Uuid::parse_str("36e2220e-9ad2-495b-b3bc-c4d2caf70a31").expect("fixture uuid should parse");
|
||||
let shard = tier_mutation_mutex_shard_index(mutation_id);
|
||||
assert!(shard < TIER_MUTATION_MUTEX_SHARDS);
|
||||
assert_eq!(shard, tier_mutation_mutex_shard_index(mutation_id));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mutation_mutex_serializes_the_same_id() {
|
||||
let mutation_id = Uuid::new_v4();
|
||||
let first = acquire_tier_mutation_mutex(mutation_id).await;
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (acquired_tx, mut acquired_rx) = tokio::sync::oneshot::channel();
|
||||
|
||||
let waiter = tokio::spawn(async move {
|
||||
started_tx.send(()).expect("test receiver should remain alive");
|
||||
let _second = acquire_tier_mutation_mutex(mutation_id).await;
|
||||
acquired_tx.send(()).expect("test receiver should remain alive");
|
||||
});
|
||||
started_rx.await.expect("waiter should start");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(25), &mut acquired_rx)
|
||||
.await
|
||||
.is_err(),
|
||||
"the same mutation id must not enter concurrently"
|
||||
);
|
||||
|
||||
drop(first);
|
||||
tokio::time::timeout(Duration::from_secs(1), &mut acquired_rx)
|
||||
.await
|
||||
.expect("waiter should acquire after release")
|
||||
.expect("waiter should report acquisition");
|
||||
waiter.await.expect("waiter task should finish");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mutation_mutex_allows_different_shards_to_progress() {
|
||||
let first_id = Uuid::new_v4();
|
||||
let first_shard = tier_mutation_mutex_shard_index(first_id);
|
||||
let second_id = (0..1024)
|
||||
.map(|_| Uuid::new_v4())
|
||||
.find(|candidate| tier_mutation_mutex_shard_index(*candidate) != first_shard)
|
||||
.expect("a distinct shard should be easy to find");
|
||||
let first = acquire_tier_mutation_mutex(first_id).await;
|
||||
|
||||
let _second = tokio::time::timeout(Duration::from_secs(1), acquire_tier_mutation_mutex(second_id))
|
||||
.await
|
||||
.expect("a different shard must not wait for the first mutation");
|
||||
drop(first);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intent_round_trip_preserves_committed_state() {
|
||||
let mut intent = prepared_intent();
|
||||
@@ -873,6 +1046,37 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn terminal_intent_reconstructs_original_prepared_abort_payload() {
|
||||
for terminal in [TierMutationIntentState::Aborted, TierMutationIntentState::Committed] {
|
||||
let original = prepared_intent();
|
||||
let mut intent = original.clone();
|
||||
let committed_etag = (terminal == TierMutationIntentState::Committed).then(|| "new-etag".to_string());
|
||||
intent
|
||||
.advance(terminal, committed_etag)
|
||||
.expect("terminal transition should succeed");
|
||||
|
||||
let reconstructed = intent
|
||||
.original_prepared()
|
||||
.expect("terminal record should recover prepared payload");
|
||||
assert_eq!(reconstructed, original);
|
||||
assert!(intent.same_identity_as(&reconstructed));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn terminal_intent_with_initial_revision_fails_prepared_reconstruction() {
|
||||
let mut corrupt = prepared_intent();
|
||||
corrupt.state = TierMutationIntentState::Aborted;
|
||||
|
||||
assert!(matches!(
|
||||
corrupt.original_prepared(),
|
||||
Err(TierMutationIntentError::Corrupt(
|
||||
"terminal intent cannot reconstruct its prepared revision"
|
||||
))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intent_validation_rejects_placeholder_identity() {
|
||||
let mut intent = prepared_intent();
|
||||
|
||||
@@ -15,12 +15,13 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase};
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::tier::{TierConfigMgr, tier_config_abort_matches, tier_config_commit_matches, tier_config_etag_matches};
|
||||
use super::tier_mutation_intent::{
|
||||
MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, advance_tier_mutation_intent_record_idempotent,
|
||||
load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
|
||||
MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, acquire_tier_mutation_mutex,
|
||||
advance_tier_mutation_intent_record_idempotent, load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
|
||||
};
|
||||
use crate::error::{Error, StorageError};
|
||||
use crate::store::ECStore;
|
||||
@@ -57,6 +58,8 @@ pub enum TierMutationPeerError {
|
||||
CommitProofMismatch,
|
||||
#[error("tier mutation peer abort proof does not match the persisted tier configuration")]
|
||||
AbortProofMismatch,
|
||||
#[error("tier mutation peer prepared intent has expired")]
|
||||
ExpiredIntent,
|
||||
#[error("tier mutation peer runtime error: {0}")]
|
||||
Runtime(#[source] AdminError),
|
||||
#[error("tier mutation peer store error: {0}")]
|
||||
@@ -79,6 +82,7 @@ pub async fn handle_tier_mutation_peer_request(
|
||||
canonical_payload: &[u8],
|
||||
) -> TierMutationPeerResult<TierMutationPeerOutcome> {
|
||||
validate_peer_request_envelope(protocol_version, mutation_id, canonical_payload)?;
|
||||
let _mutation_guard = acquire_tier_mutation_mutex(mutation_id).await;
|
||||
match phase {
|
||||
TierMutationRpcPhase::Prepare => handle_prepare(api, mutation_id, canonical_payload).await,
|
||||
TierMutationRpcPhase::Commit => handle_commit(api, mutation_id, canonical_payload).await,
|
||||
@@ -103,43 +107,57 @@ async fn handle_prepare(
|
||||
}
|
||||
let tier_config_mgr = api.tier_config_mgr();
|
||||
|
||||
match save_tier_mutation_intent_record_if_absent(api.clone(), &intent).await {
|
||||
Ok(()) => {
|
||||
TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &intent)
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
Ok(TierMutationPeerOutcome {
|
||||
state: TierMutationPeerState::Prepared,
|
||||
applied: true,
|
||||
})
|
||||
}
|
||||
Err(Error::PreconditionFailed) => {
|
||||
let existing = load_tier_mutation_intent_record(api, mutation_id).await?;
|
||||
if !existing.same_identity_as(&intent) {
|
||||
return Err(TierMutationPeerError::ConflictingIntent);
|
||||
for _ in 0..3 {
|
||||
let (stored, applied) = match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
|
||||
Ok(existing) => {
|
||||
if !existing.same_identity_as(&intent) {
|
||||
return Err(TierMutationPeerError::ConflictingIntent);
|
||||
}
|
||||
(existing, false)
|
||||
}
|
||||
match existing.state {
|
||||
TierMutationIntentState::Prepared => {
|
||||
TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &existing)
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
Err(Error::ConfigNotFound) => {
|
||||
let now = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).unwrap_or(i64::MAX);
|
||||
if intent.expires_at_unix_nanos <= now {
|
||||
return Err(TierMutationPeerError::ExpiredIntent);
|
||||
}
|
||||
TierMutationIntentState::Committed => {
|
||||
TierConfigMgr::apply_committed_mutation_intent_block(&tier_config_mgr, &existing)
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
}
|
||||
TierMutationIntentState::Aborted => {
|
||||
TierConfigMgr::request_committed_mutation_refresh(&tier_config_mgr).await;
|
||||
match save_tier_mutation_intent_record_if_absent(api.clone(), &intent).await {
|
||||
Ok(()) => (intent.clone(), true),
|
||||
Err(Error::PreconditionFailed) => continue,
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
Ok(TierMutationPeerOutcome {
|
||||
state: peer_state_from_intent(existing.state),
|
||||
applied: false,
|
||||
})
|
||||
Err(err) => return Err(err.into()),
|
||||
};
|
||||
|
||||
match stored.state {
|
||||
TierMutationIntentState::Prepared => {
|
||||
TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &stored)
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
TierConfigMgr::wait_for_blocked_tier_operation_leases(&tier_config_mgr, &stored)
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
}
|
||||
TierMutationIntentState::Committed => {
|
||||
TierConfigMgr::apply_committed_mutation_intent_block(&tier_config_mgr, &stored)
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
}
|
||||
TierMutationIntentState::Aborted => {
|
||||
TierConfigMgr::clear_prepared_mutation_intent_block(&tier_config_mgr, mutation_id)
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
TierConfigMgr::request_committed_mutation_refresh(&tier_config_mgr).await;
|
||||
}
|
||||
}
|
||||
Err(err) => Err(err.into()),
|
||||
return Ok(TierMutationPeerOutcome {
|
||||
state: peer_state_from_intent(stored.state),
|
||||
applied,
|
||||
});
|
||||
}
|
||||
Err(TierMutationPeerError::Store(Error::other(
|
||||
"tier mutation prepare raced repeatedly with another decision",
|
||||
)))
|
||||
}
|
||||
|
||||
async fn handle_commit(
|
||||
@@ -201,26 +219,106 @@ async fn handle_abort(
|
||||
mutation_id: Uuid,
|
||||
canonical_payload: &[u8],
|
||||
) -> TierMutationPeerResult<TierMutationPeerOutcome> {
|
||||
if !canonical_payload.is_empty() {
|
||||
return Err(TierMutationPeerError::InvalidPayload("abort payload must be empty".to_string()));
|
||||
let prepared = TierMutationIntent::decode(mutation_id, canonical_payload)
|
||||
.map_err(|err| TierMutationPeerError::InvalidPayload(err.to_string()))?;
|
||||
if prepared.state != TierMutationIntentState::Prepared {
|
||||
return Err(TierMutationPeerError::InvalidPayload(
|
||||
"abort payload must carry the original prepared intent".to_string(),
|
||||
));
|
||||
}
|
||||
let existing = load_tier_mutation_intent_record(api.clone(), mutation_id).await?;
|
||||
if existing.state == TierMutationIntentState::Prepared
|
||||
&& !tier_config_abort_matches(api.clone(), &existing)
|
||||
.await
|
||||
.map_err(Error::other)?
|
||||
{
|
||||
return Err(TierMutationPeerError::AbortProofMismatch);
|
||||
let mut tombstone = prepared.clone();
|
||||
tombstone
|
||||
.advance(TierMutationIntentState::Aborted, None)
|
||||
.map_err(|err| TierMutationPeerError::InvalidPayload(err.to_string()))?;
|
||||
|
||||
for _ in 0..3 {
|
||||
match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
|
||||
Ok(existing) => {
|
||||
if !existing.same_identity_as(&prepared) {
|
||||
return Err(TierMutationPeerError::ConflictingIntent);
|
||||
}
|
||||
match existing.state {
|
||||
TierMutationIntentState::Committed => {
|
||||
return Ok(TierMutationPeerOutcome {
|
||||
state: TierMutationPeerState::Committed,
|
||||
applied: false,
|
||||
});
|
||||
}
|
||||
TierMutationIntentState::Aborted => {
|
||||
TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id)
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
|
||||
return Ok(TierMutationPeerOutcome {
|
||||
state: TierMutationPeerState::Aborted,
|
||||
applied: false,
|
||||
});
|
||||
}
|
||||
TierMutationIntentState::Prepared => {}
|
||||
}
|
||||
if !tier_config_abort_matches(api.clone(), &prepared)
|
||||
.await
|
||||
.map_err(Error::other)?
|
||||
{
|
||||
return Err(TierMutationPeerError::AbortProofMismatch);
|
||||
}
|
||||
let advanced = advance_tier_mutation_intent_record_idempotent(
|
||||
api.clone(),
|
||||
mutation_id,
|
||||
TierMutationIntentState::Aborted,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
let (intent, applied) = match advanced {
|
||||
Ok(result) => result,
|
||||
Err(err) => match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
|
||||
Ok(current)
|
||||
if current.same_identity_as(&prepared) && current.state != TierMutationIntentState::Prepared =>
|
||||
{
|
||||
(current, false)
|
||||
}
|
||||
_ => return Err(err.into()),
|
||||
},
|
||||
};
|
||||
if intent.state == TierMutationIntentState::Aborted {
|
||||
TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
|
||||
TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id)
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
}
|
||||
return Ok(TierMutationPeerOutcome {
|
||||
state: peer_state_from_intent(intent.state),
|
||||
applied,
|
||||
});
|
||||
}
|
||||
Err(Error::ConfigNotFound) => {
|
||||
if !tier_config_abort_matches(api.clone(), &prepared)
|
||||
.await
|
||||
.map_err(Error::other)?
|
||||
{
|
||||
return Err(TierMutationPeerError::AbortProofMismatch);
|
||||
}
|
||||
match save_tier_mutation_intent_record_if_absent(api.clone(), &tombstone).await {
|
||||
Ok(()) => {
|
||||
TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id)
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
|
||||
return Ok(TierMutationPeerOutcome {
|
||||
state: TierMutationPeerState::Aborted,
|
||||
applied: true,
|
||||
});
|
||||
}
|
||||
Err(Error::PreconditionFailed) => continue,
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
let (intent, applied) =
|
||||
advance_tier_mutation_intent_record_idempotent(api.clone(), mutation_id, TierMutationIntentState::Aborted, None).await?;
|
||||
if intent.state == TierMutationIntentState::Aborted {
|
||||
TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
|
||||
}
|
||||
Ok(TierMutationPeerOutcome {
|
||||
state: peer_state_from_intent(intent.state),
|
||||
applied,
|
||||
})
|
||||
Err(TierMutationPeerError::Store(Error::other(
|
||||
"tier mutation abort raced repeatedly with prepare",
|
||||
)))
|
||||
}
|
||||
|
||||
fn validate_peer_request_envelope(
|
||||
@@ -228,7 +326,10 @@ fn validate_peer_request_envelope(
|
||||
mutation_id: Uuid,
|
||||
canonical_payload: &[u8],
|
||||
) -> TierMutationPeerResult<()> {
|
||||
if protocol_version != TIER_MUTATION_RPC_PROTOCOL_VERSION {
|
||||
if !matches!(
|
||||
protocol_version,
|
||||
rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION | TIER_MUTATION_RPC_PROTOCOL_VERSION
|
||||
) {
|
||||
return Err(TierMutationPeerError::UnsupportedProtocolVersion(protocol_version));
|
||||
}
|
||||
if mutation_id.is_nil() {
|
||||
@@ -276,6 +377,8 @@ mod tests {
|
||||
#[test]
|
||||
fn peer_request_envelope_fails_closed_on_old_version_nil_id_and_large_payload() {
|
||||
let mutation_id = Uuid::new_v4();
|
||||
validate_peer_request_envelope(rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION, mutation_id, b"payload")
|
||||
.expect("v3 must remain accepted during the v4 rollout");
|
||||
assert!(matches!(
|
||||
validate_peer_request_envelope(TIER_MUTATION_RPC_PROTOCOL_VERSION + 1, mutation_id, b"payload"),
|
||||
Err(TierMutationPeerError::UnsupportedProtocolVersion(_))
|
||||
|
||||
@@ -859,7 +859,9 @@ static OBJECT_LOCK_DIAG_ENABLED: OnceLock<bool> = OnceLock::new();
|
||||
|
||||
mod core;
|
||||
#[cfg(test)]
|
||||
pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, disk_call_counters, rename_fanout_barrier};
|
||||
pub(crate) use core::io_primitives::disk_call_counters;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
|
||||
mod ctx;
|
||||
mod metadata;
|
||||
mod ops;
|
||||
@@ -872,7 +874,7 @@ pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
|
||||
pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::DeleteObjectCommitBarrier;
|
||||
#[cfg(feature = "test-util")]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
|
||||
pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
|
||||
@@ -244,7 +244,7 @@ impl StaleMultipartCleanupGuard {
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum MultipartCommitPause {
|
||||
NewUploadBeforeLockLost,
|
||||
PutPartBeforeLockAcquire,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -29,6 +29,25 @@ use std::future::Future;
|
||||
|
||||
const DELETED_BUCKETS_PREFIX: &str = ".deleted";
|
||||
const SCANNER_BUCKET_LIST_SET_CONCURRENCY: usize = 4;
|
||||
const EVENT_BUCKET_DELETE_BLOCKED: &str = "bucket_delete_blocked";
|
||||
|
||||
fn record_bucket_delete_blocker(bucket: &str, kind: BucketDeleteBlockerKind, residue: &BucketMetadataLessResidue) {
|
||||
metrics::counter!("rustfs_bucket_delete_blockers_total", "kind" => kind.as_str()).increment(1);
|
||||
debug!(
|
||||
event = EVENT_BUCKET_DELETE_BLOCKED,
|
||||
component = "ecstore",
|
||||
subsystem = "bucket",
|
||||
bucket,
|
||||
blocker = kind.as_str(),
|
||||
files = residue.files,
|
||||
uuid_data_dirs = residue.uuid_data_dirs,
|
||||
entries_scanned = residue.entries_scanned,
|
||||
diagnostic_bytes_read = residue.diagnostic_bytes_read,
|
||||
diagnostic_truncated = residue.diagnostic_truncated,
|
||||
sample = residue.sample.as_deref().unwrap_or("<none>"),
|
||||
"Bucket deletion was blocked by durable local state"
|
||||
);
|
||||
}
|
||||
|
||||
fn scanner_bucket_list_set_concurrency(set_count: usize) -> usize {
|
||||
set_count.clamp(1, SCANNER_BUCKET_LIST_SET_CONCURRENCY)
|
||||
@@ -156,6 +175,7 @@ where
|
||||
async fn bucket_delete_local_blocker(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
budget: &mut BucketDeleteDiagnosticBudget,
|
||||
) -> Result<Option<StorageError>> {
|
||||
let local_disks = runtime_sources::local_disks_in(ctx).await;
|
||||
let mut residue = BucketMetadataLessResidue::default();
|
||||
@@ -164,18 +184,30 @@ async fn bucket_delete_local_blocker(
|
||||
let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else {
|
||||
continue;
|
||||
};
|
||||
let scan = scan_metadata_less_residue(&bucket_path?).await?;
|
||||
let scan = scan_metadata_less_residue_with_budget(&bucket_path?, budget).await?;
|
||||
if scan.xlmeta_found {
|
||||
record_bucket_delete_blocker(bucket, scan.xlmeta_blocker.unwrap_or(BucketDeleteBlockerKind::UnknownXlMeta), &scan);
|
||||
return Ok(Some(StorageError::BucketNotEmpty(bucket.to_string())));
|
||||
}
|
||||
residue.files = residue.files.saturating_add(scan.files);
|
||||
residue.uuid_data_dirs = residue.uuid_data_dirs.saturating_add(scan.uuid_data_dirs);
|
||||
residue.entries_scanned = residue.entries_scanned.saturating_add(scan.entries_scanned);
|
||||
residue.diagnostic_bytes_read = residue.diagnostic_bytes_read.saturating_add(scan.diagnostic_bytes_read);
|
||||
if residue.sample.is_none() {
|
||||
residue.sample = scan.sample;
|
||||
residue.sample = scan.sample.clone();
|
||||
}
|
||||
if scan.diagnostic_truncated {
|
||||
residue.diagnostic_truncated = true;
|
||||
record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::DiagnosticBudgetExceeded, &residue);
|
||||
return Ok(Some(StorageError::BucketNotEmptyWithDetails {
|
||||
bucket: bucket.to_string(),
|
||||
details: residue.describe(),
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
if residue.has_residue_without_xlmeta() {
|
||||
record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::OrphanDirectory, &residue);
|
||||
return Ok(Some(StorageError::BucketNotEmptyWithDetails {
|
||||
bucket: bucket.to_string(),
|
||||
details: residue.describe(),
|
||||
@@ -783,12 +815,14 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
};
|
||||
let mut diagnostic_budget = None;
|
||||
|
||||
if bucket_exists {
|
||||
validate_table_bucket_delete_guard(&self.ctx, bucket).await?;
|
||||
|
||||
if !opts.force {
|
||||
if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket).await? {
|
||||
let budget = diagnostic_budget.get_or_insert_with(BucketDeleteDiagnosticBudget::new);
|
||||
if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket, budget).await? {
|
||||
return Err(blocker);
|
||||
}
|
||||
delete_opts.force_if_empty = true;
|
||||
@@ -827,7 +861,12 @@ impl ECStore {
|
||||
{
|
||||
if delete_opts.force_if_empty
|
||||
&& matches!(&err, StorageError::BucketNotEmpty(_))
|
||||
&& let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket).await?
|
||||
&& let Some(blocker) = bucket_delete_local_blocker(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
diagnostic_budget.get_or_insert_with(BucketDeleteDiagnosticBudget::new),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
return Err(blocker);
|
||||
}
|
||||
@@ -856,15 +895,17 @@ impl ECStore {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
SCANNER_BUCKET_LIST_SET_CONCURRENCY, await_bucket_namespace_operation, bucket_delete_metadata_cleanup_prefixes,
|
||||
bucket_deleted_marker_prefix, bucket_deleted_marker_volume, run_bucket_usage_cleanup, run_physical_bucket_deletion,
|
||||
scan_metadata_less_residue, scanner_bucket_list_set_concurrency, should_override_created_from_metadata,
|
||||
BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES, BucketDeleteBlockerKind,
|
||||
BucketDeleteDiagnosticBudget, SCANNER_BUCKET_LIST_SET_CONCURRENCY, await_bucket_namespace_operation,
|
||||
bucket_delete_metadata_cleanup_prefixes, bucket_deleted_marker_prefix, bucket_deleted_marker_volume,
|
||||
run_bucket_usage_cleanup, run_physical_bucket_deletion, scan_metadata_less_residue,
|
||||
scan_metadata_less_residue_with_budget, scanner_bucket_list_set_concurrency, should_override_created_from_metadata,
|
||||
validate_table_bucket_delete_allowed,
|
||||
};
|
||||
use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
|
||||
use crate::bucket::metadata_sys;
|
||||
use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
|
||||
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE};
|
||||
use crate::error::StorageError;
|
||||
use crate::object_api::{ObjectOptions, PutObjReader};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
@@ -879,6 +920,7 @@ mod tests {
|
||||
layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
};
|
||||
use rustfs_data_usage::{BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageInfo};
|
||||
use rustfs_filemeta::{FileInfo, FileMeta, TRANSITION_COMPLETE};
|
||||
use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
|
||||
use serial_test::serial;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -892,6 +934,88 @@ mod tests {
|
||||
|
||||
static BUCKET_DELETE_TEST_ENV: OnceCell<(Vec<PathBuf>, Arc<ECStore>)> = OnceCell::const_new();
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn bucket_delete_diagnostic_budget_starts_with_first_scan_io_and_latches_once() {
|
||||
let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
|
||||
tokio::time::advance(Duration::from_secs(10)).await;
|
||||
|
||||
let first_polled = Arc::new(AtomicBool::new(false));
|
||||
let first_polled_for_io = first_polled.clone();
|
||||
let first = budget
|
||||
.run_io(async move {
|
||||
first_polled_for_io.store(true, Ordering::SeqCst);
|
||||
Ok::<_, std::io::Error>(7_u8)
|
||||
})
|
||||
.await
|
||||
.expect("the first diagnostic IO should succeed");
|
||||
assert_eq!(first, Some(7));
|
||||
assert!(first_polled.load(Ordering::SeqCst));
|
||||
|
||||
tokio::time::advance(Duration::from_millis(101)).await;
|
||||
let expired_polled = Arc::new(AtomicBool::new(false));
|
||||
let expired_polled_for_io = expired_polled.clone();
|
||||
let expired = budget
|
||||
.run_io(async move {
|
||||
expired_polled_for_io.store(true, Ordering::SeqCst);
|
||||
Ok::<_, std::io::Error>(9_u8)
|
||||
})
|
||||
.await
|
||||
.expect("an expired diagnostic budget should not become an IO error");
|
||||
assert_eq!(expired, None);
|
||||
assert!(
|
||||
!expired_polled.load(Ordering::SeqCst),
|
||||
"the deadline must remain latched after the first scan IO"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn bucket_delete_diagnostic_budget_times_out_its_first_pending_io() {
|
||||
let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
|
||||
let io_polled = Arc::new(AtomicBool::new(false));
|
||||
let io_polled_for_future = io_polled.clone();
|
||||
|
||||
let result = budget
|
||||
.run_io(std::future::poll_fn(move |_cx| {
|
||||
io_polled_for_future.store(true, Ordering::SeqCst);
|
||||
std::task::Poll::<std::io::Result<()>>::Pending
|
||||
}))
|
||||
.await
|
||||
.expect("a diagnostic timeout should fail closed without an IO error");
|
||||
|
||||
assert_eq!(result, None);
|
||||
assert!(
|
||||
io_polled.load(Ordering::SeqCst),
|
||||
"the first diagnostic IO must be polled before its timeout"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn delayed_metadata_less_scans_still_detect_orphans_and_xlmeta() {
|
||||
let root = tempfile::tempdir().expect("temporary delayed-scan roots should be created");
|
||||
let orphan_root = root.path().join("orphan-root");
|
||||
let xlmeta_root = root.path().join("xlmeta-root");
|
||||
std::fs::create_dir_all(&orphan_root).expect("orphan root should be created");
|
||||
std::fs::create_dir_all(&xlmeta_root).expect("xlmeta root should be created");
|
||||
std::fs::write(orphan_root.join("orphan-part"), b"orphan").expect("orphan fixture should be written");
|
||||
std::fs::write(xlmeta_root.join(STORAGE_FORMAT_FILE), b"invalid-xlmeta").expect("xl.meta fixture should be written");
|
||||
|
||||
let mut orphan_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
|
||||
let mut xlmeta_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
|
||||
let orphan = scan_metadata_less_residue_with_budget(&orphan_root, &mut orphan_budget)
|
||||
.await
|
||||
.expect("delayed orphan scan should complete");
|
||||
assert!(orphan.has_residue_without_xlmeta());
|
||||
assert!(!orphan.diagnostic_truncated);
|
||||
|
||||
let xlmeta = scan_metadata_less_residue_with_budget(&xlmeta_root, &mut xlmeta_budget)
|
||||
.await
|
||||
.expect("delayed xl.meta scan should complete");
|
||||
assert!(xlmeta.xlmeta_found);
|
||||
assert!(!xlmeta.diagnostic_truncated);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn bucket_namespace_operation_fails_closed_after_lease_expiry() {
|
||||
let ttl = Duration::from_millis(20);
|
||||
@@ -1313,6 +1437,175 @@ mod tests {
|
||||
assert!(sample.ends_with("/part.1"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_less_residue_scan_shares_one_entry_budget_across_roots() {
|
||||
let root = tempfile::tempdir().expect("temporary diagnostic roots should be created");
|
||||
let first_root = root.path().join("disk-a");
|
||||
let second_root = root.path().join("disk-b");
|
||||
tokio::fs::create_dir_all(&first_root)
|
||||
.await
|
||||
.expect("first diagnostic root should be created");
|
||||
tokio::fs::create_dir_all(&second_root)
|
||||
.await
|
||||
.expect("second diagnostic root should be created");
|
||||
for index in 0..3 {
|
||||
std::fs::write(first_root.join(format!("first-{index}")), b"").expect("first-root fixture should be written");
|
||||
std::fs::write(second_root.join(format!("second-{index}")), b"").expect("second-root fixture should be written");
|
||||
}
|
||||
|
||||
let mut budget = BucketDeleteDiagnosticBudget::with_limits(4, Duration::from_secs(5));
|
||||
let first = scan_metadata_less_residue_with_budget(&first_root, &mut budget)
|
||||
.await
|
||||
.expect("first root should fit the shared budget");
|
||||
assert!(!first.diagnostic_truncated);
|
||||
let second = scan_metadata_less_residue_with_budget(&second_root, &mut budget)
|
||||
.await
|
||||
.expect("second root should stop at the remaining shared budget");
|
||||
assert!(second.diagnostic_truncated);
|
||||
assert!(first.entries_scanned + second.entries_scanned <= 4);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_less_residue_scan_honors_an_expired_request_deadline() {
|
||||
let root = tempfile::tempdir().expect("temporary diagnostic root should be created");
|
||||
std::fs::write(root.path().join("orphan"), b"").expect("deadline fixture should be written");
|
||||
let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::ZERO);
|
||||
|
||||
let scan = scan_metadata_less_residue_with_budget(root.path(), &mut budget)
|
||||
.await
|
||||
.expect("an expired diagnostic budget should fail closed without an IO error");
|
||||
|
||||
assert!(scan.diagnostic_truncated);
|
||||
assert_eq!(scan.entries_scanned, 0);
|
||||
assert_eq!(scan.diagnostic_bytes_read, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_less_residue_scan_stops_at_diagnostic_budget() {
|
||||
let root = tempfile::tempdir().expect("temporary bucket root should be created");
|
||||
let bucket_path = root.path().join("bucket");
|
||||
tokio::fs::create_dir_all(&bucket_path)
|
||||
.await
|
||||
.expect("budget fixture directory should be created");
|
||||
for index in 0..(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES + 32) {
|
||||
std::fs::write(bucket_path.join(format!("orphan-{index:05}")), b"").expect("budget fixture file should be created");
|
||||
}
|
||||
|
||||
let residue = scan_metadata_less_residue(&bucket_path)
|
||||
.await
|
||||
.expect("budgeted residue scan should fail closed without an IO error");
|
||||
assert!(residue.diagnostic_truncated);
|
||||
assert!(residue.has_residue_without_xlmeta());
|
||||
assert!(!residue.xlmeta_found);
|
||||
assert!(residue.entries_scanned <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
|
||||
assert!(residue.files <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
|
||||
assert_eq!(residue.diagnostic_bytes_read, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_residue_scan_distinguishes_visible_and_tier_free_xlmeta() {
|
||||
let root = tempfile::tempdir().expect("temporary bucket root should be created");
|
||||
let bucket_path = root.path().join("bucket");
|
||||
let visible_path = bucket_path.join("visible").join(STORAGE_FORMAT_FILE);
|
||||
tokio::fs::create_dir_all(visible_path.parent().expect("visible xl.meta should have a parent"))
|
||||
.await
|
||||
.expect("visible object directory should be created");
|
||||
let mut visible = FileMeta::new();
|
||||
visible
|
||||
.add_version(FileInfo {
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
})
|
||||
.expect("visible version should encode");
|
||||
tokio::fs::write(&visible_path, visible.marshal_msg().expect("visible xl.meta should marshal"))
|
||||
.await
|
||||
.expect("visible xl.meta should be written");
|
||||
|
||||
let visible_scan = scan_metadata_less_residue(&bucket_path)
|
||||
.await
|
||||
.expect("visible xl.meta scan should succeed");
|
||||
assert_eq!(visible_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::VisibleVersion));
|
||||
|
||||
tokio::fs::remove_dir_all(bucket_path.join("visible"))
|
||||
.await
|
||||
.expect("visible fixture should be removed");
|
||||
let free_path = bucket_path.join("free").join(STORAGE_FORMAT_FILE);
|
||||
tokio::fs::create_dir_all(free_path.parent().expect("free xl.meta should have a parent"))
|
||||
.await
|
||||
.expect("free-version object directory should be created");
|
||||
let source_version_id = Uuid::new_v4();
|
||||
let mut free = FileMeta::new();
|
||||
free.add_version(FileInfo {
|
||||
version_id: Some(source_version_id),
|
||||
transition_status: TRANSITION_COMPLETE.to_string(),
|
||||
transitioned_objname: "remote/object".to_string(),
|
||||
transition_version_id: Some(Uuid::new_v4()),
|
||||
transition_tier: "WARM".to_string(),
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
})
|
||||
.expect("transitioned source should encode");
|
||||
let mut delete = FileInfo {
|
||||
version_id: Some(source_version_id),
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
};
|
||||
delete.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
free.delete_version(&delete)
|
||||
.expect("transitioned source delete should create a free-version");
|
||||
tokio::fs::write(&free_path, free.marshal_msg().expect("free-version xl.meta should marshal"))
|
||||
.await
|
||||
.expect("free-version xl.meta should be written");
|
||||
|
||||
let free_scan = scan_metadata_less_residue(&bucket_path)
|
||||
.await
|
||||
.expect("free-version xl.meta scan should succeed");
|
||||
assert_eq!(free_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::TierFreeVersion));
|
||||
|
||||
tokio::fs::remove_dir_all(bucket_path.join("free"))
|
||||
.await
|
||||
.expect("free-version fixture should be removed");
|
||||
|
||||
let exact_limit_path = bucket_path.join("exact-limit").join(STORAGE_FORMAT_FILE);
|
||||
tokio::fs::create_dir_all(exact_limit_path.parent().expect("exact-limit xl.meta should have a parent"))
|
||||
.await
|
||||
.expect("exact-limit object directory should be created");
|
||||
let exact_limit = tokio::fs::File::create(&exact_limit_path)
|
||||
.await
|
||||
.expect("exact-limit xl.meta should be created");
|
||||
exact_limit
|
||||
.set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES)
|
||||
.await
|
||||
.expect("exact-limit xl.meta should be extended without allocating its contents");
|
||||
let exact_limit_scan = scan_metadata_less_residue(&bucket_path)
|
||||
.await
|
||||
.expect("exact-limit xl.meta scan should remain fail closed");
|
||||
assert_eq!(exact_limit_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
|
||||
assert_eq!(exact_limit_scan.diagnostic_bytes_read, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
|
||||
tokio::fs::remove_dir_all(bucket_path.join("exact-limit"))
|
||||
.await
|
||||
.expect("exact-limit fixture should be removed");
|
||||
|
||||
let oversized_path = bucket_path.join("oversized").join(STORAGE_FORMAT_FILE);
|
||||
tokio::fs::create_dir_all(oversized_path.parent().expect("oversized xl.meta should have a parent"))
|
||||
.await
|
||||
.expect("oversized object directory should be created");
|
||||
let oversized = tokio::fs::File::create(&oversized_path)
|
||||
.await
|
||||
.expect("oversized xl.meta should be created");
|
||||
oversized
|
||||
.set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES + 1)
|
||||
.await
|
||||
.expect("oversized xl.meta should be extended without allocating its contents");
|
||||
let oversized_scan = scan_metadata_less_residue(&bucket_path)
|
||||
.await
|
||||
.expect("oversized xl.meta scan should remain fail closed");
|
||||
assert_eq!(oversized_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
|
||||
assert_eq!(oversized_scan.diagnostic_bytes_read, 0);
|
||||
assert!(oversized_scan.diagnostic_bytes_read <= BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_bucket_listing_unions_every_erasure_set() {
|
||||
|
||||
+5841
-325
File diff suppressed because it is too large
Load Diff
+177
-10
@@ -65,7 +65,7 @@ use http::HeaderMap;
|
||||
use lazy_static::lazy_static;
|
||||
use rand::RngExt as _;
|
||||
use rustfs_config::server_config::Config;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_filemeta::{FileInfo, FileMeta};
|
||||
use rustfs_heal_contracts::heal_channel::{HealItemType, HealOpts};
|
||||
use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
@@ -88,25 +88,105 @@ type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
pub const SCANNER_PUBLICATION_LEASE_TTL_MS: u64 = 60_000;
|
||||
pub(crate) const BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES: u64 = 1024 * 1024;
|
||||
pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES: usize = 4_096;
|
||||
pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED: Duration = Duration::from_millis(100);
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct BucketDeleteDiagnosticBudget {
|
||||
deadline: Option<tokio::time::Instant>,
|
||||
max_elapsed: Duration,
|
||||
entries_remaining: usize,
|
||||
}
|
||||
|
||||
impl BucketDeleteDiagnosticBudget {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self::with_limits(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED)
|
||||
}
|
||||
|
||||
fn with_limits(entries: usize, elapsed: Duration) -> Self {
|
||||
Self {
|
||||
deadline: None,
|
||||
max_elapsed: elapsed,
|
||||
entries_remaining: entries,
|
||||
}
|
||||
}
|
||||
|
||||
fn deadline(&mut self) -> tokio::time::Instant {
|
||||
let max_elapsed = self.max_elapsed;
|
||||
*self.deadline.get_or_insert_with(|| tokio::time::Instant::now() + max_elapsed)
|
||||
}
|
||||
|
||||
fn claim_entry(&mut self) -> bool {
|
||||
if self.entries_remaining == 0 {
|
||||
return false;
|
||||
}
|
||||
let deadline = self.deadline();
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return false;
|
||||
}
|
||||
self.entries_remaining -= 1;
|
||||
true
|
||||
}
|
||||
|
||||
async fn run_io<T, F>(&mut self, future: F) -> std::io::Result<Option<T>>
|
||||
where
|
||||
F: std::future::Future<Output = std::io::Result<T>>,
|
||||
{
|
||||
let deadline = self.deadline();
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Ok(None);
|
||||
}
|
||||
match tokio::time::timeout_at(deadline, future).await {
|
||||
Ok(result) => result.map(Some),
|
||||
Err(_) => Ok(None),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct BucketMetadataLessResidue {
|
||||
pub(crate) xlmeta_found: bool,
|
||||
pub(crate) xlmeta_blocker: Option<BucketDeleteBlockerKind>,
|
||||
pub(crate) files: usize,
|
||||
pub(crate) uuid_data_dirs: usize,
|
||||
pub(crate) entries_scanned: usize,
|
||||
pub(crate) diagnostic_bytes_read: u64,
|
||||
pub(crate) diagnostic_truncated: bool,
|
||||
pub(crate) sample: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum BucketDeleteBlockerKind {
|
||||
VisibleVersion,
|
||||
TierFreeVersion,
|
||||
UnknownXlMeta,
|
||||
OrphanDirectory,
|
||||
DiagnosticBudgetExceeded,
|
||||
}
|
||||
|
||||
impl BucketDeleteBlockerKind {
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::VisibleVersion => "visible_version",
|
||||
Self::TierFreeVersion => "tier_free_version",
|
||||
Self::UnknownXlMeta => "unknown_xlmeta",
|
||||
Self::OrphanDirectory => "orphan_directory",
|
||||
Self::DiagnosticBudgetExceeded => "diagnostic_budget_exceeded",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketMetadataLessResidue {
|
||||
pub(crate) fn has_residue_without_xlmeta(&self) -> bool {
|
||||
!self.xlmeta_found && self.files > 0
|
||||
!self.xlmeta_found && (self.files > 0 || self.diagnostic_truncated)
|
||||
}
|
||||
|
||||
pub(crate) fn describe(&self) -> String {
|
||||
let sample = self.sample.as_deref().unwrap_or("<none>");
|
||||
format!(
|
||||
"metadata-less on-disk residue remains after empty-bucket verification: files={}, uuid_data_dirs={}, sample={sample}",
|
||||
self.files, self.uuid_data_dirs
|
||||
"metadata-less on-disk residue remains after empty-bucket verification: files={}, uuid_data_dirs={}, entries_scanned={}, diagnostic_bytes_read={}, diagnostic_truncated={}, sample={sample}",
|
||||
self.files, self.uuid_data_dirs, self.entries_scanned, self.diagnostic_bytes_read, self.diagnostic_truncated,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -145,28 +225,112 @@ pub(crate) async fn has_xlmeta_files(path: &std::path::Path) -> std::io::Result<
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn scan_metadata_less_residue(path: &std::path::Path) -> std::io::Result<BucketMetadataLessResidue> {
|
||||
let mut budget = BucketDeleteDiagnosticBudget::new();
|
||||
scan_metadata_less_residue_with_budget(path, &mut budget).await
|
||||
}
|
||||
|
||||
async fn scan_metadata_less_residue_with_budget(
|
||||
path: &std::path::Path,
|
||||
budget: &mut BucketDeleteDiagnosticBudget,
|
||||
) -> std::io::Result<BucketMetadataLessResidue> {
|
||||
use crate::disk::STORAGE_FORMAT_FILE;
|
||||
use tokio::fs;
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let mut scan = BucketMetadataLessResidue::default();
|
||||
let mut stack = vec![path.to_path_buf()];
|
||||
|
||||
let mark_budget_exhausted = |scan: &mut BucketMetadataLessResidue| {
|
||||
scan.diagnostic_truncated = true;
|
||||
scan.sample.get_or_insert_with(|| "<diagnostic-budget-exceeded>".to_string());
|
||||
};
|
||||
|
||||
while let Some(current_path) = stack.pop() {
|
||||
let mut entries = match fs::read_dir(¤t_path).await {
|
||||
Ok(entries) => entries,
|
||||
let mut entries = match budget.run_io(fs::read_dir(¤t_path)).await {
|
||||
Ok(Some(entries)) => entries,
|
||||
Ok(None) => {
|
||||
mark_budget_exhausted(&mut scan);
|
||||
return Ok(scan);
|
||||
}
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let file_type = entry.file_type().await?;
|
||||
loop {
|
||||
let entry = match budget.run_io(entries.next_entry()).await? {
|
||||
Some(Some(entry)) => entry,
|
||||
Some(None) => break,
|
||||
None => {
|
||||
mark_budget_exhausted(&mut scan);
|
||||
return Ok(scan);
|
||||
}
|
||||
};
|
||||
if !budget.claim_entry() {
|
||||
mark_budget_exhausted(&mut scan);
|
||||
return Ok(scan);
|
||||
}
|
||||
scan.entries_scanned = scan.entries_scanned.saturating_add(1);
|
||||
let Some(file_type) = budget.run_io(entry.file_type()).await? else {
|
||||
mark_budget_exhausted(&mut scan);
|
||||
return Ok(scan);
|
||||
};
|
||||
let file_name = entry.file_name();
|
||||
let file_name_str = file_name.to_string_lossy();
|
||||
|
||||
if file_name_str == STORAGE_FORMAT_FILE {
|
||||
scan.xlmeta_found = true;
|
||||
continue;
|
||||
if scan.xlmeta_blocker.is_none() {
|
||||
let entry_path = entry.path();
|
||||
let Some(metadata) = budget.run_io(fs::metadata(&entry_path)).await? else {
|
||||
mark_budget_exhausted(&mut scan);
|
||||
scan.xlmeta_blocker = Some(BucketDeleteBlockerKind::DiagnosticBudgetExceeded);
|
||||
return Ok(scan);
|
||||
};
|
||||
scan.xlmeta_blocker = Some(if metadata.len() > BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES {
|
||||
BucketDeleteBlockerKind::UnknownXlMeta
|
||||
} else {
|
||||
match budget.run_io(fs::File::open(&entry_path)).await {
|
||||
Ok(Some(file)) => {
|
||||
let mut data = Vec::new();
|
||||
let read = budget
|
||||
.run_io(file.take(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES).read_to_end(&mut data))
|
||||
.await;
|
||||
scan.diagnostic_bytes_read = data.len() as u64;
|
||||
match read {
|
||||
Ok(Some(_)) => match FileMeta::load(&data) {
|
||||
Ok(meta)
|
||||
if !meta.versions.is_empty()
|
||||
&& meta.versions.iter().all(|version| version.header.free_version()) =>
|
||||
{
|
||||
BucketDeleteBlockerKind::TierFreeVersion
|
||||
}
|
||||
Ok(meta) if !meta.versions.is_empty() => BucketDeleteBlockerKind::VisibleVersion,
|
||||
Ok(_) | Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
|
||||
},
|
||||
Ok(None) => {
|
||||
mark_budget_exhausted(&mut scan);
|
||||
BucketDeleteBlockerKind::DiagnosticBudgetExceeded
|
||||
}
|
||||
Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
mark_budget_exhausted(&mut scan);
|
||||
BucketDeleteBlockerKind::DiagnosticBudgetExceeded
|
||||
}
|
||||
Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
|
||||
}
|
||||
});
|
||||
let sample = entry_path
|
||||
.strip_prefix(path)
|
||||
.unwrap_or(entry_path.as_path())
|
||||
.to_string_lossy()
|
||||
.replace(std::path::MAIN_SEPARATOR, "/");
|
||||
scan.sample = Some(sample);
|
||||
}
|
||||
return Ok(scan);
|
||||
}
|
||||
|
||||
if file_type.is_dir() {
|
||||
@@ -222,7 +386,10 @@ pub(crate) mod init_format;
|
||||
pub(crate) mod list_objects;
|
||||
mod multipart;
|
||||
mod object;
|
||||
pub(crate) use object::{ObjectLockDiagGuard, SourceCleanupMutationFence, tiered_data_movement_source_matches};
|
||||
pub(crate) use object::{
|
||||
DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence,
|
||||
SourceCleanupMutationFence, tiered_data_movement_source_matches,
|
||||
};
|
||||
pub use object::{
|
||||
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
|
||||
SnapshotConsistencyError,
|
||||
|
||||
@@ -846,6 +846,7 @@ impl ECStore {
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) async fn complete_multipart_upload_for_data_movement(
|
||||
self: Arc<Self>,
|
||||
target: (usize, Option<&ObjectLockDiagGuard>),
|
||||
@@ -854,6 +855,44 @@ impl ECStore {
|
||||
upload_id: &str,
|
||||
uploaded_parts: Vec<CompletePart>,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
self.complete_multipart_upload_for_data_movement_inner(target, bucket, object, upload_id, uploaded_parts, opts, None)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn complete_multipart_upload_for_data_movement_with_publication_fence(
|
||||
self: Arc<Self>,
|
||||
target_pool_idx: usize,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
upload_id: &str,
|
||||
uploaded_parts: Vec<CompletePart>,
|
||||
opts: &ObjectOptions,
|
||||
publication_fence: RemoteTuplePublicationFence,
|
||||
) -> Result<ObjectInfo> {
|
||||
self.complete_multipart_upload_for_data_movement_inner(
|
||||
(target_pool_idx, None),
|
||||
bucket,
|
||||
object,
|
||||
upload_id,
|
||||
uploaded_parts,
|
||||
opts,
|
||||
Some(publication_fence),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn complete_multipart_upload_for_data_movement_inner(
|
||||
self: Arc<Self>,
|
||||
target: (usize, Option<&ObjectLockDiagGuard>),
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
upload_id: &str,
|
||||
uploaded_parts: Vec<CompletePart>,
|
||||
opts: &ObjectOptions,
|
||||
publication_fence: Option<RemoteTuplePublicationFence>,
|
||||
) -> Result<ObjectInfo> {
|
||||
let (target_pool_idx, mutation_fence) = target;
|
||||
check_complete_multipart_args(bucket, object, upload_id)?;
|
||||
@@ -885,8 +924,36 @@ impl ECStore {
|
||||
snapshot.add_lock_fences(&mut opts);
|
||||
opts.object_lock_config_snapshot = Some(snapshot);
|
||||
}
|
||||
self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence)
|
||||
let fixed_read_anchor = publication_fence
|
||||
.as_ref()
|
||||
.and_then(RemoteTuplePublicationFence::fixed_read_anchor_guard);
|
||||
self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence.or(fixed_read_anchor))
|
||||
.await;
|
||||
// NewMultipart/UploadPart are staging only. Acquire and consume the
|
||||
// non-cloneable publication capability immediately before Complete,
|
||||
// then retain its guards until Complete has drained the commit path.
|
||||
let publication_object = encode_dir_object(object);
|
||||
let publication_guard = match publication_fence {
|
||||
Some(publication_fence) => {
|
||||
let guard = publication_fence
|
||||
.into_commit_guard(target_pool_idx, bucket, &publication_object)
|
||||
.await?;
|
||||
guard.add_namespace_lock_fence(&mut opts);
|
||||
opts.no_lock = true;
|
||||
Some(guard)
|
||||
}
|
||||
None => {
|
||||
if rustfs_utils::http::metadata_compat::contains_key_str(
|
||||
&opts.user_defined,
|
||||
rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
|
||||
) {
|
||||
return Err(Error::other(
|
||||
"data movement multipart completion cannot publish transition ownership without a publication capability",
|
||||
));
|
||||
}
|
||||
None
|
||||
}
|
||||
};
|
||||
#[cfg(test)]
|
||||
pause_data_movement_multipart_before_selected_completion(bucket).await;
|
||||
let pool = self
|
||||
@@ -911,6 +978,7 @@ impl ECStore {
|
||||
},
|
||||
)
|
||||
.await;
|
||||
drop(publication_guard);
|
||||
let result = enqueue_transition_after_write(result, LcEventSrc::S3CompleteMultipartUpload).await;
|
||||
if result.is_ok() {
|
||||
list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await;
|
||||
|
||||
+1651
-294
File diff suppressed because it is too large
Load Diff
@@ -309,6 +309,17 @@ impl ECStore {
|
||||
}
|
||||
|
||||
pub(super) async fn delete_prefix(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
|
||||
let dispatch_scope = if opts.tier_delete_journal_api.is_some() {
|
||||
let incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
|
||||
let authorization = opts
|
||||
.tier_delete_dispatch_authorization
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("prefix mutation is missing its dispatched-journal authorization"))?;
|
||||
authorization.ensure_current(bucket, incarnation, object)?;
|
||||
Some((authorization, incarnation))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if opts.lifecycle_delete_all.is_some() {
|
||||
let mut preflight_opts = opts.clone();
|
||||
preflight_opts
|
||||
@@ -317,6 +328,9 @@ impl ECStore {
|
||||
.ok_or(StorageError::PreconditionFailed)?
|
||||
.phase = crate::object_api::LifecycleDeleteAllPhase::Preflight;
|
||||
for pool in &self.pools {
|
||||
if let Some((authorization, incarnation)) = dispatch_scope {
|
||||
authorization.ensure_current(bucket, incarnation, object)?;
|
||||
}
|
||||
#[cfg(test)]
|
||||
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Preflight, pool.pool_idx)?;
|
||||
pool.delete_object(bucket, object, preflight_opts.clone()).await?;
|
||||
@@ -326,6 +340,9 @@ impl ECStore {
|
||||
.ok_or(StorageError::PreconditionFailed)?
|
||||
.lock()
|
||||
.mark_mutation_started();
|
||||
if let Some((authorization, incarnation)) = dispatch_scope {
|
||||
authorization.mark_mutation_started(bucket, incarnation, object)?;
|
||||
}
|
||||
let mut non_trigger_opts = opts.clone();
|
||||
non_trigger_opts
|
||||
.lifecycle_delete_all
|
||||
@@ -333,6 +350,9 @@ impl ECStore {
|
||||
.ok_or(StorageError::PreconditionFailed)?
|
||||
.phase = crate::object_api::LifecycleDeleteAllPhase::History;
|
||||
for pool in &self.pools {
|
||||
if let Some((authorization, incarnation)) = dispatch_scope {
|
||||
authorization.ensure_current(bucket, incarnation, object)?;
|
||||
}
|
||||
#[cfg(test)]
|
||||
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::History, pool.pool_idx)?;
|
||||
let mut pool_opts = non_trigger_opts.clone();
|
||||
@@ -348,6 +368,9 @@ impl ECStore {
|
||||
.phase = crate::object_api::LifecycleDeleteAllPhase::FinalPreflight;
|
||||
let mut trigger_pools = Vec::new();
|
||||
for (pool_index, pool) in self.pools.iter().enumerate() {
|
||||
if let Some((authorization, incarnation)) = dispatch_scope {
|
||||
authorization.ensure_current(bucket, incarnation, object)?;
|
||||
}
|
||||
#[cfg(test)]
|
||||
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::FinalPreflight, pool.pool_idx)?;
|
||||
let result = pool.delete_object(bucket, object, final_preflight_opts.clone()).await?;
|
||||
@@ -366,6 +389,9 @@ impl ECStore {
|
||||
.ok_or(StorageError::PreconditionFailed)?
|
||||
.phase = crate::object_api::LifecycleDeleteAllPhase::Trigger;
|
||||
for pool_index in trigger_pools {
|
||||
if let Some((authorization, incarnation)) = dispatch_scope {
|
||||
authorization.ensure_current(bucket, incarnation, object)?;
|
||||
}
|
||||
#[cfg(test)]
|
||||
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Trigger, pool_index)?;
|
||||
let mut pool_opts = trigger_opts.clone();
|
||||
@@ -378,7 +404,13 @@ impl ECStore {
|
||||
let mut first_error = None;
|
||||
let mut first_volume_error = None;
|
||||
let mut has_success = false;
|
||||
if let Some((authorization, incarnation)) = dispatch_scope {
|
||||
authorization.mark_mutation_started(bucket, incarnation, object)?;
|
||||
}
|
||||
for pool in &self.pools {
|
||||
if let Some((authorization, incarnation)) = dispatch_scope {
|
||||
authorization.ensure_current(bucket, incarnation, object)?;
|
||||
}
|
||||
let mut opts = opts.clone();
|
||||
opts.delete_prefix = true;
|
||||
match pool.delete_object(bucket, object, opts).await {
|
||||
|
||||
@@ -697,6 +697,11 @@ impl HealChannelProcessor {
|
||||
| HealRequestSource::Mrf => true,
|
||||
});
|
||||
|
||||
// `no_lock` is an internal coordination hint. An admin request can
|
||||
// carry the legacy field over the wire, but cannot use it as ambient
|
||||
// authority to bypass storage namespace locking.
|
||||
let no_lock = request.no_lock.unwrap_or(false) && request.source != HealRequestSource::Admin;
|
||||
|
||||
// Build HealOptions with all available fields
|
||||
let options = HealOptions {
|
||||
scan_mode: request.scan_mode.unwrap_or(HealScanMode::Normal),
|
||||
@@ -705,7 +710,7 @@ impl HealChannelProcessor {
|
||||
update_parity: request.update_parity.unwrap_or(true),
|
||||
recursive,
|
||||
dry_run: request.dry_run.unwrap_or(false),
|
||||
no_lock: request.no_lock.unwrap_or(false),
|
||||
no_lock,
|
||||
timeout: request.timeout_seconds.map(std::time::Duration::from_secs),
|
||||
pool_index: request.pool_index,
|
||||
set_index: request.set_index,
|
||||
@@ -990,6 +995,23 @@ mod tests {
|
||||
assert!(heal_request.options.no_lock);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_convert_to_heal_request_admin_cannot_bypass_object_lock() {
|
||||
let heal_manager = create_test_heal_manager();
|
||||
let processor = HealChannelProcessor::new(heal_manager);
|
||||
let channel_request = HealChannelRequest {
|
||||
id: "admin-no-lock".to_string(),
|
||||
bucket: "test-bucket".to_string(),
|
||||
object_prefix: Some("test-object".to_string()),
|
||||
no_lock: Some(true),
|
||||
source: HealRequestSource::Admin,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let heal_request = processor.convert_to_heal_request(channel_request).unwrap();
|
||||
assert!(!heal_request.options.no_lock, "admin nolock must not become storage lock authority");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_convert_to_heal_request_scanner_defaults_recreate_missing_false() {
|
||||
let heal_manager = create_test_heal_manager();
|
||||
|
||||
@@ -1521,6 +1521,8 @@ pub struct TierMutationControlResponse {
|
||||
pub error_info: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(bytes = "bytes", tag = "5")]
|
||||
pub response_proof: ::prost::bytes::Bytes,
|
||||
#[prost(enumeration = "TierMutationFailureClass", tag = "6")]
|
||||
pub failure_class: i32,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct GetLiveEventsRequest {
|
||||
@@ -1611,6 +1613,35 @@ impl TierMutationPeerState {
|
||||
}
|
||||
}
|
||||
}
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
|
||||
#[repr(i32)]
|
||||
pub enum TierMutationFailureClass {
|
||||
Unspecified = 0,
|
||||
PreDispatchRejected = 1,
|
||||
Ambiguous = 2,
|
||||
}
|
||||
impl TierMutationFailureClass {
|
||||
/// String value of the enum field names used in the ProtoBuf definition.
|
||||
///
|
||||
/// The values are not transformed in any way and thus are considered stable
|
||||
/// (if the ProtoBuf definition does not change) and safe for programmatic use.
|
||||
pub fn as_str_name(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Unspecified => "TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED",
|
||||
Self::PreDispatchRejected => "TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED",
|
||||
Self::Ambiguous => "TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS",
|
||||
}
|
||||
}
|
||||
/// Creates an enum from field names used in the ProtoBuf definition.
|
||||
pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
|
||||
match value {
|
||||
"TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED" => Some(Self::Unspecified),
|
||||
"TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED" => Some(Self::PreDispatchRejected),
|
||||
"TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS" => Some(Self::Ambiguous),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Generated client implementations.
|
||||
pub mod node_service_client {
|
||||
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
|
||||
|
||||
+146
-55
@@ -177,6 +177,7 @@ pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-re
|
||||
pub const CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-cross-pool-fence-capability-v1\0";
|
||||
pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024;
|
||||
pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024;
|
||||
pub const TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE;
|
||||
pub const TIER_MUTATION_RPC_MAX_MESSAGE_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE + 4096;
|
||||
|
||||
pub fn heal_control_coordinator_epoch(topology_fingerprint: &str) -> Result<u64, &'static str> {
|
||||
@@ -342,7 +343,16 @@ impl TierMutationRpcPhase {
|
||||
}
|
||||
}
|
||||
|
||||
pub const TIER_MUTATION_RPC_PROTOCOL_VERSION: u32 = 1;
|
||||
// Version 2 required peer Prepare to block new tier-reference creators and
|
||||
// drain their in-flight operation leases. Version 3 additionally binds Abort
|
||||
// to the canonical Prepare intent so a missing-record Abort can persist an
|
||||
// identity-bound tombstone and linearize against a delayed Prepare. Version 4
|
||||
// signs a typed failure classification while retaining the exact v3 proof
|
||||
// bytes for rolling compatibility.
|
||||
pub const TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION: u32 = 3;
|
||||
pub const TIER_MUTATION_RPC_PROTOCOL_VERSION: u32 = 4;
|
||||
pub const TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE: usize = 1024;
|
||||
pub const TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE: usize = 4096;
|
||||
|
||||
pub fn canonical_tier_mutation_rpc_body(
|
||||
version: u32,
|
||||
@@ -374,19 +384,23 @@ pub struct TierMutationRpcResponseProofInput<'a> {
|
||||
pub state: i32,
|
||||
pub applied: bool,
|
||||
pub error_info: Option<&'a str>,
|
||||
pub failure_class: i32,
|
||||
}
|
||||
|
||||
pub fn canonical_tier_mutation_rpc_response_body(
|
||||
input: TierMutationRpcResponseProofInput<'_>,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
const DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v1\0";
|
||||
const V3_DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v1\0";
|
||||
const V4_DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v2\0";
|
||||
|
||||
let phase = input.phase.as_wire_str().as_bytes();
|
||||
let mutation_id = input.mutation_id.as_bytes();
|
||||
let error_info = input.error_info.map(str::as_bytes);
|
||||
let error_info_len = error_info.map_or(0, <[u8]>::len);
|
||||
let is_v4 = input.version >= TIER_MUTATION_RPC_PROTOCOL_VERSION;
|
||||
let domain = if is_v4 { V4_DOMAIN } else { V3_DOMAIN };
|
||||
let mut body = Vec::with_capacity(
|
||||
DOMAIN.len()
|
||||
domain.len()
|
||||
+ 4
|
||||
+ 8
|
||||
+ phase.len()
|
||||
@@ -398,9 +412,10 @@ pub fn canonical_tier_mutation_rpc_response_body(
|
||||
+ 1
|
||||
+ 1
|
||||
+ 8
|
||||
+ error_info_len,
|
||||
+ error_info_len
|
||||
+ if is_v4 { 4 } else { 0 },
|
||||
);
|
||||
body.extend_from_slice(DOMAIN);
|
||||
body.extend_from_slice(domain);
|
||||
body.extend_from_slice(&input.version.to_be_bytes());
|
||||
body.extend_from_slice(&u64::try_from(phase.len())?.to_be_bytes());
|
||||
body.extend_from_slice(phase);
|
||||
@@ -415,6 +430,9 @@ pub fn canonical_tier_mutation_rpc_response_body(
|
||||
if let Some(error_info) = error_info {
|
||||
body.extend_from_slice(error_info);
|
||||
}
|
||||
if is_v4 {
|
||||
body.extend_from_slice(&input.failure_class.to_be_bytes());
|
||||
}
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
@@ -2137,10 +2155,10 @@ mod heal_control_tests {
|
||||
#[cfg(test)]
|
||||
mod tier_mutation_rpc_tests {
|
||||
use super::{
|
||||
TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase, TierMutationRpcResponseProofInput,
|
||||
canonical_tier_mutation_rpc_body, canonical_tier_mutation_rpc_response_body,
|
||||
TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION, TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase,
|
||||
TierMutationRpcResponseProofInput, canonical_tier_mutation_rpc_body, canonical_tier_mutation_rpc_response_body,
|
||||
};
|
||||
use crate::proto_gen::node_service::TierMutationPeerState;
|
||||
use crate::proto_gen::node_service::{TierMutationFailureClass, TierMutationPeerState};
|
||||
use uuid::uuid;
|
||||
|
||||
#[test]
|
||||
@@ -2155,7 +2173,7 @@ mod tier_mutation_rpc_tests {
|
||||
)
|
||||
.expect("small mutation body should encode");
|
||||
let mut golden = b"rustfs-tier-mutation-rpc-v1\0".to_vec();
|
||||
golden.extend_from_slice(&1_u32.to_be_bytes());
|
||||
golden.extend_from_slice(&TIER_MUTATION_RPC_PROTOCOL_VERSION.to_be_bytes());
|
||||
golden.extend_from_slice(&7_u64.to_be_bytes());
|
||||
golden.extend_from_slice(b"prepare");
|
||||
golden.extend_from_slice(mutation_id.as_bytes());
|
||||
@@ -2165,8 +2183,13 @@ mod tier_mutation_rpc_tests {
|
||||
|
||||
assert_ne!(
|
||||
baseline,
|
||||
canonical_tier_mutation_rpc_body(2, TierMutationRpcPhase::Prepare, mutation_id, payload)
|
||||
.expect("small mutation body should encode")
|
||||
canonical_tier_mutation_rpc_body(
|
||||
TIER_MUTATION_RPC_PROTOCOL_VERSION + 1,
|
||||
TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
payload,
|
||||
)
|
||||
.expect("small mutation body should encode")
|
||||
);
|
||||
assert_ne!(
|
||||
baseline,
|
||||
@@ -2201,11 +2224,27 @@ mod tier_mutation_rpc_tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_tier_mutation_response_binds_request_state_and_error() {
|
||||
fn tier_mutation_v3_request_and_response_golden_bytes_are_unchanged() {
|
||||
let mutation_id = uuid!("12345678-1234-5678-9abc-def012345678");
|
||||
let payload = b"canonical-intent-record";
|
||||
let baseline = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
let request = canonical_tier_mutation_rpc_body(
|
||||
TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
|
||||
TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
payload,
|
||||
)
|
||||
.expect("v3 request should encode");
|
||||
let mut request_golden = b"rustfs-tier-mutation-rpc-v1\0".to_vec();
|
||||
request_golden.extend_from_slice(&TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION.to_be_bytes());
|
||||
request_golden.extend_from_slice(&7_u64.to_be_bytes());
|
||||
request_golden.extend_from_slice(b"prepare");
|
||||
request_golden.extend_from_slice(mutation_id.as_bytes());
|
||||
request_golden.extend_from_slice(&u64::try_from(payload.len()).expect("payload length should fit").to_be_bytes());
|
||||
request_golden.extend_from_slice(payload);
|
||||
assert_eq!(request, request_golden);
|
||||
|
||||
let response = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
@@ -2213,49 +2252,97 @@ mod tier_mutation_rpc_tests {
|
||||
state: TierMutationPeerState::Prepared as i32,
|
||||
applied: true,
|
||||
error_info: None,
|
||||
// v3 must ignore the field so its authenticated bytes stay exact.
|
||||
failure_class: TierMutationFailureClass::PreDispatchRejected as i32,
|
||||
})
|
||||
.expect("small mutation response should encode");
|
||||
.expect("v3 response should encode");
|
||||
let mut response_golden = b"rustfs-tier-mutation-rpc-response-v1\0".to_vec();
|
||||
response_golden.extend_from_slice(&TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION.to_be_bytes());
|
||||
response_golden.extend_from_slice(&7_u64.to_be_bytes());
|
||||
response_golden.extend_from_slice(b"prepare");
|
||||
response_golden.extend_from_slice(mutation_id.as_bytes());
|
||||
response_golden.extend_from_slice(&u64::try_from(payload.len()).expect("payload length should fit").to_be_bytes());
|
||||
response_golden.extend_from_slice(payload);
|
||||
response_golden.push(1);
|
||||
response_golden.extend_from_slice(&(TierMutationPeerState::Prepared as i32).to_be_bytes());
|
||||
response_golden.push(1);
|
||||
response_golden.push(0);
|
||||
response_golden.extend_from_slice(&0_u64.to_be_bytes());
|
||||
assert_eq!(response, response_golden);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_tier_mutation_v4_response_binds_request_result_and_failure_class() {
|
||||
let mutation_id = uuid!("12345678-1234-5678-9abc-def012345678");
|
||||
let payload = b"canonical-intent-record";
|
||||
let baseline = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
success: false,
|
||||
state: TierMutationPeerState::Unspecified as i32,
|
||||
applied: false,
|
||||
error_info: Some("failure"),
|
||||
failure_class: TierMutationFailureClass::Ambiguous as i32,
|
||||
})
|
||||
.expect("small v4 mutation response should encode");
|
||||
|
||||
let cases = [
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: 2,
|
||||
version: TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
success: true,
|
||||
state: TierMutationPeerState::Prepared as i32,
|
||||
applied: true,
|
||||
error_info: None,
|
||||
success: false,
|
||||
state: TierMutationPeerState::Unspecified as i32,
|
||||
applied: false,
|
||||
error_info: Some("failure"),
|
||||
failure_class: TierMutationFailureClass::Ambiguous as i32,
|
||||
},
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Commit,
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
success: true,
|
||||
state: TierMutationPeerState::Prepared as i32,
|
||||
applied: true,
|
||||
error_info: None,
|
||||
success: false,
|
||||
state: TierMutationPeerState::Unspecified as i32,
|
||||
applied: false,
|
||||
error_info: Some("failure"),
|
||||
failure_class: TierMutationFailureClass::Ambiguous as i32,
|
||||
},
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id: uuid!("22345678-1234-5678-9abc-def012345678"),
|
||||
canonical_payload: payload,
|
||||
success: true,
|
||||
state: TierMutationPeerState::Prepared as i32,
|
||||
applied: true,
|
||||
error_info: None,
|
||||
success: false,
|
||||
state: TierMutationPeerState::Unspecified as i32,
|
||||
applied: false,
|
||||
error_info: Some("failure"),
|
||||
failure_class: TierMutationFailureClass::Ambiguous as i32,
|
||||
},
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: b"tampered-intent-record",
|
||||
success: false,
|
||||
state: TierMutationPeerState::Unspecified as i32,
|
||||
applied: false,
|
||||
error_info: Some("failure"),
|
||||
failure_class: TierMutationFailureClass::Ambiguous as i32,
|
||||
},
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
success: true,
|
||||
state: TierMutationPeerState::Prepared as i32,
|
||||
applied: true,
|
||||
error_info: None,
|
||||
state: TierMutationPeerState::Unspecified as i32,
|
||||
applied: false,
|
||||
error_info: Some("failure"),
|
||||
failure_class: TierMutationFailureClass::Ambiguous as i32,
|
||||
},
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
@@ -2263,39 +2350,43 @@ mod tier_mutation_rpc_tests {
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
success: false,
|
||||
state: TierMutationPeerState::Prepared as i32,
|
||||
applied: true,
|
||||
error_info: None,
|
||||
},
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
success: true,
|
||||
state: TierMutationPeerState::Committed as i32,
|
||||
applied: true,
|
||||
error_info: None,
|
||||
},
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
success: true,
|
||||
state: TierMutationPeerState::Prepared as i32,
|
||||
applied: false,
|
||||
error_info: None,
|
||||
error_info: Some("failure"),
|
||||
failure_class: TierMutationFailureClass::Ambiguous as i32,
|
||||
},
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
success: true,
|
||||
state: TierMutationPeerState::Prepared as i32,
|
||||
success: false,
|
||||
state: TierMutationPeerState::Unspecified as i32,
|
||||
applied: true,
|
||||
error_info: Some("error"),
|
||||
error_info: Some("failure"),
|
||||
failure_class: TierMutationFailureClass::Ambiguous as i32,
|
||||
},
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
success: false,
|
||||
state: TierMutationPeerState::Unspecified as i32,
|
||||
applied: false,
|
||||
error_info: Some("other failure"),
|
||||
failure_class: TierMutationFailureClass::Ambiguous as i32,
|
||||
},
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
success: false,
|
||||
state: TierMutationPeerState::Unspecified as i32,
|
||||
applied: false,
|
||||
error_info: Some("failure"),
|
||||
failure_class: TierMutationFailureClass::PreDispatchRejected as i32,
|
||||
},
|
||||
];
|
||||
for case in cases {
|
||||
|
||||
@@ -1068,12 +1068,19 @@ enum TierMutationPeerState {
|
||||
TIER_MUTATION_PEER_STATE_ABORTED = 3;
|
||||
}
|
||||
|
||||
enum TierMutationFailureClass {
|
||||
TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED = 0;
|
||||
TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED = 1;
|
||||
TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS = 2;
|
||||
}
|
||||
|
||||
message TierMutationControlResponse {
|
||||
bool success = 1;
|
||||
TierMutationPeerState state = 2;
|
||||
bool applied = 3;
|
||||
optional string error_info = 4;
|
||||
bytes response_proof = 5;
|
||||
TierMutationFailureClass failure_class = 6;
|
||||
}
|
||||
|
||||
message GetLiveEventsRequest {
|
||||
|
||||
@@ -46,7 +46,7 @@ use storage_api::lifecycle::{
|
||||
STORAGE_FORMAT_FILE, TRANSITION_PENDING, TransitionCleanupStoreBarrier, TransitionOptions, assert_transition_meta_consistent,
|
||||
enqueue_transition_for_existing_objects, expire_transitioned_object, free_version_count, get_bucket_metadata,
|
||||
get_global_tier_config_mgr, init_background_expiry, init_bucket_metadata_sys, init_local_disks, is_err_object_not_found,
|
||||
is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_tier_delete_journal_entries,
|
||||
is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_transition_transaction_records,
|
||||
register_mock_tier_util, update_bucket_metadata, wait_for_free_version_absence,
|
||||
};
|
||||
|
||||
@@ -562,6 +562,29 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
// Deep transition futures can overflow libtest's default stack before their
|
||||
// first assertion, so the serial ILM cases use one dedicated test thread.
|
||||
fn run_large_stack_async_test<F, Fut>(thread_name: &'static str, test_fn: F)
|
||||
where
|
||||
F: FnOnce() -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = ()> + 'static,
|
||||
{
|
||||
let handle = std::thread::Builder::new()
|
||||
.name(thread_name.to_string())
|
||||
.stack_size(32 * 1024 * 1024)
|
||||
.spawn(move || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("large-stack scanner test runtime should build");
|
||||
runtime.block_on(test_fn());
|
||||
})
|
||||
.expect("large-stack scanner test thread should spawn");
|
||||
if let Err(payload) = handle.join() {
|
||||
std::panic::resume_unwind(payload);
|
||||
}
|
||||
}
|
||||
|
||||
mod serial_tests {
|
||||
use super::*;
|
||||
|
||||
@@ -737,10 +760,17 @@ mod serial_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[test]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||
async fn rejected_transition_candidate_is_recovered_from_persisted_delete_journal() {
|
||||
fn rejected_transition_candidate_is_recovered_from_persisted_transaction() {
|
||||
run_large_stack_async_test(
|
||||
"scanner-rejected-transition-transaction",
|
||||
rejected_transition_candidate_is_recovered_from_persisted_transaction_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn rejected_transition_candidate_is_recovered_from_persisted_transaction_case() {
|
||||
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
@@ -790,21 +820,20 @@ mod serial_tests {
|
||||
"no cleanup path may delete the candidate while remove failures are enabled"
|
||||
);
|
||||
|
||||
let retained = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
let retained = recover_transition_transaction_records(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("tier delete journal recovery should scan the persisted candidate");
|
||||
assert_eq!(retained.scanned, 1);
|
||||
assert_eq!(retained.deleted, 0);
|
||||
assert_eq!(retained.failed, 1);
|
||||
.expect("transition transaction recovery should scan the persisted candidate");
|
||||
assert_eq!((retained.scanned, retained.recovered, retained.retained, retained.failed), (1, 0, 0, 1));
|
||||
assert_eq!(backend.object_count().await, 1, "failed recovery must retain the remote candidate");
|
||||
|
||||
backend.set_remove_failure(false);
|
||||
let recovered = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("tier delete journal recovery should delete the retained candidate");
|
||||
assert_eq!(recovered.scanned, 1);
|
||||
assert_eq!(recovered.deleted, 1);
|
||||
assert_eq!(recovered.failed, 0);
|
||||
.expect("transition transaction recovery should delete the retained candidate");
|
||||
assert_eq!(
|
||||
(recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
|
||||
(1, 1, 0, 0)
|
||||
);
|
||||
let removed_versions = backend.remove_versions().await;
|
||||
assert!(!removed_versions.is_empty(), "recovery must issue at least one successful delete");
|
||||
assert!(
|
||||
@@ -813,10 +842,10 @@ mod serial_tests {
|
||||
);
|
||||
assert_eq!(backend.object_count().await, 0, "recovery should remove the rejected remote candidate");
|
||||
|
||||
let empty = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("a removed tier delete journal entry should no longer be listed");
|
||||
assert_eq!(empty.scanned, 0, "successful recovery must remove the persisted journal entry");
|
||||
.expect("a removed transition transaction should no longer be listed");
|
||||
assert_eq!(empty.scanned, 0, "successful recovery must remove the persisted transaction");
|
||||
assert_eq!(
|
||||
read_object_fully(&ecstore, bucket_name.as_str(), object_name).await,
|
||||
payload,
|
||||
@@ -824,10 +853,17 @@ mod serial_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[test]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||
async fn cancelled_before_cleanup_store_resolution_persists_journal() {
|
||||
fn cancelled_before_cleanup_store_resolution_persists_transaction() {
|
||||
run_large_stack_async_test(
|
||||
"scanner-cancelled-transition-transaction",
|
||||
cancelled_before_cleanup_store_resolution_persists_transaction_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn cancelled_before_cleanup_store_resolution_persists_transaction_case() {
|
||||
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
@@ -876,9 +912,9 @@ mod serial_tests {
|
||||
|
||||
let retained = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let recovery = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
let recovery = recover_transition_transaction_records(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("the cancelled transition journal should be readable");
|
||||
.expect("the cancelled transition transaction should be readable");
|
||||
if recovery.scanned > 0 {
|
||||
break recovery;
|
||||
}
|
||||
@@ -887,29 +923,36 @@ mod serial_tests {
|
||||
})
|
||||
.await
|
||||
.expect("Drop should persist the rejected candidate through the saved instance context");
|
||||
assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 0, 1));
|
||||
assert_eq!((retained.scanned, retained.recovered, retained.retained, retained.failed), (1, 0, 0, 1));
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while backend.exact_remove_count() < 2 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("Drop cleanup and failed journal recovery must both preserve the exact version constraint");
|
||||
.expect("Drop cleanup and failed transaction recovery must both preserve the exact version constraint");
|
||||
let failed_exact_attempts = backend.exact_remove_count();
|
||||
assert_eq!(
|
||||
failed_exact_attempts, 2,
|
||||
"the cancelled task and the first failed recovery must each preserve the exact delete constraint"
|
||||
);
|
||||
assert_eq!(backend.object_count().await, 1);
|
||||
assert!(backend.remove_versions().await.is_empty());
|
||||
|
||||
backend.set_remove_failure(false);
|
||||
let recovered = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("recovery should delete the candidate retained by the cancelled transition");
|
||||
assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (1, 1, 0));
|
||||
assert_eq!(
|
||||
(recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
|
||||
(1, 1, 0, 0)
|
||||
);
|
||||
assert_eq!(backend.remove_versions().await, backend.put_versions().await);
|
||||
assert_eq!(backend.exact_remove_count(), failed_exact_attempts + 1);
|
||||
assert_eq!(backend.object_count().await, 0);
|
||||
let empty = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("successful recovery should remove the cancellation journal");
|
||||
.expect("successful recovery should remove the cancellation transaction");
|
||||
assert_eq!(empty.scanned, 0);
|
||||
assert_eq!(
|
||||
read_object_fully(&ecstore, bucket_name.as_str(), object_name).await,
|
||||
@@ -918,10 +961,14 @@ mod serial_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[test]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||
async fn rejected_transition_cleanup_durability_matrix() {
|
||||
fn rejected_transition_cleanup_durability_matrix() {
|
||||
run_large_stack_async_test("scanner-transition-cleanup-matrix", rejected_transition_cleanup_durability_matrix_case);
|
||||
}
|
||||
|
||||
async fn rejected_transition_cleanup_durability_matrix_case() {
|
||||
#[derive(Clone, Copy)]
|
||||
enum CleanupCase {
|
||||
Persisted,
|
||||
@@ -1010,44 +1057,67 @@ mod serial_tests {
|
||||
CleanupCase::Persisted | CleanupCase::DeleteFallback => {
|
||||
assert_eq!(backend.remove_versions().await, backend.put_versions().await);
|
||||
assert_eq!(backend.object_count().await, 0, "cleanup must remove the exact candidate");
|
||||
let recovery = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
let recovery = recover_transition_transaction_records(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("successful cleanup must not retain a journal entry");
|
||||
assert_eq!(recovery.scanned, 0);
|
||||
.expect("successful cleanup must leave no failed transition transaction");
|
||||
assert_eq!(recovery.failed, 0);
|
||||
assert_eq!(recovery.retained, 0);
|
||||
assert_eq!(recovery.recovered, recovery.scanned);
|
||||
let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("successful reconciliation must remove every transition transaction");
|
||||
assert_eq!(empty.scanned, 0);
|
||||
}
|
||||
CleanupCase::RetryPersisted => {
|
||||
assert!(
|
||||
!err.to_string().contains("journal retry error"),
|
||||
"a successful journal retry must preserve the original version-constraint error"
|
||||
"the transaction path must not surface the removed journal fallback error"
|
||||
);
|
||||
assert_eq!(backend.object_count().await, 1);
|
||||
let retained = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
let retained = recover_transition_transaction_records(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("the retried journal should be recoverable");
|
||||
assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 0, 1));
|
||||
.expect("the retained transaction should be recoverable");
|
||||
assert_eq!((retained.scanned, retained.recovered, retained.retained, retained.failed), (1, 0, 0, 1));
|
||||
backend.set_remove_failure(false);
|
||||
let recovered = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("recovery should delete the exact retried candidate");
|
||||
assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (1, 1, 0));
|
||||
.expect("recovery should delete the exact transaction candidate");
|
||||
assert_eq!(
|
||||
(recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
|
||||
(1, 1, 0, 0)
|
||||
);
|
||||
assert_eq!(backend.remove_versions().await, backend.put_versions().await);
|
||||
assert_eq!(backend.object_count().await, 0);
|
||||
let empty = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("successful recovery must remove the retried journal");
|
||||
.expect("successful recovery must remove the retained transaction");
|
||||
assert_eq!(empty.scanned, 0);
|
||||
}
|
||||
CleanupCase::FullyFailed => {
|
||||
let message = err.to_string();
|
||||
assert!(message.contains("initial journal error"), "{message}");
|
||||
assert!(message.contains("cleanup error"), "{message}");
|
||||
assert!(message.contains("journal retry error"), "{message}");
|
||||
assert_eq!(backend.object_count().await, 1, "both failed safeguards must leave the candidate visible");
|
||||
assert!(backend.remove_versions().await.is_empty());
|
||||
let recovery = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
let retained = recover_transition_transaction_records(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("failed journal writes must not create partial recovery entries");
|
||||
assert_eq!(recovery.scanned, 0);
|
||||
.expect("the pre-upload transaction must retain ownership after cleanup failure");
|
||||
assert_eq!(retained.scanned, 1, "the failed cleanup must keep one durable transaction owner");
|
||||
assert_eq!(retained.recovered, 0);
|
||||
assert_eq!(retained.retained + retained.failed, 1);
|
||||
backend.set_remove_failure(false);
|
||||
let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("recovery should delete the candidate after the backend becomes available");
|
||||
assert_eq!(
|
||||
(recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
|
||||
(1, 1, 0, 0)
|
||||
);
|
||||
assert_eq!(backend.remove_versions().await, backend.put_versions().await);
|
||||
assert_eq!(backend.object_count().await, 0);
|
||||
let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("successful recovery must remove the failed cleanup transaction");
|
||||
assert_eq!(empty.scanned, 0);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
@@ -1062,20 +1132,7 @@ mod serial_tests {
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
fn test_transition_and_restore_flows() {
|
||||
std::thread::Builder::new()
|
||||
.name("scanner-transition-restore-flows".to_string())
|
||||
.stack_size(32 * 1024 * 1024)
|
||||
.spawn(|| {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("transition and restore test runtime should build");
|
||||
|
||||
runtime.block_on(test_transition_and_restore_flows_inner());
|
||||
})
|
||||
.expect("transition and restore test thread should spawn")
|
||||
.join()
|
||||
.expect("transition and restore test thread should finish");
|
||||
run_large_stack_async_test("scanner-transition-restore-flows", test_transition_and_restore_flows_inner);
|
||||
}
|
||||
|
||||
async fn test_transition_and_restore_flows_inner() {
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::tier_delete_journal::recover_tier_delete_journal_entries;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::transition_transaction::recover_transition_transaction_records;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::{
|
||||
bucket_lifecycle_audit::LcEventSrc,
|
||||
bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, expire_transitioned_object, init_background_expiry},
|
||||
@@ -50,7 +50,7 @@ pub(crate) mod lifecycle {
|
||||
TransitionCleanupStoreBarrier, TransitionOptions, assert_transition_meta_consistent,
|
||||
enqueue_transition_for_existing_objects, expire_transitioned_object, free_version_count, get_bucket_metadata,
|
||||
get_global_tier_config_mgr, init_background_expiry, init_bucket_metadata_sys, init_local_disks, is_err_object_not_found,
|
||||
is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_tier_delete_journal_entries,
|
||||
is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_transition_transaction_records,
|
||||
register_mock_tier_util, update_bucket_metadata, wait_for_free_version_absence,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -39,6 +39,8 @@ for later deletion.
|
||||
- `rustfs-5063` pre-beta.9 Local KMS recovery: persisted Local KMS configs from beta.8 and earlier predate the explicit insecure-development flag, and encrypted key files use the legacy SHA-256 KDF. Remove the config fallback after supported upgrades have rewritten or explicitly resaved all pre-beta.9 configs with the development-default field, and remove the legacy KDF after supported upgrades have rewritten all pre-beta.9 Local KMS key files with explicit at-rest protection.
|
||||
- `sse-local-dek-json-v1` legacy local SSE DEK decoding: releases before the JSON envelope wrote wrapped DEKs as `base64(nonce):base64(ciphertext)`, so readers retain that decoder while all new writes use the versioned JSON envelope. Remove the colon decoder after the minimum supported direct-upgrade release writes JSON envelopes and migration tooling has rewritten every retained legacy object.
|
||||
- `not-initialized-error-code-v1` typed control-plane not-initialized wire code: control-plane RPC responses historically signaled an uninitialized peer only through the literal error_info string "errServerNotInitialized" (one drift site says "storage layer not initialized"). Responses now dual-carry a typed ControlPlaneErrorCode beside the legacy string, and clients prefer the code; the string stays populated and the client substring fallback (is_err_not_initialized, control_plane_failure) stays in place so mixed-version clusters keep classifying older peers' responses. Remove the substring fallback (and stop populating error_info for this case) after the minimum supported RustFS peer version always sends error_code.
|
||||
- `backlog-2097-tier-mutation-v4-error-text` tier-mutation Prepare rejection classification: a v3 server rejects a v4 request before store/runtime dispatch with the FailedPrecondition status and an authenticated, byte-exact unsupported-version message. A v4 coordinator recognizes only that exact code/message/requested-version tuple as definitely not persisted and fails the mutation without sending that peer an incompatible Abort; Unimplemented, near-text, missing/unknown failure classes, timeouts, and every other transport outcome remain ambiguous and stay in identity-bound Abort fanout. There is no automatic v3 retry. New servers retain v3 request/proof decoding for older coordinators, while operators must pause tier edit/remove/clear during a mixed v3/v4 rollout. Remove the text classifier after the minimum supported RustFS peer version returns the signed v4 PreDispatchRejected failure class.
|
||||
- `backlog-2097-tier-delete-journal-v6` tier-delete sole-owner recovery: v6 distinguishes transactions that may replace the xl.meta free-version owner and therefore require the all-pool live-source proof. v5-and-older readers reject and retain v6 records during rolling upgrades instead of performing an unsafe remote delete. Keep v1-v5 readers for upgrade recovery and keep the downgrade prohibition while any v6 record exists; retire the fence only after every supported rollback release understands and enforces v6 proof semantics.
|
||||
- `multipart-compression-default-off-window` staged multipart disk-compression rollout: releases before the resumable legacy decompressor fail transient reads of compressed objects under mid-payload suspension, so multipart uploads advertise the compression marker only when RUSTFS_COMPRESSION_MULTIPART_ENABLED is set in addition to RUSTFS_COMPRESSION_ENABLED, keeping rolling upgrades from creating new compressed multipart objects while pre-fix nodes may still serve reads. Flip the default to enabled (and retire the extra switch) after the minimum supported direct-upgrade release ships the resumable decompressor.
|
||||
|
||||
## Review Checklist
|
||||
|
||||
@@ -165,16 +165,33 @@ and all listing/GET paths exclude them.
|
||||
|
||||
### Lifecycle And Consumers
|
||||
|
||||
Creation: any local delete that removes a version whose transition status is
|
||||
`complete` appends the record via `MetaObject::delete_version` →
|
||||
`init_free_version` (skipped only when `skip_tier_free_version` is set, as on
|
||||
data-movement copies). The same deletes also persist a durable tier-journal
|
||||
entry on every user-facing path: S3 single deletes (`execute_delete_object` →
|
||||
`delete_object_with_tier_delete_journal`), S3 batch deletes, lifecycle expiry,
|
||||
and lifecycle delete-all all prepare and commit a journal entry around the
|
||||
delete. A journal entry is omitted when the removed version's transition state
|
||||
decodes as `TransitionVersionState::Unknown`, or on internal journal-less
|
||||
delete paths that never touch transitioned user objects.
|
||||
Creation: a local delete that removes a version whose transition status is
|
||||
`complete` normally appends the record via `MetaObject::delete_version` →
|
||||
`init_free_version`. User-facing single and batch deletes always retain that
|
||||
historical owner when they actually remove a transitioned source; they do not
|
||||
create a tier journal, probe a fleet capability, or issue a peer mutation RPC.
|
||||
`TransitionVersionState::Unknown` and incomplete destination identities remain
|
||||
on the same conservative free-version path. Delete-marker creation on an Enabled
|
||||
bucket remains unchanged and does not schedule remote deletion.
|
||||
|
||||
Recursive prefix/delete-all cannot preserve per-object markers across its
|
||||
physical directory purge, so it requires a v6 recoverable journal for every
|
||||
transitioned visible source plus a durable dispatch manifest for the complete
|
||||
operation. It fails closed before mutation on legacy metadata or any existing
|
||||
hidden tier free-version under the prefix. Its internal streaming walk
|
||||
discovers logical keys, then exact-loads every key from its authoritative set in
|
||||
every pool, including free versions; the S3 listing merge is never treated as a
|
||||
complete physical-owner inventory. Tier-operation leases remain held from that
|
||||
preflight through journal prepare and physical deletion. Once physical deletion
|
||||
starts, any error is mutation-ambiguous: authorized/dispatched journals remain
|
||||
for recovery to commit owners only after all physical sets prove both the source
|
||||
and exact free-version identity absent; uncertain owners are retained.
|
||||
If a retry discovers a later transitioned source after the manifest reached
|
||||
`DispatchAuthorized`, it replays only the manifest's immutable predecessor set,
|
||||
completes that operation, and leaves the newcomer for a successor dispatch.
|
||||
Operators may retry after the legacy free-version worker has durably completed
|
||||
remote and local cleanup. Journal-less internal deletes and older nodes retain
|
||||
their established marker behavior.
|
||||
|
||||
Consumption while the record exists: the background recovery loop started by
|
||||
`init_background_expiry` (spawned by `spawn_tier_free_version_recovery_once`,
|
||||
@@ -209,16 +226,93 @@ only when its free-version identity matches; a conflicting ordinary version or
|
||||
different free record is an overwrite error. This makes retries idempotent and
|
||||
prevents a free record from replacing a user-visible version.
|
||||
|
||||
### Remote-Tuple Publication Fence
|
||||
|
||||
Cross-pool capability v3 includes a commit-late publication contract for every
|
||||
path that can copy an existing transition tuple to a new physical owner. This
|
||||
capability version is independent of the tier-mutation RPC protocol version.
|
||||
A mixed fleet whose minimum cross-pool capability is below v3 cannot authorize
|
||||
journal-v6 remote deletion.
|
||||
|
||||
Data movement captures a non-cloneable, process-local source capability before
|
||||
copying, but it does not hold a namespace write lock or tier-operation lease
|
||||
while reading a large body or uploading multipart parts. `NewMultipartUpload`
|
||||
and `UploadPart` are staging only. Immediately before single-PUT rename,
|
||||
Multipart Complete, or a pure-remote/free-version metadata quorum write, the
|
||||
final consumer acquires the exact tier generation (when a remote tuple exists),
|
||||
then fixed/source/target write domains in stable order. The fixed domain is used
|
||||
only for a real remote-tuple decommission publisher; an ordinary local object
|
||||
keeps the lighter source/target commit scope.
|
||||
|
||||
While that owned scope is held, the publisher re-reads the exact source pool and
|
||||
compares version, data directory, modification time, ETag, checksums, transition
|
||||
tuple, transition-version state, and destination identity. A missing or changed
|
||||
source, changed/revoked tier generation, bucket incarnation change, or lost lock
|
||||
fails before target rename. The scope remains owned through rename quorum and
|
||||
the existing rename-tail guard handoff. Consequently, recovery-first ordering
|
||||
cannot delete the remote object and then have a stale restored-transitioned
|
||||
rebalance recreate its tuple; publisher-first ordering makes recovery wait and
|
||||
rescan the newly committed owner.
|
||||
|
||||
Full cross-key S3 Copy is not an ownership-sharing operation: it materializes
|
||||
local data and strips transition, destination, transaction, and free-version
|
||||
keys. Same-key metadata/version-only updates preserve the existing protected
|
||||
state. Admin heal keeps the legacy `nolock` request field for wire compatibility
|
||||
but ignores it as lock authority; final heal writes enter the normal locked
|
||||
path. Restore similarly ignores ambient `ObjectOptions.no_lock`, acquires its
|
||||
own commit-late PUT/Complete lock, validates the restore operation id, and keeps
|
||||
an exact tier generation lease through the local commit.
|
||||
|
||||
### Reference-Audit Result
|
||||
|
||||
After migration, user-facing GET/list/transition/replication/restore paths still
|
||||
exclude the record. Recovery, usage scanning, lifecycle tier cleanup, and heal
|
||||
continue to see it when they request free versions, so an unresolved remote
|
||||
delete remains actionable on the target pool. The committed tier journal remains
|
||||
an independent retry source where one exists; it is not used as a reason to drop
|
||||
the xl.meta record. In particular, `Unknown` transition state records are
|
||||
migrated unchanged rather than discarded: the lifecycle worker retains them if
|
||||
remote identity validation cannot make a delete request.
|
||||
continue to see a legacy/fallback record when they request free versions, so an
|
||||
unresolved remote delete remains actionable on the target pool. Only an
|
||||
authorized recursive prefix/delete-all v6 transaction may instead use a
|
||||
per-source journal as the sole retry source; ordinary single/batch deletes never
|
||||
take that path. A journal discovered
|
||||
alongside an older or fallback free-version does not authorize dropping the
|
||||
record. In particular, `Unknown` transition state records are migrated unchanged
|
||||
rather than discarded: the lifecycle worker retains them if remote identity
|
||||
validation cannot make a delete request.
|
||||
|
||||
Tier edit/remove/clear reference proof uses the internal walk with
|
||||
`include_free_versions = true`, in addition to persisted journal and transition
|
||||
transaction checks. Protocol v3 peer Prepare blocks new reference creators and
|
||||
drains existing tier-operation leases before this proof; protocol v4 preserves
|
||||
that state machine and adds a signed failure classification. Abort carries the
|
||||
canonical Prepare intent, so a peer can create an identity-bound `Aborted`
|
||||
tombstone even when Abort overtakes Prepare. A delayed matching Prepare then
|
||||
converges on `Aborted` instead of reinstalling the block; a conflicting intent
|
||||
with the same mutation id fails closed. The tombstone remains durable until the
|
||||
intent expiry plus the configured clock-skew allowance, including across reload
|
||||
and coordinator-record cleanup. After
|
||||
expiry, a missing-record replay of the original signed Prepare is rejected and
|
||||
cannot recreate a peer-only runtime fence. Abort checks an existing same-identity
|
||||
terminal record before consulting mutable current-config proof, and recovery
|
||||
reconstructs the original Prepared revision for Abort fanout.
|
||||
|
||||
A new server accepts both v3 and v4 requests and selects the matching canonical
|
||||
response proof. During a mixed rollout, an older v3 server rejects a v4 request
|
||||
with an authenticated, byte-exact unsupported-version status before dispatch;
|
||||
the v4 coordinator treats only that exact rejection as definitely not installed,
|
||||
fails the admin mutation, and does not send the peer an incompatible Abort.
|
||||
There is deliberately no automatic v3 retry. `Unimplemented`, near-text,
|
||||
timeouts, missing/unknown failure classes, and other ambiguous outcomes still
|
||||
receive Abort and retain the coordinator retry record if Abort cannot be proven.
|
||||
Operators must pause and drain tier edit/remove/clear operations before starting
|
||||
the rolling upgrade, leave them disabled while any v3-only peer remains, and
|
||||
resume only after every topology member advertises the v4-capable release.
|
||||
Ordinary object I/O and free-version cleanup remain available; xl.meta is
|
||||
unchanged by the rejected mutation.
|
||||
|
||||
Sole-owner transactions use journal v6: v5-and-older readers reject and retain
|
||||
those records, so an old recovery worker cannot bypass the all-pool proof. Older
|
||||
nodes may continue to create fallback free-versions until the rollout is
|
||||
homogeneous. A deployment must not downgrade every v6-aware recovery worker
|
||||
while any v6 record remains; drain the journal first or keep at least one v6-aware
|
||||
worker until cleanup converges.
|
||||
|
||||
Each migrated record emits `state = "free_version_migrated"` with reason
|
||||
`tier_free_version_migrated`. A record consumed before migration emits
|
||||
|
||||
@@ -1109,7 +1109,10 @@ fn build_heal_channel_request(hip: &HealInitParams) -> HealChannelRequest {
|
||||
heal_request.update_parity = Some(hip.hs.update_parity);
|
||||
heal_request.recursive = Some(recursive || root_cluster_target);
|
||||
heal_request.dry_run = Some(hip.hs.dry_run);
|
||||
heal_request.no_lock = Some(hip.hs.no_lock);
|
||||
// `nolock` is retained in the admin wire shape for compatibility, but it
|
||||
// is not transferable lock authority. Admin heals must enter the storage
|
||||
// layer through its normal object-lock path.
|
||||
heal_request.no_lock = Some(false);
|
||||
heal_request.source = HealRequestSource::Admin;
|
||||
heal_request
|
||||
}
|
||||
@@ -2588,7 +2591,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_heal_channel_request_preserves_client_options() {
|
||||
fn test_build_heal_channel_request_preserves_safe_client_options() {
|
||||
let hip = HealInitParams {
|
||||
bucket: "bucket-a".to_string(),
|
||||
obj_prefix: "prefix-a".to_string(),
|
||||
@@ -2623,7 +2626,7 @@ mod tests {
|
||||
assert_eq!(request.update_parity, Some(false));
|
||||
assert_eq!(request.recursive, Some(true));
|
||||
assert_eq!(request.dry_run, Some(true));
|
||||
assert_eq!(request.no_lock, Some(true));
|
||||
assert_eq!(request.no_lock, Some(false));
|
||||
|
||||
let envelope = rustfs_protos::heal_control::Envelope::start(
|
||||
request,
|
||||
@@ -2639,7 +2642,7 @@ mod tests {
|
||||
else {
|
||||
panic!("expected admin heal start command");
|
||||
};
|
||||
assert_eq!(request.no_lock, Some(true));
|
||||
assert_eq!(request.no_lock, Some(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -29,7 +29,7 @@ use super::storage_api::test::object_utils::to_s3s_etag;
|
||||
use super::storage_api::test::runtime::{MockWarmBackend, MockWarmOp, register_mock_tier as register_mock_tier_util};
|
||||
use super::storage_api::test::{
|
||||
ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, StorageObjectInfo as ObjectInfo,
|
||||
StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader,
|
||||
StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader, install_all_v6_fleet_capability_proof,
|
||||
};
|
||||
use super::{multipart_usecase::DefaultMultipartUsecase, object_usecase::DefaultObjectUsecase};
|
||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||
@@ -79,6 +79,12 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
||||
return (paths.clone(), ecstore.clone());
|
||||
}
|
||||
|
||||
// Production startup installs this proof through the notification-system
|
||||
// fleet probe. This isolated single-node harness constructs ECStore
|
||||
// directly, so model that completed all-v6 probe before any journal permit
|
||||
// or background expiry worker can exist.
|
||||
install_all_v6_fleet_capability_proof();
|
||||
|
||||
let test_base_dir = format!("/tmp/rustfs_app_lifecycle_test_{}", Uuid::new_v4());
|
||||
let temp_dir = PathBuf::from(&test_base_dir);
|
||||
if temp_dir.exists() {
|
||||
|
||||
@@ -1270,6 +1270,7 @@ pub(crate) mod test {
|
||||
pub(crate) mod data_usage {
|
||||
pub(crate) use super::super::data_usage::*;
|
||||
}
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::install_all_v6_fleet_capability_proof;
|
||||
pub(crate) use crate::storage::storage_api::test_consumer::{get_global_bucket_metadata_sys, set_bucket_metadata};
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
ECStore, Endpoint, Endpoints, PoolEndpoints, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader,
|
||||
|
||||
@@ -83,6 +83,9 @@ const TIER_MUTATION_PEER_STATE_UNSPECIFIED_WIRE: i32 = 0;
|
||||
const TIER_MUTATION_PEER_STATE_PREPARED_WIRE: i32 = 1;
|
||||
const TIER_MUTATION_PEER_STATE_COMMITTED_WIRE: i32 = 2;
|
||||
const TIER_MUTATION_PEER_STATE_ABORTED_WIRE: i32 = 3;
|
||||
const TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED_WIRE: i32 = 0;
|
||||
const TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED_WIRE: i32 = 1;
|
||||
const TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS_WIRE: i32 = 2;
|
||||
|
||||
fn signal_service_response(success: bool, error_info: Option<String>) -> Response<SignalServiceResponse> {
|
||||
Response::new(SignalServiceResponse {
|
||||
@@ -178,7 +181,10 @@ fn remove_heal_control_replay(
|
||||
|
||||
static HEAL_CONTROL_REPLAY_CACHE: OnceLock<tokio::sync::Mutex<HashMap<String, Arc<HealControlReplayEntry>>>> = OnceLock::new();
|
||||
static NODE_CAPABILITY_SERVER_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
|
||||
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
|
||||
// v3 additionally promises the v6 tier-delete dispatch-manifest policy. The
|
||||
// existing periodic topology probe carries both capabilities so normal object
|
||||
// operations do not add another peer RPC.
|
||||
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 3;
|
||||
|
||||
fn admit_heal_control_replay(
|
||||
replay_cache: &mut HashMap<String, Arc<HealControlReplayEntry>>,
|
||||
@@ -723,9 +729,28 @@ impl TierMutationControlRpcService {
|
||||
.map_err(|_| Status::invalid_argument("tier mutation request length cannot be represented"))?;
|
||||
verify_tonic_canonical_body_digest(request, &body)
|
||||
.map_err(|err| Status::permission_denied(format!("tier mutation authentication failed: {err}")))?;
|
||||
let store = self
|
||||
.resolve_object_store()
|
||||
.ok_or_else(|| Status::failed_precondition("tier mutation object store is not initialized"))?;
|
||||
if !tier_mutation_protocol_version_is_supported(version) {
|
||||
return Err(Status::failed_precondition(format!(
|
||||
"unsupported tier mutation peer protocol version: {version}"
|
||||
)));
|
||||
}
|
||||
let store = match self.resolve_object_store() {
|
||||
Some(store) => store,
|
||||
None if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION => {
|
||||
return tier_mutation_control_response(TierMutationControlResponseInput {
|
||||
version,
|
||||
phase,
|
||||
mutation_id,
|
||||
canonical_payload,
|
||||
success: false,
|
||||
state: TIER_MUTATION_PEER_STATE_UNSPECIFIED_WIRE,
|
||||
applied: false,
|
||||
error_info: Some("tier mutation object store is not initialized".to_string()),
|
||||
failure_class: TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED_WIRE,
|
||||
});
|
||||
}
|
||||
None => return Err(Status::failed_precondition("tier mutation object store is not initialized")),
|
||||
};
|
||||
|
||||
match tier_mutation_peer::handle_tier_mutation_peer_request(store, version, phase, mutation_id, canonical_payload).await {
|
||||
Ok(outcome) => tier_mutation_control_response(TierMutationControlResponseInput {
|
||||
@@ -737,6 +762,7 @@ impl TierMutationControlRpcService {
|
||||
state: tier_mutation_peer_state_to_proto_wire(outcome.state),
|
||||
applied: outcome.applied,
|
||||
error_info: None,
|
||||
failure_class: TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED_WIRE,
|
||||
}),
|
||||
Err(err) => tier_mutation_control_response(TierMutationControlResponseInput {
|
||||
version,
|
||||
@@ -746,7 +772,12 @@ impl TierMutationControlRpcService {
|
||||
success: false,
|
||||
state: TIER_MUTATION_PEER_STATE_UNSPECIFIED_WIRE,
|
||||
applied: false,
|
||||
error_info: Some(err.to_string()),
|
||||
error_info: Some(bounded_tier_mutation_error_info(err.to_string())),
|
||||
failure_class: if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION {
|
||||
TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS_WIRE
|
||||
} else {
|
||||
TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED_WIRE
|
||||
},
|
||||
}),
|
||||
}
|
||||
}
|
||||
@@ -811,15 +842,35 @@ fn parse_tier_mutation_id(mutation_id: &str) -> Result<Uuid, Status> {
|
||||
Ok(parsed)
|
||||
}
|
||||
|
||||
fn tier_mutation_protocol_version_is_supported(version: u32) -> bool {
|
||||
matches!(
|
||||
version,
|
||||
rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION | rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
|
||||
)
|
||||
}
|
||||
|
||||
fn bounded_tier_mutation_error_info(mut error_info: String) -> String {
|
||||
let limit = rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE;
|
||||
if error_info.len() <= limit {
|
||||
return error_info;
|
||||
}
|
||||
let mut boundary = limit;
|
||||
while !error_info.is_char_boundary(boundary) {
|
||||
boundary -= 1;
|
||||
}
|
||||
error_info.truncate(boundary);
|
||||
error_info
|
||||
}
|
||||
|
||||
fn validate_tier_mutation_payload_size(phase: rustfs_protos::TierMutationRpcPhase, payload_len: usize) -> Result<(), Status> {
|
||||
let limit = match phase {
|
||||
rustfs_protos::TierMutationRpcPhase::Prepare => rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE,
|
||||
rustfs_protos::TierMutationRpcPhase::Commit => rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE,
|
||||
rustfs_protos::TierMutationRpcPhase::Abort => {
|
||||
if payload_len != 0 {
|
||||
return Err(Status::invalid_argument("tier mutation abort payload must be empty"));
|
||||
if payload_len == 0 {
|
||||
return Err(Status::invalid_argument("tier mutation abort payload is empty"));
|
||||
}
|
||||
return Ok(());
|
||||
rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE
|
||||
}
|
||||
_ => return Err(Status::invalid_argument("tier mutation rpc phase is unsupported")),
|
||||
};
|
||||
@@ -838,6 +889,7 @@ struct TierMutationControlResponseInput<'a> {
|
||||
state: i32,
|
||||
applied: bool,
|
||||
error_info: Option<String>,
|
||||
failure_class: i32,
|
||||
}
|
||||
|
||||
fn tier_mutation_control_response(
|
||||
@@ -853,6 +905,7 @@ fn tier_mutation_control_response(
|
||||
state: input.state,
|
||||
applied: input.applied,
|
||||
error_info: input.error_info.as_deref(),
|
||||
failure_class: input.failure_class,
|
||||
})
|
||||
.map_err(|_| Status::internal("tier mutation response length cannot be represented"))?;
|
||||
let response_proof = sign_tonic_rpc_response_proof(&canonical_response)
|
||||
@@ -863,6 +916,7 @@ fn tier_mutation_control_response(
|
||||
applied: input.applied,
|
||||
error_info: input.error_info,
|
||||
response_proof: response_proof.into(),
|
||||
failure_class: input.failure_class,
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -2469,10 +2523,10 @@ impl Node for NodeService {
|
||||
#[allow(unused_imports)]
|
||||
mod tests {
|
||||
use super::{
|
||||
CollectMetricsOpts, DiskStore, Error, HEAL_CONTROL_PAYLOAD_MAX_SIZE, KMS_SIGNAL_SUBSYSTEM, MetricType, Node as _,
|
||||
NodeService, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_PUBLICATION_LEASE_TTL_MS, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS, admit_heal_control_replay,
|
||||
CROSS_POOL_FENCE_SUPPORTED_VERSION, CollectMetricsOpts, DiskStore, Error, HEAL_CONTROL_PAYLOAD_MAX_SIZE,
|
||||
KMS_SIGNAL_SUBSYSTEM, MetricType, Node as _, NodeService, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_PUBLICATION_LEASE_TTL_MS,
|
||||
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS, admit_heal_control_replay,
|
||||
background_rebalance_start_error_message, execute_heal_control_envelope_with_manager,
|
||||
initialize_heal_topology_fingerprint, initialize_heal_topology_fingerprint_with_probe, legacy_scanner_activity_response,
|
||||
make_heal_control_server, make_heal_control_server_with_cache, make_server, make_server_for_context,
|
||||
@@ -2514,9 +2568,9 @@ mod tests {
|
||||
RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerPublicationLeaseReleaseRequest,
|
||||
ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest,
|
||||
SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest,
|
||||
StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationPeerState, TierMutationPrepareRequest,
|
||||
UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
|
||||
WriteRequest,
|
||||
StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationFailureClass,
|
||||
TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest,
|
||||
VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, WriteRequest,
|
||||
heal_control_service_client::HealControlServiceClient,
|
||||
heal_control_service_server::{HealControlService as _, HealControlServiceServer},
|
||||
node_service_client::NodeServiceClient,
|
||||
@@ -3286,6 +3340,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_mutation_control_requires_body_bound_auth_before_store_lookup() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("tier-mutation-control-auth-test-secret".to_string());
|
||||
let service = make_tier_mutation_control_server_for_context(None);
|
||||
let mutation_id = uuid::Uuid::new_v4();
|
||||
let unsigned = service
|
||||
@@ -3317,8 +3372,26 @@ mod tests {
|
||||
let unavailable = service
|
||||
.prepare_tier_mutation(signed)
|
||||
.await
|
||||
.expect_err("authenticated request still requires initialized object store");
|
||||
assert_eq!(unavailable.code(), tonic::Code::FailedPrecondition);
|
||||
.expect("authenticated v4 pre-dispatch rejection should be signed")
|
||||
.into_inner();
|
||||
assert!(!unavailable.success);
|
||||
assert_eq!(unavailable.failure_class, TierMutationFailureClass::PreDispatchRejected as i32);
|
||||
assert_eq!(unavailable.error_info.as_deref(), Some("tier mutation object store is not initialized"));
|
||||
let canonical =
|
||||
rustfs_protos::canonical_tier_mutation_rpc_response_body(rustfs_protos::TierMutationRpcResponseProofInput {
|
||||
version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: rustfs_protos::TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: b"intent",
|
||||
success: false,
|
||||
state: TierMutationPeerState::Unspecified as i32,
|
||||
applied: false,
|
||||
error_info: Some("tier mutation object store is not initialized"),
|
||||
failure_class: TierMutationFailureClass::PreDispatchRejected as i32,
|
||||
})
|
||||
.expect("small response should encode");
|
||||
crate::storage::storage_api::verify_tonic_rpc_response_proof(&canonical, &unavailable.response_proof)
|
||||
.expect("v4 pre-dispatch rejection must authenticate its failure class");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -3337,12 +3410,13 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_mutation_control_rejects_old_protocol_version_before_store_lookup() {
|
||||
async fn tier_mutation_control_rejects_unsupported_protocol_version_before_store_lookup() {
|
||||
let service = make_tier_mutation_control_server_for_context(None);
|
||||
let mutation_id = uuid::Uuid::new_v4();
|
||||
let payload = Bytes::from_static(b"intent");
|
||||
let unsupported_version = rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION - 1;
|
||||
let mut request = Request::new(TierMutationPrepareRequest {
|
||||
version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION - 1,
|
||||
version: unsupported_version,
|
||||
mutation_id: mutation_id.to_string(),
|
||||
canonical_payload: payload,
|
||||
});
|
||||
@@ -3361,6 +3435,38 @@ mod tests {
|
||||
.await
|
||||
.expect_err("old tier mutation protocol version must fail closed");
|
||||
assert_eq!(error.code(), tonic::Code::FailedPrecondition);
|
||||
assert_eq!(
|
||||
error.message(),
|
||||
format!("unsupported tier mutation peer protocol version: {}", unsupported_version)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_mutation_control_accepts_v3_before_store_lookup() {
|
||||
let service = make_tier_mutation_control_server_for_context(None);
|
||||
let mutation_id = uuid::Uuid::new_v4();
|
||||
let payload = Bytes::from_static(b"intent");
|
||||
let mut request = Request::new(TierMutationPrepareRequest {
|
||||
version: rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
|
||||
mutation_id: mutation_id.to_string(),
|
||||
canonical_payload: payload,
|
||||
});
|
||||
let body = rustfs_protos::canonical_tier_mutation_rpc_body(
|
||||
request.get_ref().version,
|
||||
rustfs_protos::TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
&request.get_ref().canonical_payload,
|
||||
)
|
||||
.expect("v3 request should encode");
|
||||
set_tonic_canonical_body_digest(&mut request, &body).expect("digest metadata should encode");
|
||||
mark_v2_authenticated(&mut request);
|
||||
|
||||
let error = service
|
||||
.prepare_tier_mutation(request)
|
||||
.await
|
||||
.expect_err("accepted v3 request should reach the missing-store check");
|
||||
assert_eq!(error.code(), tonic::Code::FailedPrecondition);
|
||||
assert_eq!(error.message(), "tier mutation object store is not initialized");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -3379,6 +3485,32 @@ mod tests {
|
||||
assert_eq!(error.code(), tonic::Code::InvalidArgument);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_mutation_control_rejects_invalid_abort_payload_before_auth_and_store_lookup() {
|
||||
let service = make_tier_mutation_control_server_for_context(None);
|
||||
let mutation_id = uuid::Uuid::new_v4();
|
||||
let empty = service
|
||||
.abort_tier_mutation(Request::new(TierMutationAbortRequest {
|
||||
version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
mutation_id: mutation_id.to_string(),
|
||||
canonical_payload: Bytes::new(),
|
||||
}))
|
||||
.await
|
||||
.expect_err("empty abort must fail before digest construction");
|
||||
assert_eq!(empty.code(), tonic::Code::InvalidArgument);
|
||||
|
||||
let oversized = Bytes::from(vec![0; rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE + 1]);
|
||||
let error = service
|
||||
.abort_tier_mutation(Request::new(TierMutationAbortRequest {
|
||||
version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
mutation_id: mutation_id.to_string(),
|
||||
canonical_payload: oversized,
|
||||
}))
|
||||
.await
|
||||
.expect_err("oversized abort must fail before digest construction");
|
||||
assert_eq!(error.code(), tonic::Code::InvalidArgument);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_mutation_peer_state_wire_constants_match_generated_proto() {
|
||||
assert_eq!(
|
||||
@@ -3390,6 +3522,19 @@ mod tests {
|
||||
assert_eq!(super::TIER_MUTATION_PEER_STATE_ABORTED_WIRE, TierMutationPeerState::Aborted as i32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_mutation_error_info_bound_preserves_utf8_and_byte_limit() {
|
||||
let ascii = "a".repeat(rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE + 1);
|
||||
let bounded = super::bounded_tier_mutation_error_info(ascii);
|
||||
assert_eq!(bounded.len(), rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE);
|
||||
|
||||
let unicode = "界".repeat(rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE);
|
||||
let bounded = super::bounded_tier_mutation_error_info(unicode);
|
||||
assert!(bounded.len() <= rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE);
|
||||
assert!(bounded.is_char_boundary(bounded.len()));
|
||||
assert!(bounded.chars().all(|character| character == '界'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_mutation_control_response_proof_binds_request_and_result() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("tier-mutation-control-response-proof-test-secret".to_string());
|
||||
@@ -3404,6 +3549,7 @@ mod tests {
|
||||
state: TierMutationPeerState::Unspecified as i32,
|
||||
applied: false,
|
||||
error_info: Some("store failed".to_string()),
|
||||
failure_class: TierMutationFailureClass::Ambiguous as i32,
|
||||
})
|
||||
.expect("response proof should be signed")
|
||||
.into_inner();
|
||||
@@ -3417,6 +3563,7 @@ mod tests {
|
||||
state: TierMutationPeerState::Unspecified as i32,
|
||||
applied: false,
|
||||
error_info: Some("store failed"),
|
||||
failure_class: TierMutationFailureClass::Ambiguous as i32,
|
||||
})
|
||||
.expect("small mutation response should encode");
|
||||
crate::storage::storage_api::verify_tonic_rpc_response_proof(&canonical, &response.response_proof)
|
||||
@@ -3432,6 +3579,7 @@ mod tests {
|
||||
state: TierMutationPeerState::Unspecified as i32,
|
||||
applied: false,
|
||||
error_info: Some("store failed"),
|
||||
failure_class: TierMutationFailureClass::Ambiguous as i32,
|
||||
})
|
||||
.expect("small mutation response should encode");
|
||||
let error = crate::storage::storage_api::verify_tonic_rpc_response_proof(&tampered, &response.response_proof)
|
||||
@@ -3599,7 +3747,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cross_pool_fence_probe_authenticates_supported_v2_state() {
|
||||
async fn cross_pool_fence_probe_authenticates_supported_v3_state() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("cross-pool-fence-node-service-test-secret".to_string());
|
||||
let endpoints = heal_control_test_endpoints_with_coordinator("node-0", true);
|
||||
assert!(
|
||||
@@ -3664,7 +3812,7 @@ mod tests {
|
||||
|
||||
assert!(response.success);
|
||||
assert_eq!(response.error_info, None);
|
||||
assert_eq!(&response.result[..4], &2_u32.to_be_bytes());
|
||||
assert_eq!(&response.result[..4], &CROSS_POOL_FENCE_SUPPORTED_VERSION.to_be_bytes());
|
||||
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(&response.result[4..])
|
||||
.expect("capability identity should decode");
|
||||
assert_eq!(topology_member, "node-a:9000");
|
||||
|
||||
@@ -404,6 +404,8 @@ pub(crate) mod ecstore_admin {
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_bucket {
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::tier_delete_journal::test_util::install_all_v6_fleet_capability_proof;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::{
|
||||
bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, migration, object_lock, policy_sys,
|
||||
replication, tagging, target, utils,
|
||||
|
||||
@@ -21,12 +21,12 @@
|
||||
2|crates/ecstore/src/bucket/replication/replication_target_boundary.rs
|
||||
3|crates/ecstore/src/cluster/rpc/http_auth.rs
|
||||
2|crates/ecstore/src/cluster/rpc/internode_data_transport.rs
|
||||
11|crates/ecstore/src/cluster/rpc/peer_rest_client.rs
|
||||
10|crates/ecstore/src/cluster/rpc/peer_rest_client.rs
|
||||
10|crates/ecstore/src/cluster/rpc/peer_s3_client.rs
|
||||
10|crates/ecstore/src/cluster/rpc/remote_disk.rs
|
||||
6|crates/ecstore/src/config/com.rs
|
||||
14|crates/ecstore/src/config/storageclass.rs
|
||||
183|crates/ecstore/src/core/pools.rs
|
||||
182|crates/ecstore/src/core/pools.rs
|
||||
8|crates/ecstore/src/data_movement/mod.rs
|
||||
2|crates/ecstore/src/data_usage/local_snapshot.rs
|
||||
12|crates/ecstore/src/data_usage/mod.rs
|
||||
@@ -70,5 +70,5 @@
|
||||
12|crates/ecstore/src/store/init.rs
|
||||
2|crates/ecstore/src/store/init_format.rs
|
||||
3|crates/ecstore/src/store/multipart.rs
|
||||
8|crates/ecstore/src/store/object.rs
|
||||
7|crates/ecstore/src/store/object.rs
|
||||
5|crates/ecstore/src/store/rebalance/support.rs
|
||||
|
||||
Reference in New Issue
Block a user