fix(ecstore): make transitioned cleanup crash-safe (#6978)

* fix(ecstore): fence transitioned object cleanup

* fix(ecstore): address ILM recovery review findings

* fix(ecstore): complete crash-safe tier cleanup recovery

* test(ecstore): avoid typo false positive

* fix(ecstore): stabilize decommission error buckets

* fix(ecstore): stabilize transition delete validation

* fix(ecstore): resume authorized tier delete dispatch

* fix(ecstore): satisfy feature clippy
This commit is contained in:
cxymds
2026-09-01 19:09:22 +08:00
committed by GitHub
parent bd66fa9dca
commit 6e26769265
46 changed files with 16999 additions and 1865 deletions
+15
View File
@@ -75,6 +75,10 @@ pub mod bucket {
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
inspect_transition_transaction_for_operator,
};
#[cfg(feature = "test-util")]
pub use crate::bucket::lifecycle::transition_transaction::{
TransitionTransactionRecoveryStats, recover_transition_transaction_records,
};
}
pub mod evaluator {
@@ -99,6 +103,17 @@ pub mod bucket {
pub use crate::bucket::lifecycle::tier_delete_journal::{
persist_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
};
#[cfg(feature = "test-util")]
pub mod test_util {
/// Model a single-node, all-v6 fleet after its capability probe has completed.
///
/// Call this only once while constructing an isolated test store, before any
/// tier-delete journal permit or background worker can be active.
pub fn install_all_v6_fleet_capability_proof() {
crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
}
}
}
pub mod tier_last_day_stats {
@@ -41,7 +41,7 @@ use crate::bucket::lifecycle::tier_free_version_recovery::{
DEFAULT_FREE_VERSION_RECOVERY_LIMIT, FreeVersionRecoveryStats, recover_tier_free_versions_with_cancel,
};
use crate::bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent_with_manager_and_identity};
use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_with_lease_idempotent};
use crate::bucket::lifecycle::transition_transaction::run_transition_transaction_recovery_loop;
use crate::bucket::object_lock::ObjectLockApi;
use crate::bucket::versioning::VersioningApi as _;
@@ -50,7 +50,10 @@ use crate::disk::error::DiskError;
use crate::disk::{DeleteOptions, Disk, DiskAPI, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, STORAGE_FORMAT_FILE};
use crate::error::Error;
use crate::error::StorageError;
use crate::error::{is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, is_network_or_host_down};
use crate::error::{
is_err_object_not_found, is_err_read_quorum, is_err_strict_volume_not_found, is_err_version_not_found,
is_network_or_host_down,
};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use crate::object_api::{ObjectEncryptionResolver, ReadPlan};
use crate::services::tier::{
@@ -586,25 +589,217 @@ impl ExpiryOp for FreeVersionTask {
}
}
async fn delete_free_version_remote_object(
async fn acquire_free_version_tier_lease(
oi: &ObjectInfo,
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
) -> Result<(), std::io::Error> {
) -> Result<(TierOperationLease, bool), std::io::Error> {
let version_id_exact = validate_transition_remote_version(oi)?;
let identity = tier_destination_id_from_metadata(&oi.user_defined)?
.ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?;
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
let lease =
TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, &oi.transitioned_object.tier, identity)
.await
.map_err(std::io::Error::other)?;
Ok((lease, version_id_exact))
}
async fn delete_free_version_remote_object_with_lease(
oi: &ObjectInfo,
lease: &TierOperationLease,
version_id_exact: bool,
) -> Result<(), std::io::Error> {
delete_object_from_remote_tier_with_lease_idempotent(
&oi.transitioned_object.name,
&oi.transitioned_object.version_id,
&oi.transitioned_object.tier,
identity,
tier_config_mgr,
lease,
version_id_exact,
)
.await?;
Ok(())
}
fn free_version_physical_topology_generation(api: &ECStore) -> String {
let mut hasher = Sha256::new();
for pool in &api.pools {
hasher.update(pool.pool_idx.to_be_bytes());
hasher.update(pool.disk_set.len().to_be_bytes());
for set in &pool.disk_set {
hasher.update(set.set_index.to_be_bytes());
}
}
rustfs_utils::crypto::hex(hasher.finalize().as_slice())
}
fn free_version_remote_tuple_matches(candidate: &ObjectInfo, expected: &ObjectInfo) -> std::io::Result<bool> {
if candidate.transitioned_object.tier != expected.transitioned_object.tier
|| candidate.transitioned_object.name != expected.transitioned_object.name
{
return Ok(false);
}
let candidate_identity = tier_destination_id_from_metadata(&candidate.user_defined)?
.ok_or_else(|| std::io::Error::other("tier free-version is missing its backend identity"))?;
let expected_identity = tier_destination_id_from_metadata(&expected.user_defined)?
.ok_or_else(|| std::io::Error::other("tier free-version task is missing its backend identity"))?;
if candidate_identity != expected_identity {
return Ok(false);
}
if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|| expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
{
return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"tier free-version remote version state is unknown",
));
}
Ok(candidate.transition_version_state == expected.transition_version_state
&& candidate.transitioned_object.version_id == expected.transitioned_object.version_id)
}
async fn scan_exact_free_version_targets(
api: &ECStore,
oi: &ObjectInfo,
local_object: &str,
) -> std::io::Result<Vec<(Arc<SetDisks>, FileInfo)>> {
let mut targets = Vec::new();
for pool in &api.pools {
for set in &pool.disk_set {
let versions = match set.load_file_info_versions_exact(&oi.bucket, &oi.name).await {
Ok(Some(versions)) => versions,
Ok(None) => continue,
Err(err) if is_err_strict_volume_not_found(&err) => continue,
Err(err) => return Err(std::io::Error::other(err)),
};
for version in versions.versions.iter().chain(versions.free_versions.iter()) {
let candidate = ObjectInfo::from_file_info(version, &oi.bucket, &oi.name, true);
if free_version_remote_tuple_matches(&candidate, oi)? {
if candidate.transitioned_object.free_version {
// Data movement can leave the same remote tuple in
// several physical pools. Ordinary deletion assigns a
// fresh local free-version UUID to each copy, but all
// of those markers own the same idempotent remote
// DELETE. Consume them together while holding every
// physical object lock; treating their local UUIDs as
// conflicting would strand cleanup forever.
let mut actual = version.clone();
actual.name = local_object.to_string();
targets.push((Arc::clone(set), actual));
} else {
return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"a live transitioned source still references the free-version remote tuple",
));
}
}
}
}
}
Ok(targets)
}
fn free_version_cleanup_fences_current(
topology_generation: &str,
api: &ECStore,
bucket_guard: &rustfs_lock::NamespaceLockGuard,
object_guards: &[crate::store::ObjectLockDiagGuard],
lease: &TierOperationLease,
cancel: &CancellationToken,
deadline: tokio::time::Instant,
) -> bool {
!cancel.is_cancelled()
&& tokio::time::Instant::now() < deadline
&& !bucket_guard.is_lock_lost()
&& object_guards.iter().all(|guard| !guard.is_lock_lost())
&& lease.is_current_generation()
&& free_version_physical_topology_generation(api) == topology_generation
}
async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel: &CancellationToken) -> std::io::Result<bool> {
const FREE_VERSION_REMOTE_DEADLINE: StdDuration = StdDuration::from_secs(30);
let topology_generation = free_version_physical_topology_generation(&api);
let bucket_guard = api
.acquire_bucket_lifecycle_read_lock(&oi.bucket)
.await
.map_err(std::io::Error::other)?;
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
let local_object = encode_dir_object(&oi.name);
let object_guards = api
.acquire_all_physical_object_write_locks("tier_free_version_cleanup", &oi.bucket, &local_object)
.await
.map_err(std::io::Error::other)?;
let targets = scan_exact_free_version_targets(&api, oi, &local_object).await?;
if targets.is_empty() {
return Ok(false);
}
let deadline = tokio::time::Instant::now() + FREE_VERSION_REMOTE_DEADLINE;
if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"tier free-version cleanup fence is invalid before remote delete",
));
}
tokio::select! {
_ = cancel.cancelled() => {
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled"));
}
result = tokio::time::timeout_at(
deadline,
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact),
) => {
result
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out"))??;
}
}
if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
// Remote DELETE is idempotent, but a changed fence makes the local
// outcome ambiguous. Keep every marker for a fully fenced retry.
return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"tier free-version cleanup fence changed after remote delete",
));
}
let mut first_error = None;
for (set, actual) in &targets {
let mut delete_request = FileInfo {
name: local_object.clone(),
version_id: actual.version_id,
..Default::default()
};
delete_request.set_tier_free_version();
if let Err(err) = set
.delete_object_version(&oi.bucket, &local_object, &delete_request, false)
.await
&& first_error.is_none()
{
first_error = Some(std::io::Error::other(err));
}
}
let remaining = scan_exact_free_version_targets(&api, oi, &local_object).await?;
if !remaining.is_empty() {
return Err(first_error.unwrap_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"tier free-version cleanup remained on at least one physical set",
)
}));
}
if let Some(err) = first_error {
return Err(err);
}
Ok(true)
}
#[cfg(all(test, feature = "test-util"))]
async fn delete_free_version_remote_object(
oi: &ObjectInfo,
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
) -> Result<(), std::io::Error> {
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await
}
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
@@ -618,8 +813,11 @@ where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = T>,
{
delete_free_version_remote_object(oi, tier_config_mgr).await?;
Ok(delete_local().await)
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await?;
let result = delete_local().await;
drop(lease);
Ok(result)
}
struct NewerNoncurrentTask {
@@ -690,6 +888,10 @@ impl ExpiryState {
usize::try_from(self.stats.pending_tasks().max(0)).unwrap_or(usize::MAX)
}
pub fn active_tasks(&self) -> usize {
usize::try_from(self.stats.active_tasks().max(0)).unwrap_or(usize::MAX)
}
fn send_expiry_task(&self, wrkr: Sender<Option<ExpiryOpType>>, task: ExpiryOpType) -> bool {
let queued = wrkr.try_send(Some(task)).is_ok();
if queued {
@@ -826,7 +1028,7 @@ impl ExpiryState {
}
pub async fn resize_workers(n: usize, api: Arc<ECStore>) {
let expiry_state = runtime_sources::expiry_state_handle();
let expiry_state = api.ctx.expiry_state();
if n == expiry_state.read().await.tasks_tx.len() || n < 1 {
return;
}
@@ -867,7 +1069,7 @@ impl ExpiryState {
stats: Arc<ExpiryStats>,
recovery_notify: Arc<Notify>,
) {
let cancel_token = runtime_sources::background_services_cancel_token().unwrap_or_else(|| {
let cancel_token = api.ctx.background_cancel_token().unwrap_or_else(|| {
static FALLBACK: std::sync::OnceLock<tokio_util::sync::CancellationToken> = std::sync::OnceLock::new();
FALLBACK.get_or_init(tokio_util::sync::CancellationToken::new).clone()
});
@@ -968,119 +1170,33 @@ impl ExpiryState {
else if v.as_any().is::<FreeVersionTask>() {
let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("FreeVersionTask downcast failed");
let oi = v.0.clone();
if let Err(err) = delete_free_version_remote_object(&oi, &api.tier_config_mgr()).await {
recovery_notify.notify_one();
debug!(
bucket = %oi.bucket,
object = %oi.name,
remote_object = %oi.transitioned_object.name,
remote_version_id = %oi.transitioned_object.version_id,
tier = %oi.transitioned_object.tier,
error = ?err,
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
reason = "remote_tier_delete_failed",
"Lifecycle worker skipped remote tier delete"
);
continue;
}
let local_object = encode_dir_object(&oi.name);
let mut fi = FileInfo {
name: local_object.clone(),
version_id: oi.version_id,
..Default::default()
};
// This removes an existing internal cleanup marker. Keeping
// `deleted` false makes duplicate tasks return not-found
// instead of creating an ordinary delete marker.
fi.set_tier_free_version();
let mut deleted_locally = false;
for pool in &api.pools {
let set = pool.get_disks_by_key(&local_object);
let ns_lock = match set.new_ns_lock(&oi.bucket, &local_object).await {
Ok(lock) => lock,
Err(err) => {
recovery_notify.notify_one();
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
pool_index = pool.pool_idx,
set_index = set.set_index,
error = ?err,
reason = "local_free_version_lock_failed",
"Lifecycle worker failed to create local free-version cleanup lock"
);
continue;
}
};
let _object_lock_guard =
match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await {
Ok(guard) => guard,
Err(err) => {
recovery_notify.notify_one();
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
pool_index = pool.pool_idx,
set_index = set.set_index,
error = ?err,
reason = "local_free_version_lock_failed",
"Lifecycle worker failed to acquire local free-version cleanup lock"
);
continue;
}
};
match set
.delete_object_version(&oi.bucket, &local_object, &fi, false)
.await
{
Ok(()) => {
deleted_locally = true;
break;
}
Err(err) if is_err_version_not_found(&err) || is_err_object_not_found(&err) => continue,
Err(err) => {
recovery_notify.notify_one();
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
remote_object = %oi.transitioned_object.name,
remote_version_id = %oi.transitioned_object.version_id,
tier = %oi.transitioned_object.tier,
error = ?err,
reason = "local_free_version_delete_failed",
"Lifecycle worker failed local free-version cleanup"
);
break;
}
}
}
if !deleted_locally {
debug!(
match cleanup_free_version_exact(api.clone(), &oi, &cancel_token).await {
Ok(true) => {}
Ok(false) => debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
remote_object = %oi.transitioned_object.name,
remote_version_id = %oi.transitioned_object.version_id,
tier = %oi.transitioned_object.tier,
reason = "local_free_version_missing",
"Lifecycle worker could not find transitioned free version locally"
);
"Lifecycle worker found that the exact free-version was already absent"
),
Err(err) => {
recovery_notify.notify_one();
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
remote_object = %oi.transitioned_object.name,
remote_version_id = %oi.transitioned_object.version_id,
tier = %oi.transitioned_object.tier,
error = ?err,
reason = "free_version_exact_cleanup_deferred",
"Lifecycle worker retained the exact free-version for a fenced retry"
);
}
}
}
else {
@@ -1152,8 +1268,8 @@ fn set_recovered_free_version_enqueue_observer(
RecoveredFreeVersionEnqueueObserverGuard
}
pub async fn enqueue_recovered_free_version(oi: ObjectInfo) -> bool {
let expiry_state = runtime_sources::expiry_state_handle();
pub async fn enqueue_recovered_free_version(api: &ECStore, oi: ObjectInfo) -> bool {
let expiry_state = api.ctx.expiry_state();
let queued = enqueue_recovered_free_version_with_state(&expiry_state, oi).await;
#[cfg(test)]
@@ -2580,8 +2696,8 @@ fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<(
}
Some(tokio::spawn(async move {
let cancel_token = runtime_sources::background_services_cancel_token().unwrap_or_default();
let expiry_state = runtime_sources::expiry_state_handle();
let cancel_token = api.ctx.background_cancel_token().unwrap_or_default();
let expiry_state = api.ctx.expiry_state();
run_tier_free_version_recovery_loop(
cancel_token,
expiry_state,
@@ -6229,9 +6345,18 @@ mod tests {
rustfs_utils::crypto::hex(old_identity),
);
oi.user_defined = Arc::new(metadata.clone());
let lease_observed_during_local_delete = Arc::new(std::sync::atomic::AtomicBool::new(false));
delete_free_version_remote_object_then(&oi, &manager, {
let local_delete_calls = Arc::clone(&local_delete_calls);
let lease_observed_during_local_delete = Arc::clone(&lease_observed_during_local_delete);
let manager = manager.clone();
move || async move {
assert_eq!(
crate::services::tier::tier::TierConfigMgr::active_operation_lease_count(&manager, "WARM").await,
1,
"the identity-bound tier lease must span the exact local marker delete"
);
lease_observed_during_local_delete.store(true, Ordering::Relaxed);
local_delete_calls.fetch_add(1, Ordering::Relaxed);
}
})
@@ -6239,6 +6364,12 @@ mod tests {
.expect("matching destination identity should allow idempotent remote cleanup");
assert_eq!(old_backend.remove_count().await, 1);
assert_eq!(local_delete_calls.load(Ordering::Relaxed), 1);
assert!(lease_observed_during_local_delete.load(Ordering::Relaxed));
assert_eq!(
crate::services::tier::tier::TierConfigMgr::active_operation_lease_count(&manager, "WARM").await,
0,
"the tier lease should be released after the local marker delete completes"
);
let mut single_prefix_metadata = HashMap::new();
single_prefix_metadata.insert(
@@ -6472,6 +6603,7 @@ mod tests {
let state = ExpiryState::new();
let mut state = state.write().await;
let je = Jentry {
persisted_version: 0,
obj_name: "remote/object".to_string(),
version_id: "remote-version".to_string(),
tier_name: "WARM".to_string(),
@@ -6480,6 +6612,7 @@ mod tests {
version_state: rustfs_filemeta::TransitionVersionState::Exact,
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
source: None,
dispatch: None,
};
let err = state
@@ -6620,6 +6753,7 @@ mod tests {
let state = ExpiryState::new_with_unconsumed_worker_channel(1);
let mut state = state.write().await;
let je = Jentry {
persisted_version: 0,
obj_name: "remote/object".to_string(),
version_id: "remote-version".to_string(),
tier_name: "WARM".to_string(),
@@ -6628,6 +6762,7 @@ mod tests {
version_state: rustfs_filemeta::TransitionVersionState::Exact,
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
source: None,
dispatch: None,
};
state
@@ -6759,7 +6894,7 @@ mod tests {
};
assert!(
super::enqueue_recovered_free_version(oi).await,
super::enqueue_recovered_free_version(&ecstore, oi).await,
"the resized production worker queue should accept the task"
);
stop_tx.send(None).await.expect("worker stop signal should be delivered");
@@ -6875,12 +7010,12 @@ mod tests {
.await
.expect("free-version task should reach the worker");
tokio::time::timeout(StdDuration::from_secs(30), async {
while remote_backend.remove_count().await == 0 {
while stats.active_tasks() == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("worker should complete remote cleanup before taking the local lock");
.expect("worker should mark the cleanup task active before the lock assertion");
let completed_while_locked = tokio::time::timeout(StdDuration::from_millis(100), async {
while stats.active_tasks() != 0 {
tokio::task::yield_now().await;
@@ -6889,7 +7024,12 @@ mod tests {
.await;
assert!(
completed_while_locked.is_err(),
"local cleanup must wait while a competing object writer owns the namespace lock"
"the cleanup task must wait while a competing object writer owns the namespace lock"
);
assert_eq!(
remote_backend.remove_count().await,
0,
"the remote tuple must not be deleted before the all-physical namespace fence is acquired"
);
for disk_path in &disk_paths {
assert!(
@@ -6900,6 +7040,13 @@ mod tests {
}
drop(object_lock_guard);
tokio::time::timeout(StdDuration::from_secs(30), async {
while remote_backend.remove_count().await == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("worker should delete the remote tuple after acquiring the released namespace fence");
tx.send(None).await.expect("worker stop signal should be delivered");
worker.await.expect("free-version worker should stop cleanly");
@@ -6995,6 +7142,7 @@ mod tests {
.next()
.expect("seeded free version should be recoverable");
let stale_version_id = oi.version_id.expect("free version should have a concrete UUID");
let ordinary_marker_mod_time = OffsetDateTime::now_utc();
for disk_path in &disk_paths {
let metadata_path = disk_path.join(&bucket).join(object).join(STORAGE_FORMAT_FILE);
@@ -7017,7 +7165,7 @@ mod tests {
name: object.to_string(),
version_id: Some(stale_version_id),
deleted: true,
mod_time: Some(OffsetDateTime::now_utc()),
mod_time: Some(ordinary_marker_mod_time),
..Default::default()
})
.expect("same-ID ordinary marker should replace the stale free version");
@@ -7031,6 +7179,13 @@ mod tests {
.expect("same-ID ordinary marker metadata should be written");
}
assert!(
!super::cleanup_free_version_exact(Arc::clone(&ecstore), &oi, &CancellationToken::new())
.await
.expect("a stale task whose local UUID now names an ordinary marker should be an idempotent no-op"),
"the stale free-version task must not report local cleanup"
);
let state = ExpiryState::new();
let (stats, recovery_notify) = {
let state = state.read().await;
@@ -11522,7 +11677,7 @@ mod tests {
#[cfg(feature = "test-util")]
#[tokio::test]
async fn journal_replay_rejects_unknown_version_state_before_backend_io() {
async fn journal_replay_quarantines_legacy_unknown_version_state_before_backend_io() {
let (_disk_paths, ecstore) = setup_test_env().await;
let (backend, _) = register_recovery_mock_tier(&ecstore).await;
let identity = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
@@ -11530,6 +11685,7 @@ mod tests {
.expect("mock tier lease should be available")
.backend_identity();
let je = Jentry {
persisted_version: 0,
obj_name: "remote/object".to_string(),
version_id: "legacy-version".to_string(),
tier_name: "WARM".to_string(),
@@ -11538,25 +11694,28 @@ mod tests {
version_state: rustfs_filemeta::TransitionVersionState::Unknown,
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
source: None,
dispatch: None,
};
crate::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(ecstore.clone(), &je)
.await
.expect("legacy unknown journal should remain byte-compatible and persistable");
let err = crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
.await
.expect_err("unknown journal state must fail before backend IO");
.expect_err("legacy unknown journal must be quarantined before backend IO");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock);
assert_eq!(backend.remove_count().await, 0);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn journal_replay_deletes_confirmed_exact_provider_token() {
async fn rejected_upload_cleanup_retries_confirmed_exact_provider_token_without_legacy_journal() {
let (_disk_paths, ecstore) = setup_test_env().await;
let (backend, _) = register_recovery_mock_tier(&ecstore).await;
let lease = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
.await
.expect("mock tier lease should be available");
let identity = lease.backend_identity();
backend
.set_put_remote_version(Some("provider-version-token".to_string()))
.await;
@@ -11570,34 +11729,30 @@ mod tests {
.expect("confirmed remote candidate should be seeded");
backend.set_remove_failure(true);
backend.set_reject_non_empty_remote_versions(true);
let je = Jentry {
obj_name: "remote/object".to_string(),
version_id: "provider-version-token".to_string(),
tier_name: "WARM".to_string(),
backend_identity: Some(identity),
version_id_exact: true,
version_state: rustfs_filemeta::TransitionVersionState::Exact,
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
source: None,
};
crate::set_disk::cleanup_rejected_transition_upload_durably(
let err = crate::set_disk::cleanup_rejected_transition_upload_durably(
&lease,
&je.obj_name,
&je.version_id,
"remote/object",
"provider-version-token",
true,
Some(ecstore.clone()),
)
.await
.expect("failed immediate cleanup should remain durable in the journal");
assert!(backend.contains(&je.obj_name).await);
.expect_err("a failed immediate cleanup must remain owned by the caller's transition transaction");
assert_eq!(err.kind(), std::io::ErrorKind::Other);
assert!(backend.contains("remote/object").await);
backend.set_remove_failure(false);
crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
.await
.expect("identity-bound exact journal must retry confirmed candidate cleanup");
crate::set_disk::cleanup_rejected_transition_upload_durably(
&lease,
"remote/object",
"provider-version-token",
true,
Some(ecstore),
)
.await
.expect("the transaction retry must delete the same confirmed candidate");
assert!(!backend.contains(&je.obj_name).await);
assert!(!backend.contains("remote/object").await);
assert_eq!(backend.exact_remove_count(), 2);
assert_eq!(
backend.remove_versions().await,
@@ -11760,11 +11915,14 @@ mod tests {
};
let mut recovery_rx = recovery_rx.lock().await;
assert!(
super::enqueue_recovered_free_version(ObjectInfo {
bucket: "prefill".to_string(),
name: "prefill".to_string(),
..Default::default()
})
super::enqueue_recovered_free_version(
&ecstore,
ObjectInfo {
bucket: "prefill".to_string(),
name: "prefill".to_string(),
..Default::default()
},
)
.await,
"the production recovery queue should accept its first task"
);
@@ -33,6 +33,7 @@ const MANUAL_TRANSITION_CURSOR_MARKER_PROOF_MAX_SIZE: usize = 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum DurableIlmRecordKind {
TierDeleteJournal,
TierDeleteDispatchManifest,
TransitionTransaction,
ManualTransitionJob,
ManualTransitionScope,
@@ -54,6 +55,18 @@ pub(crate) const TIER_DELETE_JOURNAL_NAMESPACE: DurableIlmNamespace = DurableIlm
max_record_size: 64 * 1024,
kind: DurableIlmRecordKind::TierDeleteJournal,
};
pub(crate) const TIER_DELETE_JOURNAL_V6_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
name: "tier-delete-journal-v6",
prefix: "ilm/tier-delete-journal-v6/",
max_record_size: 64 * 1024,
kind: DurableIlmRecordKind::TierDeleteJournal,
};
pub(crate) const TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
name: "tier-delete-dispatch-manifest",
prefix: tier_delete_journal::TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
max_record_size: tier_delete_journal::MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE,
kind: DurableIlmRecordKind::TierDeleteDispatchManifest,
};
pub(crate) const TRANSITION_TRANSACTION_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
name: "transition-transaction",
prefix: "ilm/transition-transactions/records",
@@ -85,8 +98,10 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace
kind: DurableIlmRecordKind::ManualTransitionWorkerResult,
};
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 6] = [
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 8] = [
TIER_DELETE_JOURNAL_NAMESPACE,
TIER_DELETE_JOURNAL_V6_NAMESPACE,
TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
TRANSITION_TRANSACTION_NAMESPACE,
MANUAL_TRANSITION_JOB_NAMESPACE,
MANUAL_TRANSITION_SCOPE_NAMESPACE,
@@ -157,6 +172,15 @@ pub(crate) enum DurableIlmRecordCheckpoint {
content_sha256: String,
identity_sha256: String,
committed: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
dispatch_identity_sha256: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
state: Option<super::tier_sweeper::TierDeleteJournalState>,
},
TierDeleteDispatchManifest {
content_sha256: String,
identity_sha256: String,
state: tier_delete_journal::TierDeleteDispatchManifestState,
},
TransitionTransaction {
content_sha256: String,
@@ -195,6 +219,7 @@ impl DurableIlmRecordCheckpoint {
pub(crate) fn content_sha256(&self) -> &str {
match self {
Self::TierDeleteJournal { content_sha256, .. }
| Self::TierDeleteDispatchManifest { content_sha256, .. }
| Self::TransitionTransaction { content_sha256, .. }
| Self::ManualTransitionJob { content_sha256, .. }
| Self::ManualTransitionScope { content_sha256, .. }
@@ -228,6 +253,19 @@ impl DurableIlmRecordCheckpoint {
}
pub(crate) fn validate_successor(&self, next: &Self) -> Result<()> {
for checkpoint in [self, next] {
if let Self::TierDeleteJournal {
committed,
dispatch_identity_sha256,
state,
..
} = checkpoint
&& (state.is_some() != dispatch_identity_sha256.is_some()
|| state.is_some_and(|state| *committed != (state == super::tier_sweeper::TierDeleteJournalState::Committed)))
{
return Err(Error::other("durable ILM tier delete journal checkpoint is invalid"));
}
}
if self == next {
if let Self::ManualTransitionJob {
progress,
@@ -244,18 +282,64 @@ impl DurableIlmRecordCheckpoint {
let valid = match (self, next) {
(
Self::TierDeleteJournal {
content_sha256: previous_content,
identity_sha256: previous_identity,
committed: previous_committed,
dispatch_identity_sha256: previous_dispatch_identity,
state: previous_state,
..
},
Self::TierDeleteJournal {
content_sha256: next_content,
identity_sha256: next_identity,
committed: next_committed,
dispatch_identity_sha256: next_dispatch_identity,
state: next_state,
..
},
) => {
use super::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared};
let dispatch_identity_is_monotonic = match (previous_dispatch_identity, next_dispatch_identity) {
(Some(previous), Some(next)) => previous == next,
(None, None) => true,
// Old receipts did not record the v6 dispatch binding. A
// byte-identical observation may adopt the stronger proof,
// but an in-flight mutation must fail closed instead of
// guessing which operation owned the journal.
(None, Some(_)) => previous_content == next_content,
(Some(_), None) => false,
};
let state_is_monotonic = match (previous_state, next_state) {
(Some(previous), Some(next)) => {
previous == next || matches!((previous, next), (Prepared, Dispatched) | (Dispatched, Committed))
}
(None, None) => previous_committed == next_committed || (!previous_committed && *next_committed),
(None, Some(_)) => previous_content == next_content,
(Some(_), None) => false,
};
previous_identity == next_identity && dispatch_identity_is_monotonic && state_is_monotonic
}
(
Self::TierDeleteDispatchManifest {
identity_sha256: previous_identity,
state: previous_state,
..
},
Self::TierDeleteDispatchManifest {
identity_sha256: next_identity,
state: next_state,
..
},
) => {
use tier_delete_journal::TierDeleteDispatchManifestState::{
Aborted, Aborting, Completed, DispatchAuthorized, Preparing,
};
previous_identity == next_identity
&& (previous_committed == next_committed || (!previous_committed && *next_committed))
&& matches!(
(previous_state, next_state),
(Preparing, DispatchAuthorized | Aborting) | (Aborting, Aborted) | (DispatchAuthorized, Completed)
)
}
(
Self::TransitionTransaction {
@@ -351,6 +435,49 @@ impl DurableIlmRecordCheckpoint {
Err(Error::other("durable ILM record generation is not a monotonic successor"))
}
}
/// Whether `self` is an older generation of the same immutable record
/// that can reach `terminal` through one or more valid state transitions.
/// This is deliberately broader than `validate_successor`, which remains
/// adjacent-only for receipt advancement. Terminal cleanup uses this only
/// after the exact terminal ETag and terminal receipt were committed, to
/// purge older object versions exposed by that deletion.
pub(crate) fn is_predecessor_of_terminal(&self, terminal: &Self) -> bool {
if self == terminal || self.validate_successor(terminal).is_ok() {
return true;
}
match (self, terminal) {
(
Self::TierDeleteJournal {
identity_sha256: previous_identity,
dispatch_identity_sha256: previous_dispatch,
state: Some(super::tier_sweeper::TierDeleteJournalState::Prepared),
..
},
Self::TierDeleteJournal {
identity_sha256: terminal_identity,
dispatch_identity_sha256: terminal_dispatch,
state: Some(super::tier_sweeper::TierDeleteJournalState::Committed),
..
},
) => previous_identity == terminal_identity && previous_dispatch == terminal_dispatch,
(
Self::TierDeleteDispatchManifest {
identity_sha256: previous_identity,
state: tier_delete_journal::TierDeleteDispatchManifestState::Preparing,
..
},
Self::TierDeleteDispatchManifest {
identity_sha256: terminal_identity,
state:
tier_delete_journal::TierDeleteDispatchManifestState::Aborted
| tier_delete_journal::TierDeleteDispatchManifestState::Completed,
..
},
) => previous_identity == terminal_identity,
_ => false,
}
}
}
fn transition_state_distance(
@@ -750,10 +877,19 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
if tier_delete_journal::tier_delete_journal_object_name(&entry) != path {
return Err(Error::other("tier delete journal content does not match its path"));
}
let operation_id = path
let legacy_operation_id = path
.strip_prefix(namespace.prefix)
.and_then(|suffix| suffix.strip_suffix(".json"))
.ok_or_else(|| Error::other("tier delete journal path is invalid"))?;
// Legacy v1-v5 paths already expose a 64-hex operation id and
// must remain receipt-compatible. V6 uses an operation-scoped
// nested path, so derive a fixed, path-unique receipt id instead
// of embedding slashes in the receipt locator.
let operation_id = if entry.persisted_version == 6 {
hex_sha256(path.as_bytes(), ToOwned::to_owned)
} else {
legacy_operation_id.to_string()
};
let identity_sha256 = checkpoint_hash(&(
&entry.obj_name,
&entry.version_id,
@@ -763,13 +899,29 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
entry.version_state,
&entry.source,
))?;
let dispatch_identity_sha256 = entry.dispatch.as_ref().map(checkpoint_hash).transpose()?;
(
"operation_id",
operation_id.to_string(),
operation_id,
DurableIlmRecordCheckpoint::TierDeleteJournal {
content_sha256,
identity_sha256,
committed: entry.state == super::tier_sweeper::TierDeleteJournalState::Committed,
dispatch_identity_sha256,
state: (entry.persisted_version == 6).then_some(entry.state),
},
)
}
DurableIlmRecordKind::TierDeleteDispatchManifest => {
let (operation_id, identity_sha256, state) =
tier_delete_journal::validate_tier_delete_dispatch_manifest_record(path, data)?;
(
"operation_id",
hex_sha256(operation_id.as_bytes(), ToOwned::to_owned),
DurableIlmRecordCheckpoint::TierDeleteDispatchManifest {
content_sha256,
identity_sha256,
state,
},
)
}
@@ -956,6 +1108,87 @@ mod tests {
}
}
#[test]
fn tier_delete_dispatch_manifest_namespace_validates_monotonic_branches() {
use tier_delete_journal::TierDeleteDispatchManifestState::{Aborted, Aborting, Completed, DispatchAuthorized, Preparing};
let operation_id = Uuid::new_v4();
let checkpoint = |state| {
let (path, data) = tier_delete_journal::test_tier_delete_dispatch_manifest_record(operation_id, state);
let namespace = classify_durable_ilm_record(&path)
.expect("dispatch manifest namespace should classify")
.expect("dispatch manifest should be durable");
assert_eq!(namespace, &TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE);
validate_durable_ilm_record(&path, &data)
.expect("dispatch manifest should validate")
.checkpoint
};
let preparing = checkpoint(Preparing);
let authorized = checkpoint(DispatchAuthorized);
let completed = checkpoint(Completed);
let aborting = checkpoint(Aborting);
let aborted = checkpoint(Aborted);
preparing
.validate_successor(&authorized)
.expect("Preparing may become DispatchAuthorized");
authorized
.validate_successor(&completed)
.expect("DispatchAuthorized may become Completed");
preparing.validate_successor(&aborting).expect("Preparing may enter rollback");
aborting.validate_successor(&aborted).expect("Aborting may become Aborted");
assert!(authorized.validate_successor(&aborting).is_err());
assert!(completed.validate_successor(&authorized).is_err());
assert!(aborted.validate_successor(&preparing).is_err());
}
#[test]
fn tier_delete_journal_checkpoint_binds_dispatch_and_full_state_monotonically() {
use crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared};
let checkpoint = |content: &str, dispatch: Option<&str>, state| DurableIlmRecordCheckpoint::TierDeleteJournal {
content_sha256: content.repeat(64),
identity_sha256: "i".repeat(64),
committed: state == Some(Committed),
dispatch_identity_sha256: dispatch.map(|value| value.repeat(64)),
state,
};
let prepared = checkpoint("a", Some("d"), Some(Prepared));
let dispatched = checkpoint("b", Some("d"), Some(Dispatched));
let committed = checkpoint("c", Some("d"), Some(Committed));
prepared
.validate_successor(&dispatched)
.expect("Prepared may advance to Dispatched");
dispatched
.validate_successor(&committed)
.expect("Dispatched may advance to Committed");
assert!(prepared.validate_successor(&committed).is_err());
assert!(dispatched.validate_successor(&prepared).is_err());
let rebound = checkpoint("b", Some("e"), Some(Dispatched));
assert!(dispatched.validate_successor(&rebound).is_err());
let legacy: DurableIlmRecordCheckpoint = serde_json::from_value(serde_json::json!({
"kind": "tier_delete_journal",
"content_sha256": "a".repeat(64),
"identity_sha256": "i".repeat(64),
"committed": false
}))
.expect("legacy tier-delete checkpoint should remain decodable");
legacy
.validate_successor(&prepared)
.expect("byte-identical legacy receipt may adopt the stronger v6 proof");
let changed_legacy = DurableIlmRecordCheckpoint::TierDeleteJournal {
content_sha256: "z".repeat(64),
identity_sha256: "i".repeat(64),
committed: false,
dispatch_identity_sha256: None,
state: None,
};
assert!(changed_legacy.validate_successor(&prepared).is_err());
}
#[test]
fn manual_transition_job_checkpoint_compacts_legacy_progress_compatibly() {
let options = super::super::bucket_lifecycle_ops::ManualTransitionRunOptions::default();
+2 -2
View File
@@ -34,6 +34,6 @@ pub mod tier_sweeper;
pub mod transition_transaction;
pub(crate) use durable_namespace::{
DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record,
validate_durable_ilm_record,
DurableIlmRecordCheckpoint, ILM_META_PREFIX, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE, ValidatedDurableIlmRecord,
classify_durable_ilm_record, validate_durable_ilm_record,
};
File diff suppressed because it is too large Load Diff
@@ -172,7 +172,8 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
return Err(std::io::Error::other("free-version recovery limit must be greater than zero").into());
}
let page = list_tier_free_versions(api, limit, bucket_marker.clone(), object_marker.clone(), cancel_token.clone()).await?;
let page =
list_tier_free_versions(api.clone(), limit, bucket_marker.clone(), object_marker.clone(), cancel_token.clone()).await?;
let mut stats = FreeVersionRecoveryStats {
scanned: 0,
enqueued: 0,
@@ -190,7 +191,7 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
return Err(tier_free_version_recovery_cancelled());
}
retry_cursor.visit(&oi);
if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(oi).await) {
if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(&api, oi).await) {
let (bucket_marker, object_marker) = retry_cursor.retry_markers();
stats.truncated = true;
stats.next_bucket_marker = bucket_marker;
@@ -255,6 +255,7 @@ impl ObjSweeper {
}
if del_tier {
return Some(Jentry {
persisted_version: 0,
obj_name: self.remote_object.clone(),
version_id: self.transition_version_id.clone(),
tier_name: self.transition_tier.clone(),
@@ -266,6 +267,7 @@ impl ObjSweeper {
version_state: self.transition_version_state,
state: TierDeleteJournalState::Committed,
source: None,
dispatch: None,
});
}
None
@@ -298,9 +300,19 @@ impl ObjSweeper {
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
pub(crate) enum TierDeleteJournalState {
Prepared,
Dispatched,
Committed,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub(crate) struct TierDeleteDispatchBinding {
pub(crate) operation_id: Uuid,
pub(crate) manifest_object: String,
pub(crate) journal_set_sha256: String,
pub(crate) topology_generation: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub(crate) struct TierDeleteSourceIdentity {
@@ -342,6 +354,10 @@ impl TierDeleteSourceIdentity {
#[derive(Debug, Clone)]
#[allow(unused_assignments)]
pub struct Jentry {
/// On-disk format version when decoded. Newly constructed entries use 0;
/// the encoder chooses their format from the durable ownership fields.
/// Recovery uses this value to quarantine v1-v5 without rewriting them.
pub(crate) persisted_version: u8,
pub(crate) obj_name: String,
pub(crate) version_id: String,
pub(crate) tier_name: String,
@@ -350,6 +366,23 @@ pub struct Jentry {
pub(crate) version_state: rustfs_filemeta::TransitionVersionState,
pub(crate) state: TierDeleteJournalState,
pub(crate) source: Option<TierDeleteSourceIdentity>,
pub(crate) dispatch: Option<TierDeleteDispatchBinding>,
}
impl Jentry {
/// Whether this prepared transaction is eligible to become the sole
/// cleanup owner for its transitioned source. The caller may use this to
/// decide whether to persist it, but must not set `skip_free_version`
/// until persistence succeeds.
pub(crate) fn can_replace_tier_free_version(&self) -> bool {
self.state == TierDeleteJournalState::Prepared
&& self.backend_identity.is_some()
&& self.version_state != rustfs_filemeta::TransitionVersionState::Unknown
&& self
.source
.as_ref()
.is_some_and(TierDeleteSourceIdentity::has_stable_identity)
}
}
impl ExpiryOp for Jentry {
@@ -617,6 +650,7 @@ pub fn transitioned_force_delete_journal_entry(
}
Some(Jentry {
persisted_version: 0,
obj_name: transitioned.name.clone(),
version_id: transitioned.version_id.clone(),
tier_name: transitioned.tier.clone(),
@@ -628,6 +662,7 @@ pub fn transitioned_force_delete_journal_entry(
version_state: transition_version_state,
state: TierDeleteJournalState::Committed,
source: None,
dispatch: None,
})
}
@@ -673,17 +708,73 @@ mod test {
use rustfs_s3_client::signer_error::invalid_utf8_header_error;
use super::{
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED,
RemoteDeleteBreaker, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, Jentry,
RemoteDeleteBreaker, RemoteTierDeleteOutcome, TierDeleteJournalState, TierDeleteSourceIdentity,
delete_confirmed_transition_candidate_exact_with_manager_and_identity, delete_object_from_remote_tier_idempotent,
delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error,
is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, should_record_remote_delete_failure,
transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
};
use crate::storage_api_contracts::lifecycle::TransitionedObject;
use rustfs_filemeta::TransitionVersionState;
use std::io::{Error, ErrorKind};
use std::time::{Duration, Instant};
fn stable_prepared_journal() -> Jentry {
Jentry {
persisted_version: 0,
obj_name: "remote/object".to_string(),
version_id: "remote-version".to_string(),
tier_name: "WARM".to_string(),
backend_identity: Some([7; 32]),
version_id_exact: true,
version_state: TransitionVersionState::Exact,
state: TierDeleteJournalState::Prepared,
source: Some(TierDeleteSourceIdentity {
bucket: "bucket".to_string(),
object: "object".to_string(),
version_id: Some(uuid::Uuid::new_v4().to_string()),
versioned: true,
version_suspended: false,
data_dir: None,
etag: None,
mod_time: None,
}),
dispatch: None,
}
}
#[test]
fn only_stable_prepared_journal_can_replace_tier_free_version() {
let stable = stable_prepared_journal();
assert!(stable.can_replace_tier_free_version());
let mut committed = stable.clone();
committed.state = TierDeleteJournalState::Committed;
assert!(!committed.can_replace_tier_free_version());
let mut unbound = stable.clone();
unbound.backend_identity = None;
assert!(!unbound.can_replace_tier_free_version());
let mut unknown = stable.clone();
unknown.version_state = TransitionVersionState::Unknown;
assert!(!unknown.can_replace_tier_free_version());
let mut unstable = stable;
unstable.source = Some(TierDeleteSourceIdentity {
bucket: "bucket".to_string(),
object: "object".to_string(),
version_id: None,
versioned: false,
version_suspended: false,
data_dir: None,
etag: Some("etag-only".to_string()),
mod_time: None,
});
assert!(!unstable.can_replace_tier_free_version());
}
#[test]
fn signer_header_error_detection_matches_utf8_failures() {
let err = Error::new(
@@ -49,8 +49,8 @@ use rustfs_protos::proto_gen::node_service::{
ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
ScannerPublicationLeaseResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
tier_mutation_control_service_client::TierMutationControlServiceClient,
TierMutationControlResponse, TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest,
node_service_client::NodeServiceClient, tier_mutation_control_service_client::TierMutationControlServiceClient,
};
pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
use rustfs_protos::{TierMutationRpcPhase, evict_failed_connection};
@@ -462,6 +462,31 @@ pub struct PeerTierMutationOutcome {
pub applied: bool,
}
#[derive(Debug, thiserror::Error)]
#[error("{message}")]
struct TierMutationDefinitelyRejected {
message: String,
}
fn tier_mutation_definitely_rejected_error(message: String) -> Error {
Error::other(TierMutationDefinitelyRejected { message })
}
#[cfg(test)]
pub(crate) fn test_tier_mutation_definitely_rejected_error(message: &str) -> Error {
tier_mutation_definitely_rejected_error(message.to_string())
}
pub(crate) fn tier_mutation_error_is_definitely_rejected(error: &Error) -> bool {
matches!(
error,
Error::Io(io_error)
if io_error
.get_ref()
.is_some_and(|source| source.downcast_ref::<TierMutationDefinitelyRejected>().is_some())
)
}
fn validate_tier_mutation_response_proof(
version: u32,
phase: TierMutationRpcPhase,
@@ -469,6 +494,16 @@ fn validate_tier_mutation_response_proof(
canonical_payload: &[u8],
response: &TierMutationControlResponse,
) -> Result<()> {
if response.response_proof.len() > rustfs_protos::TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE {
return Err(Error::other("peer tier mutation response proof exceeds size limit"));
}
if response
.error_info
.as_ref()
.is_some_and(|error| error.len() > rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE)
{
return Err(Error::other("peer tier mutation error response exceeds size limit"));
}
let canonical_response =
rustfs_protos::canonical_tier_mutation_rpc_response_body(rustfs_protos::TierMutationRpcResponseProofInput {
version,
@@ -479,6 +514,7 @@ fn validate_tier_mutation_response_proof(
state: response.state,
applied: response.applied,
error_info: response.error_info.as_deref(),
failure_class: response.failure_class,
})
.map_err(|_| Error::other("tier mutation response length cannot be represented"))?;
verify_tonic_rpc_response_proof(&canonical_response, &response.response_proof)
@@ -500,9 +536,9 @@ fn validate_tier_mutation_payload_len(phase: TierMutationRpcPhase, payload_len:
TierMutationRpcPhase::Commit => rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE,
TierMutationRpcPhase::Abort => {
if payload_len == 0 {
return Ok(());
return Err(Error::other("tier mutation abort payload is empty"));
}
return Err(Error::other("tier mutation abort payload must be empty"));
rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE
}
_ => return Err(Error::other("tier mutation rpc phase is unsupported")),
};
@@ -521,8 +557,29 @@ fn tier_mutation_phase_label(phase: TierMutationRpcPhase) -> &'static str {
}
}
fn tier_mutation_control_status_error(phase: TierMutationRpcPhase, status: tonic::Status) -> Error {
Error::other(format!("peer tier mutation {} RPC failed: {status}", tier_mutation_phase_label(phase)))
fn tier_mutation_control_status_error(phase: TierMutationRpcPhase, requested_version: u32, status: tonic::Status) -> Error {
let message = format!("peer tier mutation {} RPC failed: {status}", tier_mutation_phase_label(phase));
let legacy_rejection = format!("unsupported tier mutation peer protocol version: {requested_version}");
// RUSTFS_COMPAT_TODO(backlog-2097-tier-mutation-v4-error-text): retain this exact v3-server rejection classifier for mixed-version peers. Remove after every supported peer returns the signed v4 failure class.
if requested_version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
&& status.code() == tonic::Code::FailedPrecondition
&& status.message().as_bytes() == legacy_rejection.as_bytes()
{
return tier_mutation_definitely_rejected_error(message);
}
Error::other(message)
}
fn tier_mutation_failed_response_error(version: u32, failure_class: i32, error_info: Option<String>) -> Error {
let message = error_info.unwrap_or_else(|| "peer tier mutation failed without an error".to_string());
if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
&& TierMutationFailureClass::try_from(failure_class).ok() == Some(TierMutationFailureClass::PreDispatchRejected)
{
return tier_mutation_definitely_rejected_error(message);
}
// Missing/zero, unknown, and explicit Ambiguous are deliberately the same
// fail-closed result: the coordinator must include this peer in Abort.
Error::other(message)
}
impl PeerRestClient {
@@ -1315,8 +1372,12 @@ impl PeerRestClient {
.await
}
pub async fn abort_tier_mutation(&self, mutation_id: Uuid) -> Result<PeerTierMutationOutcome> {
self.tier_mutation_control(TierMutationRpcPhase::Abort, mutation_id, Bytes::new())
pub async fn abort_tier_mutation(
&self,
mutation_id: Uuid,
canonical_prepare_payload: Bytes,
) -> Result<PeerTierMutationOutcome> {
self.tier_mutation_control(TierMutationRpcPhase::Abort, mutation_id, canonical_prepare_payload)
.await
}
@@ -1350,7 +1411,7 @@ impl PeerRestClient {
client
.prepare_tier_mutation(request)
.await
.map_err(|status| tier_mutation_control_status_error(phase, status))?
.map_err(|status| tier_mutation_control_status_error(phase, version, status))?
.into_inner()
}
TierMutationRpcPhase::Commit => {
@@ -1363,7 +1424,7 @@ impl PeerRestClient {
client
.commit_tier_mutation(request)
.await
.map_err(|status| tier_mutation_control_status_error(phase, status))?
.map_err(|status| tier_mutation_control_status_error(phase, version, status))?
.into_inner()
}
TierMutationRpcPhase::Abort => {
@@ -1376,18 +1437,19 @@ impl PeerRestClient {
client
.abort_tier_mutation(request)
.await
.map_err(|status| tier_mutation_control_status_error(phase, status))?
.map_err(|status| tier_mutation_control_status_error(phase, version, status))?
.into_inner()
}
_ => return Err(Error::other("tier mutation rpc phase is unsupported")),
};
validate_tier_mutation_response_proof(version, phase, mutation_id, &canonical_payload, &response)?;
if !response.success {
return Err(Error::other(
response
.error_info
.unwrap_or_else(|| "peer tier mutation failed without an error".to_string()),
));
return Err(tier_mutation_failed_response_error(version, response.failure_class, response.error_info));
}
if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
&& response.failure_class != TierMutationFailureClass::Unspecified as i32
{
return Err(Error::other("successful peer tier mutation response carried a failure class"));
}
let state = decode_tier_mutation_peer_state(response.state)?;
Ok(PeerTierMutationOutcome {
@@ -3332,6 +3394,7 @@ mod tests {
state: i32,
applied: bool,
error_info: Option<&'a str>,
failure_class: i32,
}
fn signed_tier_mutation_response(input: TierMutationResponseFixture<'_>) -> TierMutationControlResponse {
@@ -3345,6 +3408,7 @@ mod tests {
state: input.state,
applied: input.applied,
error_info: input.error_info,
failure_class: input.failure_class,
})
.expect("small tier mutation response should encode");
let response_proof =
@@ -3355,6 +3419,7 @@ mod tests {
applied: input.applied,
error_info: input.error_info.map(str::to_string),
response_proof: response_proof.into(),
failure_class: input.failure_class,
}
}
@@ -3372,6 +3437,7 @@ mod tests {
state: TierMutationPeerState::Prepared as i32,
applied: true,
error_info: None,
failure_class: TierMutationFailureClass::Unspecified as i32,
});
validate_tier_mutation_response_proof(
rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
@@ -3396,6 +3462,10 @@ mod tests {
applied: false,
..response.clone()
},
TierMutationControlResponse {
failure_class: TierMutationFailureClass::Ambiguous as i32,
..response.clone()
},
] {
let err = validate_tier_mutation_response_proof(
rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
@@ -3419,6 +3489,44 @@ mod tests {
assert!(err.to_string().contains("invalid tier mutation response proof"));
}
#[test]
fn tier_mutation_response_rejects_oversized_proof_and_error_before_verification() {
let mutation_id = Uuid::new_v4();
let payload = b"tier-mutation-prepare";
let oversized_proof = TierMutationControlResponse {
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: None,
response_proof: vec![0; rustfs_protos::TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE + 1].into(),
failure_class: TierMutationFailureClass::Ambiguous as i32,
};
let err = validate_tier_mutation_response_proof(
rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
TierMutationRpcPhase::Prepare,
mutation_id,
payload,
&oversized_proof,
)
.expect_err("oversized proof must fail before cryptographic verification");
assert!(err.to_string().contains("response proof exceeds size limit"));
let oversized_error = TierMutationControlResponse {
response_proof: Bytes::new(),
error_info: Some("e".repeat(rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE + 1)),
..oversized_proof
};
let err = validate_tier_mutation_response_proof(
rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
TierMutationRpcPhase::Prepare,
mutation_id,
payload,
&oversized_error,
)
.expect_err("oversized error detail must fail before proof construction");
assert!(err.to_string().contains("error response exceeds size limit"));
}
#[test]
fn tier_mutation_peer_state_decode_fails_closed() {
assert_eq!(
@@ -3463,8 +3571,17 @@ mod tests {
)
.is_err()
);
validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 0).expect("empty abort payload should fit");
assert!(validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 1).is_err());
assert!(validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 0).is_err());
validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 1).expect("non-empty abort payload should fit");
validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE)
.expect("max abort payload should fit");
assert!(
validate_tier_mutation_payload_len(
TierMutationRpcPhase::Abort,
rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE + 1,
)
.is_err()
);
}
#[test]
@@ -3479,7 +3596,7 @@ mod tests {
tonic::Status::deadline_exceeded("peer tier mutation control timed out"),
tonic::Status::unavailable("peer tier mutation control unavailable"),
] {
let err = tier_mutation_control_status_error(phase, status);
let err = tier_mutation_control_status_error(phase, rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION, status);
let rendered = err.to_string();
assert!(rendered.contains(&format!("peer tier mutation {label} RPC failed")), "{rendered}");
assert!(
@@ -3493,6 +3610,61 @@ mod tests {
}
}
#[test]
fn tier_mutation_v4_to_v3_rejection_classification_requires_exact_status_and_message() {
let version = rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION;
let exact = format!("unsupported tier mutation peer protocol version: {version}");
let rejected = tier_mutation_control_status_error(
TierMutationRpcPhase::Prepare,
version,
tonic::Status::failed_precondition(exact.clone()),
);
assert!(tier_mutation_error_is_definitely_rejected(&rejected));
for status in [
tonic::Status::failed_precondition(format!("{exact}.")),
tonic::Status::failed_precondition(format!("unsupported tier mutation peer protocol version: {}", version - 1)),
tonic::Status::invalid_argument(exact.clone()),
tonic::Status::unimplemented(exact),
] {
let ambiguous = tier_mutation_control_status_error(TierMutationRpcPhase::Prepare, version, status);
assert!(
!tier_mutation_error_is_definitely_rejected(&ambiguous),
"near-text, wrong-code, and Unimplemented failures must remain ambiguous"
);
}
}
#[test]
fn tier_mutation_v4_failure_class_is_typed_and_fails_closed() {
let version = rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION;
let rejected = tier_mutation_failed_response_error(
version,
TierMutationFailureClass::PreDispatchRejected as i32,
Some("rejected".to_string()),
);
assert!(tier_mutation_error_is_definitely_rejected(&rejected));
for failure_class in [
TierMutationFailureClass::Unspecified as i32,
TierMutationFailureClass::Ambiguous as i32,
99,
] {
let ambiguous = tier_mutation_failed_response_error(version, failure_class, None);
assert!(
!tier_mutation_error_is_definitely_rejected(&ambiguous),
"missing, unknown, and explicit ambiguous classes must trigger Abort fanout"
);
}
let v3_ignores_v4_class = tier_mutation_failed_response_error(
rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
TierMutationFailureClass::PreDispatchRejected as i32,
Some("legacy failure".to_string()),
);
assert!(!tier_mutation_error_is_definitely_rejected(&v3_ignores_v4_class));
}
#[tokio::test]
async fn peer_rest_client_rejects_oversized_tier_prepare_before_dialing() {
let client = test_peer_client();
+589 -19
View File
@@ -18,7 +18,8 @@ use crate::bucket::utils::is_meta_bucketname;
use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::bucket::{
lifecycle::{
DurableIlmRecordCheckpoint, ILM_META_PREFIX, LifecycleExpiryConfigs, ValidatedDurableIlmRecord,
DurableIlmRecordCheckpoint, ILM_META_PREFIX, LifecycleExpiryConfigs, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
ValidatedDurableIlmRecord,
bucket_lifecycle_audit::LcEventSrc,
bucket_lifecycle_ops::{
LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_for_data_movement, apply_expiry_rule_in,
@@ -26,6 +27,7 @@ use crate::bucket::{
},
classify_durable_ilm_record, get_expiry_configs,
lifecycle::IlmAction,
tier_delete_journal::durable_ilm_v6_topology_generation,
validate_durable_ilm_record,
},
metadata_sys,
@@ -49,6 +51,9 @@ use crate::error::{
use crate::layout::endpoints::EndpointServerPools;
use crate::object_api::{DecommissionCapacityOptions, GetObjectReader, ObjectOptions};
use crate::runtime::sources as runtime_sources;
use crate::services::notification_sys::{
acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches, tier_delete_journal_topology_generation,
};
use crate::services::rebalance::{REBAL_META_NAME, RebalanceMeta, is_rebalance_conflicting_with_decommission};
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
use crate::storage_api_contracts::{
@@ -2384,10 +2389,12 @@ struct DecommissionDurableIlmReceipt {
id: String,
checkpoint: DurableIlmRecordCheckpoint,
terminal_checkpoint: Option<DurableIlmRecordCheckpoint>,
#[serde(default, skip_serializing_if = "Option::is_none")]
fleet_topology_generation: Option<String>,
}
impl DecommissionDurableIlmReceipt {
fn new(path: &str, record: &ValidatedDurableIlmRecord) -> Self {
fn new(path: &str, record: &ValidatedDurableIlmRecord, fleet_topology_generation: Option<String>) -> Self {
Self {
source_path: path.to_string(),
namespace: record.namespace.to_string(),
@@ -2395,6 +2402,7 @@ impl DecommissionDurableIlmReceipt {
id: record.id.clone(),
checkpoint: record.checkpoint.clone(),
terminal_checkpoint: None,
fleet_topology_generation,
}
}
@@ -2433,6 +2441,16 @@ impl DecommissionDurableIlmReceipt {
))
})?;
}
if self
.fleet_topology_generation
.as_deref()
.is_some_and(|generation| !is_sha256_checksum(generation))
{
return Err(Error::other_with_context(
"receipt fleet topology generation is invalid",
format!("source path `{}` {}", self.source_path, self.context()),
));
}
Ok(())
}
@@ -2541,6 +2559,21 @@ fn merge_decommission_durable_ilm_receipts(
id: existing.id.clone(),
checkpoint,
terminal_checkpoint,
fleet_topology_generation: match (
existing.fleet_topology_generation.as_ref(),
incoming.fleet_topology_generation.as_ref(),
) {
(Some(existing_generation), Some(incoming_generation)) if existing_generation == incoming_generation => {
Some(existing_generation.clone())
}
(None, None) => None,
_ => {
return Err(Error::other_with_context(
"durable ILM receipt fleet topology conflict",
format!("source path `{}` {}", existing.source_path, existing.context()),
));
}
},
};
merged.validate()?;
Ok(merged)
@@ -7050,6 +7083,15 @@ pub(crate) struct DecommissionCapacityOwner {
pub(crate) mutation_id: Option<uuid::Uuid>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct DecommissionDurableIlmCheckpointTarget {
pub(crate) source_pool_index: usize,
pub(crate) target_pool_index: usize,
pub(crate) capacity_owner: DecommissionCapacityOwner,
pub(crate) already_committed: bool,
pub(crate) target_etag: Option<String>,
}
impl DecommissionCapacityOwner {
pub(crate) fn apply_to(self, opts: &mut ObjectOptions) {
opts.src_pool_idx = self.source_pool_index;
@@ -7457,6 +7499,7 @@ impl DecommissionCapacityLockOrderBarrier {
Self { state }
}
#[cfg(feature = "test-util")]
pub(crate) async fn wait_until_owner_paused(&self) {
tokio::time::timeout(std::time::Duration::from_secs(30), self.state.owner_arrived.notified())
.await
@@ -7469,6 +7512,7 @@ impl DecommissionCapacityLockOrderBarrier {
.expect("external mutation should release capacity before waiting for the object namespace");
}
#[cfg(feature = "test-util")]
pub(crate) async fn wait_until_external_object_capacity_probe_acquired(&self) {
tokio::time::timeout(
std::time::Duration::from_secs(30),
@@ -7478,6 +7522,7 @@ impl DecommissionCapacityLockOrderBarrier {
.expect("external object mutation should acquire its no-active capacity probe");
}
#[cfg(feature = "test-util")]
pub(crate) async fn wait_until_external_object_commit_phase_started(&self) {
tokio::time::timeout(
std::time::Duration::from_secs(30),
@@ -7502,24 +7547,29 @@ impl DecommissionCapacityLockOrderBarrier {
.expect("external heal should attempt the target namespace lock after capacity admission");
}
#[cfg(feature = "test-util")]
pub(crate) fn release_owner(&self) {
self.state.owner_release.notify_one();
}
#[cfg(feature = "test-util")]
pub(crate) fn pause_external_object_commit_phase(&self) {
self.state.external_object_commit_phase_paused.store(true, Ordering::Release);
}
#[cfg(feature = "test-util")]
pub(crate) fn release_external_object_commit_phase(&self) {
self.state.external_object_commit_phase_release.notify_one();
}
#[cfg(feature = "test-util")]
pub(crate) fn pause_external_object_capacity_probe(&self) {
self.state
.external_object_capacity_probe_paused
.store(true, Ordering::Release);
}
#[cfg(feature = "test-util")]
pub(crate) fn release_external_object_capacity_probe(&self) {
self.state.external_object_capacity_probe_release.notify_one();
}
@@ -8315,6 +8365,122 @@ impl ECStore {
.await
}
/// Finish the capacity transaction for an identity-preserving temporary
/// replacement whose target bytes are already durably present. This is
/// the crash-recovery half of `run_decommission_capacity_temporary_mutation`:
/// it never writes the target again and only resolves a pending intent
/// owned by the exact deterministic mutation id.
pub(crate) async fn reconcile_decommission_capacity_after_equivalent_temporary_target(
&self,
owner: DecommissionCapacityOwner,
target_pool_index: usize,
expected_data_bytes: usize,
) -> Result<()> {
let mut save_guard = self.pool_meta_save_gate.lock().await;
let (pool_meta_guard, mut snapshot) = self
.acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent temporary target reconciliation failed")
.await?;
let source_pool_index = owner.source_pool_index;
let mutation_id = owner
.mutation_id
.ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target mutation identity is missing"))?;
let (target_layout, pending_physical_bytes, pending_mutation_id, already_reconciled) = {
let reservation = snapshot
.pools
.get(source_pool_index)
.and_then(|pool| pool.decommission.as_ref())
.and_then(|info| info.capacity_reservation.as_ref())
.filter(|reservation| reservation.admits_cleanup_owner(owner))
.ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target owner is stale"))?;
let target = reservation
.targets
.iter()
.find(|target| target.pool_index == target_pool_index)
.ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target allocation is missing"))?;
(
target.layout,
target.pending_physical_bytes,
target.pending_mutation_id,
target
.temporary_mutations
.iter()
.any(|mutation| mutation.mutation_id == mutation_id),
)
};
if pending_physical_bytes == 0 {
// Either the successful attempt already saved its progress, or a
// byte-neutral replacement had no inflight delta to record.
ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation fence failed")?;
return Ok(());
}
if pending_mutation_id != Some(mutation_id) {
return Err(decommission_capacity_blocked_error(
"equivalent temporary target pending intent belongs to another mutation",
));
}
if already_reconciled {
return Err(decommission_capacity_blocked_error(
"equivalent temporary target has both pending and reconciled state",
));
}
let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?;
if pending_physical_bytes < expected_target_physical_bytes {
return Err(decommission_capacity_blocked_error(
"equivalent temporary target pending capacity is smaller than the committed checkpoint",
));
}
resolve_decommission_target_pending(
&mut snapshot,
source_pool_index,
target_pool_index,
expected_target_physical_bytes,
mutation_id,
)?;
// The replacement is byte-non-growing and its exact bytes were read
// before this call, so no new physical delta is inferred on replay.
// A prior successful progress save would have taken the idempotent
// pending==0 return above.
record_decommission_target_inflight(
&mut snapshot,
source_pool_index,
target_pool_index,
0,
mutation_id,
OffsetDateTime::now_utc(),
)?;
let outcome = snapshot
.save_no_lock_armed(
self.pools.clone(),
&mut save_guard,
pool_meta_guard.lock_lost_signal(),
&[source_pool_index],
)
.await?;
ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?;
let persisted_info = outcome
.committed
.pools
.get(source_pool_index)
.and_then(|pool| pool.decommission.as_ref())
.cloned()
.ok_or_else(|| decommission_metadata_not_initialized_error("publish equivalent temporary target reconciliation"))?;
{
let mut pool_meta = self.pool_meta.write().await;
let pool_count = pool_meta.pools.len();
pool_meta.version = pool_meta.version.max(outcome.committed.version);
let info = pool_meta
.pools
.get_mut(source_pool_index)
.and_then(|pool| pool.decommission.as_mut())
.ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
info.capacity_reservation = persisted_info.capacity_reservation;
info.capacity_blocked_reason = persisted_info.capacity_blocked_reason;
}
ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?;
outcome.disarm();
Ok(())
}
pub(crate) async fn has_decommission_capacity_temporary_mutation_state(
&self,
target_pool_index: usize,
@@ -12847,6 +13013,7 @@ impl ECStore {
let receipt_path = decommission_durable_ilm_receipt_path(run_token, path, source_record.id_kind, &source_record.id);
let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
let mut proof = None::<DecommissionDurableIlmReceipt>;
let mut nonterminal_receipt_found = false;
for pool_idx in 0..self.pools.len() {
if pool_idx == source_pool_idx {
continue;
@@ -12890,23 +13057,35 @@ impl ECStore {
receipt.context()
)));
}
source_record
.checkpoint
.validate_successor(&receipt.checkpoint)
.map_err(|err| {
Error::other(format!(
"terminal durable ILM decommission receipt does not cover source at path `{path}` {}: {err}",
source_record.context()
))
})?;
if receipt.terminal_checkpoint.is_some() {
if let Some(terminal_checkpoint) = receipt.terminal_checkpoint.as_ref() {
if !source_record.checkpoint.is_predecessor_of_terminal(terminal_checkpoint) {
return Err(Error::other_with_context(
"terminal durable ILM decommission receipt does not cover source",
format!("path `{path}` {}", source_record.context()),
));
}
proof = Some(match proof {
Some(existing) => merge_decommission_durable_ilm_receipts(&existing, &receipt)?,
None => receipt,
});
} else {
source_record
.checkpoint
.validate_successor(&receipt.checkpoint)
.map_err(|err| {
Error::other_with_context(
"durable ILM decommission receipt does not cover source",
format!("path `{path}` {}: {err}", source_record.context()),
)
})?;
nonterminal_receipt_found = true;
}
}
Ok(proof)
// A terminal receipt on one target must not hide another target copy
// whose receipt was installed later and has not reached terminal yet.
// Returning no proof makes recovery advance every outstanding copy
// before source cleanup can treat the operation as complete.
if nonterminal_receipt_found { Ok(None) } else { Ok(proof) }
}
async fn verify_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> {
@@ -13050,6 +13229,7 @@ impl ECStore {
pool_idx: usize,
receipt_path: &str,
record: &ValidatedDurableIlmRecord,
fleet_topology_generation: Option<&str>,
terminal: bool,
) -> Result<bool> {
let stage = if terminal { "terminal" } else { "progress" };
@@ -13085,6 +13265,12 @@ impl ECStore {
))
})?;
Self::validate_decommission_durable_ilm_receipt_locator(receipt_path, &locator, &receipt)?;
if receipt.fleet_topology_generation.as_deref() != fleet_topology_generation {
return Err(Error::other_with_context(
"durable ILM decommission receipt fleet topology mismatch",
format!("path `{}` {}", receipt.source_path, receipt.context()),
));
}
receipt.checkpoint.validate_successor(&record.checkpoint).map_err(|err| {
Error::other(format!(
"{stage} durable ILM record generation mismatch at path `{}` {}: {err}",
@@ -13197,6 +13383,7 @@ impl ECStore {
let stage = if terminal { "terminal" } else { "progress" };
let record = validate_durable_ilm_record(path, data)
.map_err(|err| Error::other(format!("{stage} durable ILM record is invalid at path `{path}`: {err}")))?;
let fleet_topology_generation = durable_ilm_v6_topology_generation(path, data)?;
let active_source_pool_indices = active_runs.iter().map(|(pool_idx, _)| *pool_idx).collect::<Vec<_>>();
let mut terminal_target_pool_indices = Vec::new();
for (source_pool_idx, run_token) in active_runs {
@@ -13205,7 +13392,13 @@ impl ECStore {
for pool_idx in 0..self.pools.len() {
if pool_idx != source_pool_idx {
let found = self
.advance_durable_ilm_decommission_receipt(pool_idx, &receipt_path, &record, terminal)
.advance_durable_ilm_decommission_receipt(
pool_idx,
&receipt_path,
&record,
fleet_topology_generation.as_deref(),
terminal,
)
.await?;
receipt_found |= found;
if terminal
@@ -13227,6 +13420,240 @@ impl ECStore {
Ok(Some(terminal_target_pool_indices))
}
/// Resolve the exact receipt-bearing target copies on which a v6 dispatch
/// manifest may advance while a decommission reservation is active.
///
/// This is intentionally not a general capacity bypass. The target copy
/// must still be covered by the active source reservation and its durable
/// receipt, the ETag must be the caller's exact CAS generation, and the
/// replacement must be a byte-non-growing adjacent manifest checkpoint.
pub(crate) async fn decommission_durable_ilm_checkpoint_targets(
&self,
path: &str,
next_data: &[u8],
expected_etag: &str,
) -> Result<Option<Vec<DecommissionDurableIlmCheckpointTarget>>> {
let active_runs = {
let pool_meta = self.pool_meta.read().await;
let mut active_runs = Vec::new();
for (source_pool_index, pool) in pool_meta.pools.iter().enumerate() {
let Some(info) = pool
.decommission
.as_ref()
.filter(|info| info.has_decommission_state() && !info.complete)
else {
continue;
};
let Some(start_time) = info.start_time else {
continue;
};
let reservation = info.capacity_reservation.clone().ok_or_else(|| {
decommission_capacity_blocked_error(format!(
"active decommission source pool {source_pool_index} has no reservation for durable ILM checkpoint"
))
})?;
if !reservation.active() {
return Err(decommission_capacity_blocked_error(format!(
"active decommission source pool {source_pool_index} has a released reservation for durable ILM checkpoint"
)));
}
active_runs.push((
source_pool_index,
decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time),
reservation,
));
}
active_runs
};
if active_runs.is_empty() {
return Ok(None);
}
let next_record = validate_durable_ilm_record(path, next_data)?;
if next_record.namespace != TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE.name {
return Ok(None);
}
let next_fleet_topology_generation = durable_ilm_v6_topology_generation(path, next_data)?;
let mut targets = Vec::<DecommissionDurableIlmCheckpointTarget>::new();
let mut missing_source_receipts = 0usize;
let mut stale_target_etag_mismatch = false;
for (source_pool_index, run_token, reservation) in active_runs {
let receipt_path = decommission_durable_ilm_receipt_path(&run_token, path, next_record.id_kind, &next_record.id);
let mut source_receipt_found = false;
for allocation in &reservation.targets {
let receipt_data = match read_config_limited_preserve_empty(
self.pools[allocation.pool_index].clone(),
&receipt_path,
DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
)
.await
{
Ok(data) => data,
Err(err)
if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
|| is_err_object_not_found(&err)
|| is_err_version_not_found(&err) =>
{
continue;
}
Err(err) => return Err(err),
};
let receipt = DecommissionDurableIlmReceipt::decode(&receipt_data)?;
let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &receipt)?;
if receipt.source_path != path
|| receipt.namespace != next_record.namespace
|| receipt.id_kind != next_record.id_kind
|| receipt.id != next_record.id
{
return Err(Error::other_with_context(
"durable ILM checkpoint receipt identity does not authorize source",
format!("path `{path}` {}", next_record.context()),
));
}
if receipt.fleet_topology_generation != next_fleet_topology_generation {
return Err(Error::other_with_context(
"durable ILM checkpoint receipt fleet topology does not authorize source",
format!("path `{path}` {}", next_record.context()),
));
}
receipt
.checkpoint
.validate_successor(&next_record.checkpoint)
.map_err(|err| {
Error::other_with_context(
"durable ILM checkpoint receipt is not a predecessor of the requested generation",
format!(
"target pool {}, path `{path}` {}; receipt checkpoint {:?}, requested checkpoint {:?}: {err}",
allocation.pool_index,
next_record.context(),
receipt.checkpoint,
next_record.checkpoint
),
)
})?;
let (target_data, metadata) = read_config_limited_preserve_empty_with_metadata(
self.pools[allocation.pool_index].clone(),
path,
TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE.max_record_size,
)
.await?;
let target_record = validate_durable_ilm_record(path, &target_data)?;
if target_record.namespace != next_record.namespace
|| target_record.id_kind != next_record.id_kind
|| target_record.id != next_record.id
{
return Err(Error::other_with_context(
"durable ILM checkpoint target identity does not match source",
format!("path `{path}` {}", next_record.context()),
));
}
let already_committed = target_data.as_slice() == next_data;
let target_etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| {
Error::other_with_context("durable ILM checkpoint target is missing an ETag", format!("path `{path}`"))
})?;
if already_committed {
receipt
.checkpoint
.validate_successor(&target_record.checkpoint)
.map_err(|err| {
Error::other_with_context(
"durable ILM checkpoint receipt is not a predecessor of the committed target generation",
format!("path `{path}` {}: {err}", next_record.context()),
)
})?;
} else {
if target_record.checkpoint != receipt.checkpoint {
return Err(Error::other_with_context(
"durable ILM checkpoint target is not the receipt generation",
format!("path `{path}` {}", next_record.context()),
));
}
target_record
.checkpoint
.validate_successor(&next_record.checkpoint)
.map_err(|err| {
Error::other_with_context(
"durable ILM checkpoint target is not a predecessor of the requested generation",
format!("path `{path}` {}: {err}", next_record.context()),
)
})?;
if next_data.len() > target_data.len() {
return Err(decommission_capacity_blocked_error(format!(
"durable ILM checkpoint update at `{path}` grows from {} to {} bytes",
target_data.len(),
next_data.len()
)));
}
stale_target_etag_mismatch |= target_etag != expected_etag;
}
source_receipt_found = true;
if let Some(existing) = targets
.iter_mut()
.find(|target| target.target_pool_index == allocation.pool_index)
{
if existing.already_committed != already_committed {
return Err(Error::other_with_context(
"durable ILM checkpoint target state changed during authorization",
format!("path `{path}`"),
));
}
continue;
}
let base_owner = DecommissionCapacityOwner {
source_pool_index,
operation_id: reservation.operation_id,
generation: reservation.generation,
owner_nonce: reservation.owner_nonce,
mutation_id: None,
};
let mutation_id = decommission_capacity_mutation_id(
base_owner,
RUSTFS_META_BUCKET,
path,
Some(next_record.checkpoint.content_sha256()),
false,
None,
);
targets.push(DecommissionDurableIlmCheckpointTarget {
source_pool_index,
target_pool_index: allocation.pool_index,
capacity_owner: base_owner.with_mutation_id(mutation_id),
already_committed,
target_etag: Some(target_etag),
});
}
if !source_receipt_found {
missing_source_receipts = missing_source_receipts.saturating_add(1);
}
}
if missing_source_receipts > 0 {
return Err(decommission_capacity_blocked_error(format!(
"durable ILM checkpoint at `{path}` is missing receipt coverage for {missing_source_receipts} active source(s)"
)));
}
if targets.is_empty() {
return Err(decommission_capacity_blocked_error(format!(
"durable ILM checkpoint at `{path}` has no receipt-bearing reservation target"
)));
}
if stale_target_etag_mismatch && !targets.iter().any(|target| target.already_committed) {
return Err(Error::PreconditionFailed);
}
// After a partial multi-target commit, an aggregate read may return
// the ETag of the already-advanced target while another authorized
// target still has the predecessor ETag. The exact committed bytes
// plus every target's receipt/checkpoint proof authorize repairing
// that predecessor with its own target-local CAS. Without an exact
// committed target, retain the caller-ETag requirement above.
targets.sort_unstable_by_key(|target| target.target_pool_index);
Ok(Some(targets))
}
pub(crate) async fn record_durable_ilm_decommission_progress(&self, path: &str, data: &[u8]) -> Result<()> {
self.advance_durable_ilm_decommission_receipts(path, data, false)
.await
@@ -13248,6 +13675,66 @@ impl ECStore {
self.advance_durable_ilm_decommission_receipts(path, data, true).await
}
/// Return true only when `data` is the exact copy still owned by an active
/// decommission source and a target-side terminal receipt authorizes that
/// source's later verified cleanup. Lifecycle recovery may then regard the
/// logical record as terminal without deleting the source checkpoint.
pub(crate) async fn durable_ilm_terminal_receipt_covers_active_source(&self, path: &str, data: &[u8]) -> Result<bool> {
let namespace = classify_durable_ilm_record(path)?
.ok_or_else(|| Error::other_with_context("path is not a durable ILM record", format!("path `{path}`")))?;
let source_record = validate_durable_ilm_record(path, data)?;
let active_runs = {
let pool_meta = self.pool_meta.read().await;
pool_meta
.pools
.iter()
.enumerate()
.filter_map(|(source_pool_index, pool)| {
pool.decommission
.as_ref()
.filter(|info| info.has_decommission_state() && !info.complete)
.and_then(|info| info.start_time)
.map(|start_time| {
(source_pool_index, decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time))
})
})
.collect::<Vec<_>>()
};
let mut covered_active_source = false;
for (source_pool_index, run_token) in active_runs {
let source_data =
match read_config_limited_preserve_empty(self.pools[source_pool_index].clone(), path, namespace.max_record_size)
.await
{
Ok(source_data) => source_data,
Err(err)
if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
|| is_err_object_not_found(&err)
|| is_err_version_not_found(&err) =>
{
continue;
}
Err(err) => return Err(err),
};
if source_data.as_slice() != data {
continue;
}
if self
.load_decommission_durable_ilm_terminal_receipt_for_run(source_pool_index, path, &source_record, &run_token)
.await?
.is_some()
{
covered_active_source = true;
} else {
// Every active source that still stores this exact generation
// needs complete terminal receipt coverage. One covered source
// cannot authorize lifecycle recovery to skip another.
return Ok(false);
}
}
Ok(covered_active_source)
}
async fn cleanup_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> {
for (pool_idx, receipt_path) in self.list_decommission_durable_ilm_receipts(source_pool_idx).await? {
match delete_config(self.pools[pool_idx].clone(), &receipt_path).await {
@@ -13316,12 +13803,20 @@ impl ECStore {
.map_err(|err| Error::other(format!("failed to read source durable ILM record at path `{path}`: {err}")))?;
let source_record = validate_durable_ilm_record(path, &source)
.map_err(|err| Error::other(format!("source durable ILM record is invalid at path `{path}`: {err}")))?;
let source_fleet_topology_generation = durable_ilm_v6_topology_generation(path, &source)?;
let target = self
.load_decommissioned_durable_ilm_target(source_pool_idx, path, namespace.max_record_size, &source_record.context())
.await?;
let manifest_receipt = if let Some((target_pool_idx, target)) = target {
let target_record = validate_decommission_durable_ilm_copy(path, &source_record, &target)?;
let receipt = DecommissionDurableIlmReceipt::new(path, &target_record);
let target_fleet_topology_generation = durable_ilm_v6_topology_generation(path, &target)?;
if target_fleet_topology_generation != source_fleet_topology_generation {
return Err(Error::other_with_context(
"target durable ILM fleet topology generation differs from source",
format!("path `{path}` {}", source_record.context()),
));
}
let receipt = DecommissionDurableIlmReceipt::new(path, &target_record, target_fleet_topology_generation);
self.persist_decommission_durable_ilm_receipt_for_run(target_pool_idx, &receipt, run_token)
.await?;
receipt
@@ -13335,9 +13830,34 @@ impl ECStore {
))
})?
};
if manifest_receipt.fleet_topology_generation != source_fleet_topology_generation {
return Err(Error::other_with_context(
"terminal durable ILM receipt fleet topology generation does not cover source",
format!("path `{path}` {}", source_record.context()),
));
}
let fleet_proof = if let Some(expected_generation) = source_fleet_topology_generation.as_deref() {
let proof = acquire_tier_delete_journal_fleet_proof()
.ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable for source cleanup"))?;
if tier_delete_journal_topology_generation(&proof) != expected_generation
|| !tier_delete_journal_fleet_proof_matches(&proof)
{
return Err(Error::other("tier delete journal v6 fleet generation changed before source cleanup"));
}
Some(proof)
} else {
None
};
self.persist_decommission_durable_ilm_receipt_for_run(source_pool_idx, &manifest_receipt, run_token)
.await?;
if fleet_proof
.as_ref()
.is_some_and(|proof| !tier_delete_journal_fleet_proof_matches(proof))
{
return Err(Error::other("tier delete journal v6 fleet proof changed before source cleanup"));
}
let cleanup_result = data_movement::cleanup_source_entry_if_unchanged(
source_set,
RUSTFS_META_BUCKET,
@@ -13354,7 +13874,16 @@ impl ECStore {
source_record.context()
))
});
resolve_decommission_entry_cleanup_delete_result(cleanup_result, RUSTFS_META_BUCKET, path)
let cleanup_result = resolve_decommission_entry_cleanup_delete_result(cleanup_result, RUSTFS_META_BUCKET, path);
if fleet_proof
.as_ref()
.is_some_and(|proof| !tier_delete_journal_fleet_proof_matches(proof))
{
return Err(Error::other(
"tier delete journal v6 fleet proof changed during source cleanup; exact source outcome requires verification",
));
}
cleanup_result
}
#[cfg(all(test, feature = "test-util"))]
@@ -13390,7 +13919,26 @@ impl ECStore {
record: &ValidatedDurableIlmRecord,
terminal: bool,
) -> Result<String> {
let mut receipt = DecommissionDurableIlmReceipt::new(source_path, record);
let fleet_topology_generation = match read_config_limited_preserve_empty(
self.pools[target_pool_idx].clone(),
source_path,
classify_durable_ilm_record(source_path)?
.ok_or_else(|| Error::other_with_context("path is not a durable ILM record", format!("path `{source_path}`")))?
.max_record_size,
)
.await
{
Ok(target_data) => durable_ilm_v6_topology_generation(source_path, &target_data)?,
Err(err)
if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
|| is_err_object_not_found(&err)
|| is_err_version_not_found(&err) =>
{
None
}
Err(err) => return Err(err),
};
let mut receipt = DecommissionDurableIlmReceipt::new(source_path, record, fleet_topology_generation);
if terminal {
receipt.terminal_checkpoint = Some(record.checkpoint.clone());
}
@@ -17361,11 +17909,15 @@ mod pools_tests {
content_sha256: "b".repeat(64),
identity_sha256: "c".repeat(64),
committed: false,
dispatch_identity_sha256: None,
state: None,
};
let terminal_checkpoint = DurableIlmRecordCheckpoint::TierDeleteJournal {
content_sha256: "d".repeat(64),
identity_sha256: "c".repeat(64),
committed: true,
dispatch_identity_sha256: None,
state: None,
};
let incoming = DecommissionDurableIlmReceipt {
source_path,
@@ -17374,6 +17926,7 @@ mod pools_tests {
id: operation_id,
checkpoint: checkpoint.clone(),
terminal_checkpoint: None,
fleet_topology_generation: None,
};
let existing = DecommissionDurableIlmReceipt {
terminal_checkpoint: Some(terminal_checkpoint.clone()),
@@ -17384,7 +17937,24 @@ mod pools_tests {
.expect("retry receipt must merge with a terminal receipt");
assert_eq!(merged.checkpoint, checkpoint);
assert_eq!(merged.terminal_checkpoint, Some(terminal_checkpoint));
assert_eq!(merged.terminal_checkpoint, Some(terminal_checkpoint.clone()));
let topology_bound = DecommissionDurableIlmReceipt {
fleet_topology_generation: Some("e".repeat(64)),
..incoming
};
let mixed_error = merge_decommission_durable_ilm_receipts(&existing, &topology_bound)
.expect_err("a topology-bound v6 receipt must not mask an unbound receipt")
.to_string();
assert!(mixed_error.contains("fleet topology conflict"));
let topology_existing = DecommissionDurableIlmReceipt {
terminal_checkpoint: Some(terminal_checkpoint),
..topology_bound.clone()
};
let topology_merged = merge_decommission_durable_ilm_receipts(&topology_existing, &topology_bound)
.expect("receipts bound to the same fleet topology should merge");
assert_eq!(topology_merged.fleet_topology_generation, Some("e".repeat(64)));
}
#[test]
@@ -17409,7 +17979,7 @@ mod pools_tests {
assert!(job_bytes.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE);
let record = validate_durable_ilm_record(&path, &job_bytes).expect("large manual job should validate");
let expected_checkpoint = record.checkpoint.clone();
let mut receipt = DecommissionDurableIlmReceipt::new(&path, &record);
let mut receipt = DecommissionDurableIlmReceipt::new(&path, &record, None);
receipt.terminal_checkpoint = Some(record.checkpoint);
let encoded = receipt.encode().expect("bounded progress proof should fit the receipt limit");
+16 -2
View File
@@ -37,8 +37,9 @@ use crate::{
runtime::instance::{InstanceContext, bootstrap_ctx},
runtime::sources as runtime_sources,
set_disk::{PreparedGetObjectMetadata, SetDisks},
store::init_format::{
check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum,
store::{
RemoteTuplePublicationFence,
init_format::{check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum},
},
};
use futures::{
@@ -625,6 +626,19 @@ impl Sets {
.put_object_with_old_current_size(bucket, object, data, opts)
.await
}
pub(crate) async fn put_object_with_old_current_size_for_data_movement(
&self,
bucket: &str,
object: &str,
data: &mut PutObjReader,
opts: &ObjectOptions,
publication_fence: RemoteTuplePublicationFence,
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
self.get_disks_by_key(object)
.put_object_with_old_current_size_for_data_movement(bucket, object, data, opts, publication_fence)
.await
}
}
#[async_trait::async_trait]
+49 -21
View File
@@ -27,7 +27,7 @@ use crate::storage_api_contracts::{
namespace::NamespaceLocking as _,
object::{HTTPPreconditions, ObjectOperations as _},
};
use crate::store::{ECStore, ObjectLockDiagGuard, SourceCleanupMutationFence};
use crate::store::{DecommissionFixedReadAnchor, ECStore, SourceCleanupMutationFence};
use bytes::Bytes;
use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
@@ -161,7 +161,7 @@ pub fn mark_multipart_upload_completed(flag: &Arc<AtomicBool>) {
flag.store(false, Ordering::Relaxed);
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
struct DataMovementMultipartAbortBarrierState {
bucket: String,
object: String,
@@ -169,17 +169,17 @@ struct DataMovementMultipartAbortBarrierState {
release: tokio::sync::Notify,
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pub(crate) struct DataMovementMultipartAbortBarrier {
state: Arc<DataMovementMultipartAbortBarrierState>,
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
static DATA_MOVEMENT_MULTIPART_ABORT_BARRIER: std::sync::OnceLock<
std::sync::Mutex<Option<Arc<DataMovementMultipartAbortBarrierState>>>,
> = std::sync::OnceLock::new();
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
impl DataMovementMultipartAbortBarrier {
pub(crate) fn install(bucket: &str, object: &str) -> Self {
let state = Arc::new(DataMovementMultipartAbortBarrierState {
@@ -204,7 +204,7 @@ impl DataMovementMultipartAbortBarrier {
}
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
impl Drop for DataMovementMultipartAbortBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
@@ -218,7 +218,7 @@ impl Drop for DataMovementMultipartAbortBarrier {
}
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
async fn pause_data_movement_multipart_before_abort(bucket: &str, object: &str) {
let barrier = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
@@ -1518,7 +1518,7 @@ pub(crate) async fn migrate_decommission_object(
capacity_owner: Option<DecommissionCapacityOwner>,
) -> Result<()> {
let source = rd.object_info.clone();
let _mutation_fence = store
let mutation_fence = store
.acquire_decommission_object_mutation_fence(&bucket, &source.name)
.await?;
let current = find_data_movement_target_info(store.as_ref(), pool_idx, &bucket, &source)
@@ -1537,7 +1537,7 @@ pub(crate) async fn migrate_decommission_object(
op_label,
None,
capacity_owner,
Some(&_mutation_fence),
Some(mutation_fence),
)
.await
}
@@ -1588,8 +1588,9 @@ async fn migrate_object_inner(
op_label: &str,
lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
capacity_owner: Option<DecommissionCapacityOwner>,
mutation_fence: Option<&ObjectLockDiagGuard>,
mutation_fence: Option<DecommissionFixedReadAnchor>,
) -> Result<()> {
let mut mutation_fence = mutation_fence;
let object_info = rd.object_info.clone();
let capacity_owner = capacity_owner.map(|owner| {
let version_id = object_info.version_id.map(|version_id| version_id.to_string());
@@ -1605,6 +1606,13 @@ async fn migrate_object_inner(
});
owner.with_mutation_id(mutation_id)
});
// Capture the exact source/tier identity before any client-paced read, but
// defer both the tier lease and source/target write locks to the final
// publication. Decommission already owns main's fixed-domain mutation
// fence, so reacquiring that domain as a write lock would self-deadlock.
let remote_tuple_publication_fence = store
.acquire_remote_tuple_publication_fence(&bucket, pool_idx, &object_info, false)
.await?;
let has_part_checksums = object_info
.parts
.iter()
@@ -1656,8 +1664,8 @@ async fn migrate_object_inner(
}
let mut cleanup_opts =
data_movement_abort_opts(pool_idx, source_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
if let Some(fence) = mutation_fence {
fence.add_namespace_lock_fence(&mut cleanup_opts);
if let Some(anchor) = mutation_fence.as_ref() {
anchor.guard().add_namespace_lock_fence(&mut cleanup_opts);
}
if let Some(fence) = multipart_mutation_fence.as_ref() {
fence.add_namespace_lock_fence(&mut cleanup_opts);
@@ -1684,7 +1692,12 @@ async fn migrate_object_inner(
}
}
let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
.handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
.handle_new_multipart_upload_with_pool_idx(
&bucket,
&object_info.name,
&new_multipart_opts,
mutation_fence.as_ref().map(DecommissionFixedReadAnchor::guard),
)
.await
{
Ok(res) => res,
@@ -1797,15 +1810,20 @@ async fn migrate_object_inner(
if let Some(fence) = multipart_mutation_fence.as_ref() {
fence.add_namespace_lock_fence(&mut complete_multipart_opts);
}
let remote_tuple_publication_fence = match mutation_fence.take() {
Some(anchor) => remote_tuple_publication_fence.under_fixed_read_anchor(anchor)?,
None => remote_tuple_publication_fence,
};
if let Err(err) = store
.clone()
.complete_multipart_upload_for_data_movement(
(target_pool_idx, mutation_fence),
.complete_multipart_upload_for_data_movement_with_publication_fence(
target_pool_idx,
&bucket,
&object_info.name,
&res.upload_id,
parts,
&complete_multipart_opts,
remote_tuple_publication_fence,
)
.await
{
@@ -1849,8 +1867,8 @@ async fn migrate_object_inner(
if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) {
let mut abort_opts =
data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
if let Some(fence) = mutation_fence {
fence.add_namespace_lock_fence(&mut abort_opts);
if let Some(anchor) = mutation_fence.as_ref() {
anchor.guard().add_namespace_lock_fence(&mut abort_opts);
}
if let Some(fence) = multipart_mutation_fence.as_ref() {
fence.add_namespace_lock_fence(&mut abort_opts);
@@ -1923,12 +1941,12 @@ async fn migrate_object_inner(
if let Err(primary_err) = multipart_result {
if should_abort_multipart_upload(&abort_multipart_flag) {
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pause_data_movement_multipart_before_abort(&bucket, &object_info.name).await;
let mut abort_opts =
data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
if let Some(fence) = mutation_fence {
fence.add_namespace_lock_fence(&mut abort_opts);
if let Some(anchor) = mutation_fence.as_ref() {
anchor.guard().add_namespace_lock_fence(&mut abort_opts);
}
if let Some(fence) = multipart_mutation_fence.as_ref() {
fence.add_namespace_lock_fence(&mut abort_opts);
@@ -1982,8 +2000,18 @@ async fn migrate_object_inner(
if let Some(signal) = lock_lost_signal {
put_opts.add_namespace_lock_lost_signal(signal);
}
let remote_tuple_publication_fence = match mutation_fence.take() {
Some(anchor) => remote_tuple_publication_fence.under_fixed_read_anchor(anchor)?,
None => remote_tuple_publication_fence,
};
let (target_pool_idx, put_result) = store
.put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence)
.put_object_for_data_movement_with_publication_fence(
&bucket,
&object_info.name,
&mut data,
&put_opts,
remote_tuple_publication_fence,
)
.await
.map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?;
if let Err(err) = put_result {
@@ -278,3 +278,17 @@ fn reduce_errs_buckets_identical_other_messages_together() {
assert_eq!(count, 3);
assert_eq!(err, Some(DiskError::other("can not get client")));
}
#[test]
fn stable_io_context_buckets_by_cause_and_preserves_diagnostic_source() {
let first = StorageError::other_with_context("tier mutation intent changed", "mutation-a");
let second = StorageError::other_with_context("tier mutation intent changed", "mutation-b");
assert_eq!(first, second, "diagnostic identity must not split quorum buckets");
let StorageError::Io(io_error) = first else {
panic!("stable context must remain an io error");
};
assert_eq!(io_error.to_string(), "tier mutation intent changed");
let context = io_error.get_ref().expect("stable context must remain downcastable");
assert_eq!(context.source().expect("diagnostic source must be retained").to_string(), "mutation-a");
}
+37
View File
@@ -23,6 +23,36 @@ use s3s::S3ErrorCode;
pub type Error = StorageError;
pub type Result<T> = core::result::Result<T, Error>;
/// Keeps high-cardinality diagnostic detail in the error source while making
/// the rendered `io::Error` stable for quorum aggregation.
#[derive(Debug)]
struct StableIoContextError {
message: &'static str,
source: Box<dyn std::error::Error + Send + Sync>,
}
impl std::fmt::Display for StableIoContextError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(self.message)
}
}
impl std::error::Error for StableIoContextError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(self.source.as_ref())
}
}
pub(crate) fn stable_io_error<E>(message: &'static str, source: E) -> std::io::Error
where
E: Into<Box<dyn std::error::Error + Send + Sync>>,
{
std::io::Error::other(StableIoContextError {
message,
source: source.into(),
})
}
/// Storage layer error type covering disk, volume, bucket, object, multipart,
/// erasure-coding, and operational error conditions.
///
@@ -264,6 +294,13 @@ impl StorageError {
StorageError::Io(std::io::Error::other(error))
}
pub(crate) fn other_with_context<E>(message: &'static str, source: E) -> Self
where
E: Into<Box<dyn std::error::Error + Send + Sync>>,
{
StorageError::Io(stable_io_error(message, source))
}
pub fn is_not_found(&self) -> bool {
matches!(
self,
+9 -15
View File
@@ -301,36 +301,23 @@ pub enum LifecycleDeleteAllPhase {
#[doc(hidden)]
#[derive(Default)]
pub struct LifecycleDeleteAllJournalState {
prepared: HashMap<String, crate::bucket::lifecycle::tier_sweeper::Jentry>,
mutation_started: bool,
}
impl Debug for LifecycleDeleteAllJournalState {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("LifecycleDeleteAllJournalState")
.field("prepared_count", &self.prepared.len())
.field("mutation_started", &self.mutation_started)
.finish()
}
}
impl LifecycleDeleteAllJournalState {
pub(crate) fn contains(&self, name: &str) -> bool {
self.prepared.contains_key(name)
}
pub(crate) fn insert(&mut self, name: String, entry: crate::bucket::lifecycle::tier_sweeper::Jentry) {
self.prepared.insert(name, entry);
}
pub(crate) fn prepared_entries(&self) -> Vec<crate::bucket::lifecycle::tier_sweeper::Jentry> {
self.prepared.values().cloned().collect()
}
pub(crate) fn mark_mutation_started(&mut self) {
self.mutation_started = true;
}
#[cfg(test)]
pub(crate) fn mutation_started(&self) -> bool {
self.mutation_started
}
@@ -706,6 +693,12 @@ pub struct ObjectOptions {
pub lifecycle_delete_all: Option<LifecycleDeleteAllRequest>,
#[doc(hidden)]
pub lifecycle_delete_all_journal: Option<Arc<parking_lot::Mutex<LifecycleDeleteAllJournalState>>>,
/// Whole-operation authorization created only by consuming a validated
/// v6 dispatch-manifest permit. Clones share the authorization, not the
/// one-shot permit itself.
#[doc(hidden)]
pub tier_delete_dispatch_authorization:
Option<crate::bucket::lifecycle::tier_delete_journal::TierDeleteDispatchAuthorization>,
/// RustFS-only compare-and-set condition checked under the object write lock.
pub expected_current_version_id: Option<String>,
/// Persisted bucket incarnation observed before authorization.
@@ -847,6 +840,7 @@ impl std::fmt::Debug for ObjectOptions {
.field("version_id", &self.version_id.is_some())
.field("lifecycle_delete_all", &self.lifecycle_delete_all.is_some())
.field("lifecycle_delete_all_journal", &self.lifecycle_delete_all_journal.is_some())
.field("tier_delete_dispatch_authorization", &self.tier_delete_dispatch_authorization.is_some())
.field("expected_current_version_id", &self.expected_current_version_id.is_some())
.field("expected_bucket_incarnation_id", &self.expected_bucket_incarnation_id)
.field("no_lock", &self.no_lock)
@@ -972,7 +966,7 @@ impl ObjectOptions {
}
#[cfg(test)]
pub(crate) fn add_namespace_lock_fence_for_test(&mut self, fence: &NamespaceLockFence) {
pub(crate) fn add_namespace_lock_fence(&mut self, fence: &NamespaceLockFence) {
self.namespace_lock_fence
.get_or_insert_with(NamespaceLockFence::new)
.extend(fence);
+581 -74
View File
@@ -29,10 +29,14 @@ use rustfs_madmin::metrics::RealtimeMetrics;
use rustfs_madmin::net::NetInfo;
use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
use rustfs_utils::XHost;
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
use std::future::Future;
use std::hash::{Hash, Hasher};
use std::sync::{Arc, Mutex, OnceLock};
use std::sync::{
Arc, Mutex, OnceLock,
atomic::{AtomicBool, AtomicUsize, Ordering},
};
use std::time::{Duration, Instant, SystemTime};
use tokio::time::{sleep, timeout};
use tokio_util::sync::CancellationToken;
@@ -52,6 +56,20 @@ const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
const TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION: u32 = 3;
type CrossPoolFencePolicyResult = Result<BTreeMap<String, Uuid>>;
fn cross_pool_fence_policy_results(
peer_epochs: BTreeMap<String, Uuid>,
minimum_version: u32,
) -> (CrossPoolFencePolicyResult, CrossPoolFencePolicyResult) {
let journal_result = if minimum_version >= TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION {
Ok(peer_epochs.clone())
} else {
Err(Error::other("tier delete journal v6 policy capability version is unsupported"))
};
(Ok(peer_epochs), journal_result)
}
#[derive(Clone, Debug)]
pub struct ScannerPublicationLeaseGrant {
@@ -107,15 +125,91 @@ struct FleetCapabilityProof {
topology_fingerprint: String,
peer_epochs: Arc<BTreeMap<String, Uuid>>,
expires_at: Instant,
generation: Arc<FleetCapabilityProofGeneration>,
}
impl FleetCapabilityProof {
fn new(topology_fingerprint: String, peer_epochs: Arc<BTreeMap<String, Uuid>>, expires_at: Instant) -> Self {
Self {
topology_fingerprint,
peer_epochs,
expires_at,
generation: FleetCapabilityProofGeneration::fresh(),
}
}
fn token(&self) -> FleetCapabilityProofToken {
FleetCapabilityProofToken {
topology_fingerprint: self.topology_fingerprint.clone(),
peer_epochs: self.peer_epochs.clone(),
}
}
#[cfg(any(test, feature = "test-util"))]
fn with_fresh_generation(&self) -> Self {
Self::new(self.topology_fingerprint.clone(), Arc::clone(&self.peer_epochs), self.expires_at)
}
}
/// Admission generation for effects that must not straddle a fleet-proof
/// replacement. Revocation is deliberately non-blocking: it closes admission
/// immediately, while the proof slot withholds the successor generation until
/// every admitted operation has drained.
#[derive(Default)]
struct FleetCapabilityProofGeneration {
accepting: AtomicBool,
active: AtomicUsize,
}
impl FleetCapabilityProofGeneration {
fn fresh() -> Arc<Self> {
Arc::new(Self {
accepting: AtomicBool::new(true),
active: AtomicUsize::new(0),
})
}
fn try_acquire(self: &Arc<Self>) -> Option<FleetCapabilityProofPermit> {
if !self.accepting.load(Ordering::Acquire) {
return None;
}
self.active.fetch_add(1, Ordering::AcqRel);
if self.accepting.load(Ordering::Acquire) {
Some(FleetCapabilityProofPermit {
generation: Arc::clone(self),
})
} else {
self.release();
None
}
}
fn revoke(&self) {
self.accepting.store(false, Ordering::Release);
}
fn is_accepting(&self) -> bool {
self.accepting.load(Ordering::Acquire)
}
fn is_drained(&self) -> bool {
self.active.load(Ordering::Acquire) == 0
}
fn release(&self) {
let previous = self.active.fetch_sub(1, Ordering::AcqRel);
debug_assert!(previous > 0, "fleet capability permit count underflow");
}
}
struct FleetCapabilityProofPermit {
generation: Arc<FleetCapabilityProofGeneration>,
}
impl Drop for FleetCapabilityProofPermit {
fn drop(&mut self) {
self.generation.release();
}
}
#[derive(Clone, PartialEq, Eq)]
@@ -127,6 +221,7 @@ struct FleetCapabilityProofToken {
#[derive(Default)]
struct FleetCapabilityProofState {
proof: Option<FleetCapabilityProof>,
draining_generation: Option<Arc<FleetCapabilityProofGeneration>>,
topology_conflict: bool,
}
@@ -136,8 +231,17 @@ pub(crate) struct RemoteVersionStateFleetProofToken(FleetCapabilityProofToken);
#[derive(Clone, PartialEq, Eq)]
pub struct CrossPoolFenceFleetProofToken(FleetCapabilityProofToken);
/// A point-in-time proof that every current storage member implements the v6
/// dispatch-manifest policy. It intentionally has no `Clone` implementation:
/// one acquisition authorizes one manifest construction attempt.
pub(crate) struct TierDeleteJournalFleetProofToken {
token: FleetCapabilityProofToken,
_permit: FleetCapabilityProofPermit,
}
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
static TIER_DELETE_JOURNAL_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
@@ -148,8 +252,35 @@ fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCa
REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
}
fn replace_fleet_capability_proof(slot: &std::sync::RwLock<FleetCapabilityProofState>, proof: Option<FleetCapabilityProof>) {
slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
fn tier_delete_journal_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
TIER_DELETE_JOURNAL_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
}
fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) {
if let Some(proof) = state.proof.take() {
proof.generation.revoke();
if !proof.generation.is_drained() {
state.draining_generation = Some(proof.generation);
}
}
if state
.draining_generation
.as_ref()
.is_some_and(|generation| generation.is_drained())
{
state.draining_generation = None;
}
}
fn revoke_fleet_capability_proof(slot: &std::sync::RwLock<FleetCapabilityProofState>) {
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
revoke_fleet_capability_proof_state(&mut state);
}
fn mark_fleet_capability_topology_conflict(slot: &std::sync::RwLock<FleetCapabilityProofState>) {
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
state.topology_conflict = true;
revoke_fleet_capability_proof_state(&mut state);
}
fn publish_fleet_capability_probe_result(
@@ -161,21 +292,42 @@ fn publish_fleet_capability_probe_result(
match result {
Ok(peer_epochs) => {
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
let peer_epochs = state
if let Some(current) = state
.proof
.as_ref()
.as_mut()
.filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
.map(|proof| Arc::clone(&proof.peer_epochs))
.unwrap_or_else(|| Arc::new(peer_epochs));
state.proof = Some(FleetCapabilityProof {
topology_fingerprint: topology_fingerprint.to_string(),
peer_epochs,
expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
});
{
current.expires_at = observed_at + REMOTE_VERSION_STATE_PROOF_TTL;
return None;
}
if let Some(previous) = state.proof.take() {
previous.generation.revoke();
if !previous.generation.is_drained() {
state.draining_generation = Some(previous.generation);
}
}
if state
.draining_generation
.as_ref()
.is_some_and(|generation| generation.is_drained())
{
state.draining_generation = None;
}
if state.draining_generation.is_some() {
return Some(Error::other(
"fleet capability proof successor waits for the previous generation to drain",
));
}
state.proof = Some(FleetCapabilityProof::new(
topology_fingerprint.to_string(),
Arc::new(peer_epochs),
observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
));
None
}
Err(err) => {
replace_fleet_capability_proof(slot, None);
revoke_fleet_capability_proof(slot);
Some(err)
}
}
@@ -216,7 +368,72 @@ pub fn cross_pool_fence_fleet_proof_matches(proof: &CrossPoolFenceFleetProofToke
fleet_capability_proof_matches(cross_pool_fence_fleet_proof_slot(), &proof.0)
}
#[cfg(test)]
pub(crate) fn acquire_tier_delete_journal_fleet_proof() -> Option<TierDeleteJournalFleetProofToken> {
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
let state = tier_delete_journal_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
acquire_tier_delete_journal_fleet_proof_from(&state, expected_topology, Instant::now())
}
fn acquire_tier_delete_journal_fleet_proof_from(
state: &FleetCapabilityProofState,
expected_topology: &str,
now: Instant,
) -> Option<TierDeleteJournalFleetProofToken> {
let token = acquire_fleet_capability_proof_from(state, expected_topology, now)?;
let permit = state.proof.as_ref()?.generation.try_acquire()?;
Some(TierDeleteJournalFleetProofToken { token, _permit: permit })
}
pub(crate) fn tier_delete_journal_fleet_proof_matches(proof: &TierDeleteJournalFleetProofToken) -> bool {
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
return false;
};
let state = tier_delete_journal_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
tier_delete_journal_fleet_proof_matches_at(&state, proof, expected_topology, Instant::now())
}
fn tier_delete_journal_fleet_proof_matches_at(
state: &FleetCapabilityProofState,
proof: &TierDeleteJournalFleetProofToken,
expected_topology: &str,
now: Instant,
) -> bool {
proof._permit.generation.is_accepting()
&& fleet_capability_proof_matches_at(state, &proof.token, expected_topology, now)
&& state
.proof
.as_ref()
.is_some_and(|current| Arc::ptr_eq(&current.generation, &proof._permit.generation))
}
pub(crate) fn tier_delete_journal_topology_generation(proof: &TierDeleteJournalFleetProofToken) -> String {
stable_tier_delete_journal_topology_generation(&proof.token.topology_fingerprint)
}
#[cfg(all(test, feature = "test-util"))]
pub(crate) fn tier_delete_journal_fleet_proof_has_inflight_for_test() -> bool {
let state = tier_delete_journal_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.proof.as_ref().is_some_and(|proof| !proof.generation.is_drained())
|| state
.draining_generation
.as_ref()
.is_some_and(|generation| !generation.is_drained())
}
fn stable_tier_delete_journal_topology_generation(topology_fingerprint: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(b"rustfs-tier-delete-journal-topology-v1\0");
hasher.update(topology_fingerprint.as_bytes());
rustfs_utils::crypto::hex(hasher.finalize().as_slice())
}
#[cfg(any(test, feature = "test-util"))]
pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
let topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY
.get()
@@ -226,18 +443,39 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
let mut state = cross_pool_fence_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = Instant::now();
let proof = if !state.topology_conflict && fleet_capability_proof_valid_at(state.proof.as_ref(), &topology, now) {
state.proof.clone()
} else {
Some(FleetCapabilityProof::new(
topology,
Arc::new(BTreeMap::new()),
now + Duration::from_secs(60 * 60),
))
};
state.topology_conflict = false;
state.proof = Some(FleetCapabilityProof {
topology_fingerprint: topology,
peer_epochs: Arc::new(BTreeMap::new()),
expires_at: Instant::now() + Duration::from_secs(60 * 60),
});
state.proof = proof.clone();
drop(state);
let mut journal_state = tier_delete_journal_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
debug_assert!(
journal_state
.proof
.as_ref()
.is_none_or(|current| current.generation.is_drained())
);
journal_state.topology_conflict = false;
journal_state.draining_generation = None;
journal_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation);
}
#[cfg(test)]
pub(crate) struct CrossPoolFenceFleetProofGuard {
previous_proof: Option<FleetCapabilityProof>,
previous_topology_conflict: bool,
previous_journal_proof: Option<FleetCapabilityProof>,
previous_journal_topology_conflict: bool,
}
#[cfg(test)]
@@ -246,8 +484,24 @@ impl Drop for CrossPoolFenceFleetProofGuard {
let mut state = cross_pool_fence_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.proof = self.previous_proof.take();
state.proof = self
.previous_proof
.take()
.as_ref()
.map(FleetCapabilityProof::with_fresh_generation);
state.draining_generation = None;
state.topology_conflict = self.previous_topology_conflict;
drop(state);
let mut journal_state = tier_delete_journal_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
journal_state.proof = self
.previous_journal_proof
.take()
.as_ref()
.map(FleetCapabilityProof::with_fresh_generation);
journal_state.draining_generation = None;
journal_state.topology_conflict = self.previous_journal_topology_conflict;
}
}
@@ -258,12 +512,29 @@ pub(crate) fn without_cross_pool_fence_fleet_proof_for_test() -> CrossPoolFenceF
let mut state = cross_pool_fence_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let mut journal_state = tier_delete_journal_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let guard = CrossPoolFenceFleetProofGuard {
previous_proof: state.proof.clone(),
previous_topology_conflict: state.topology_conflict,
previous_journal_proof: journal_state.proof.clone(),
previous_journal_topology_conflict: journal_state.topology_conflict,
};
state.proof = None;
if let Some(proof) = state.proof.take() {
proof.generation.revoke();
if !proof.generation.is_drained() {
state.draining_generation = Some(proof.generation);
}
}
state.topology_conflict = true;
if let Some(proof) = journal_state.proof.take() {
proof.generation.revoke();
if !proof.generation.is_drained() {
journal_state.draining_generation = Some(proof.generation);
}
}
journal_state.topology_conflict = true;
guard
}
@@ -275,11 +546,33 @@ pub fn rotate_cross_pool_fence_fleet_proof_for_test() -> bool {
let Some(current) = state.proof.as_ref() else {
return false;
};
state.proof = Some(FleetCapabilityProof {
topology_fingerprint: current.topology_fingerprint.clone(),
peer_epochs: Arc::new(current.peer_epochs.as_ref().clone()),
expires_at: current.expires_at,
});
let proof = FleetCapabilityProof::new(
current.topology_fingerprint.clone(),
Arc::new(current.peer_epochs.as_ref().clone()),
current.expires_at,
);
state.proof = Some(proof.clone());
drop(state);
let mut journal_state = tier_delete_journal_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
journal_state.topology_conflict = false;
if let Some(previous) = journal_state.proof.take() {
previous.generation.revoke();
if !previous.generation.is_drained() {
journal_state.draining_generation = Some(previous.generation);
}
}
if journal_state
.draining_generation
.as_ref()
.is_some_and(|generation| generation.is_drained())
{
journal_state.draining_generation = None;
}
if journal_state.draining_generation.is_none() {
journal_state.proof = Some(proof.with_fresh_generation());
}
true
}
@@ -291,15 +584,22 @@ fn fleet_capability_proof_matches(
return false;
};
let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner);
if state.topology_conflict {
return false;
}
state.proof.as_ref().is_some_and(|current| {
current.topology_fingerprint == *expected_topology
&& current.topology_fingerprint == proof.topology_fingerprint
&& Arc::ptr_eq(&current.peer_epochs, &proof.peer_epochs)
&& Instant::now() < current.expires_at
})
fleet_capability_proof_matches_at(&state, proof, expected_topology, Instant::now())
}
fn fleet_capability_proof_matches_at(
state: &FleetCapabilityProofState,
proof: &FleetCapabilityProofToken,
expected_topology: &str,
now: Instant,
) -> bool {
!state.topology_conflict
&& state.proof.as_ref().is_some_and(|current| {
current.topology_fingerprint == expected_topology
&& current.topology_fingerprint == proof.topology_fingerprint
&& Arc::ptr_eq(&current.peer_epochs, &proof.peer_epochs)
&& now < current.expires_at
})
}
fn fleet_capability_proof_valid_at(proof: Option<&FleetCapabilityProof>, expected_topology: &str, now: Instant) -> bool {
@@ -312,7 +612,7 @@ pub(crate) struct RemoteVersionStateFleetProofGuard;
#[cfg(test)]
impl Drop for RemoteVersionStateFleetProofGuard {
fn drop(&mut self) {
replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None);
revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot());
}
}
@@ -348,10 +648,12 @@ fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, pe
pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
for slot in [remote_version_state_fleet_proof_slot(), cross_pool_fence_fleet_proof_slot()] {
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
state.topology_conflict = true;
state.proof = None;
for slot in [
remote_version_state_fleet_proof_slot(),
cross_pool_fence_fleet_proof_slot(),
tier_delete_journal_fleet_proof_slot(),
] {
mark_fleet_capability_topology_conflict(slot);
}
}
return;
@@ -373,7 +675,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
}
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
};
let fence_result = match get_global_notification_sys() {
let fence_probe = match get_global_notification_sys() {
Some(notification_sys) => timeout(
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
@@ -382,13 +684,21 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
};
let (fence_result, journal_result) = match fence_probe {
Ok((peer_epochs, minimum_version)) => cross_pool_fence_policy_results(peer_epochs, minimum_version),
Err(err) => {
let message = err.to_string();
(Err(Error::other(message.clone())), Err(Error::other(message)))
}
};
let topology_conflict = remote_version_state_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.topology_conflict;
if topology_conflict {
replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None);
replace_fleet_capability_proof(cross_pool_fence_fleet_proof_slot(), None);
revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot());
revoke_fleet_capability_proof(cross_pool_fence_fleet_proof_slot());
revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot());
} else if let Some(err) = publish_fleet_capability_probe_result(
remote_version_state_fleet_proof_slot(),
&topology_fingerprint,
@@ -409,7 +719,25 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
capability = "cross_pool_fence_v2",
capability = "cross_pool_fence",
state = "failed_closed",
error = %err,
"notification capability probe"
);
}
if !topology_conflict
&& let Some(err) = publish_fleet_capability_probe_result(
tier_delete_journal_fleet_proof_slot(),
&topology_fingerprint,
journal_result,
Instant::now(),
)
{
debug!(
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
capability = "tier_delete_journal_v6_policy",
state = "failed_closed",
error = %err,
"notification capability probe"
@@ -483,7 +811,7 @@ impl NotificationSys {
Ok(peer_epochs)
}
async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result<(BTreeMap<String, Uuid>, u32)> {
if self.peer_clients.len() != self.peer_topology_hosts.len() {
return Err(Error::other("cross-pool fence capability fleet membership is incomplete"));
}
@@ -494,14 +822,21 @@ impl NotificationSys {
client.probe_cross_pool_fence(topology_fingerprint.to_string()).await
});
let mut peer_epochs = BTreeMap::new();
let mut minimum_version = u32::MAX;
for result in join_all(probes).await {
let (peer, version, epoch) = result?;
if version < CROSS_POOL_FENCE_SUPPORTED_VERSION {
return Err(Error::other("cross-pool fence capability version is unsupported"));
}
minimum_version = minimum_version.min(version);
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
}
Ok(peer_epochs)
// A single-node deployment has no remote member to lower the local
// policy version advertised by this binary.
if minimum_version == u32::MAX {
minimum_version = TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION;
}
Ok((peer_epochs, minimum_version))
}
}
@@ -1827,12 +2162,13 @@ impl NotificationSys {
join_all(futures).await
}
pub async fn abort_tier_mutation(&self, mutation_id: Uuid) -> Vec<NotificationPeerErr> {
pub async fn abort_tier_mutation(&self, mutation_id: Uuid, canonical_prepare_payload: Bytes) -> Vec<NotificationPeerErr> {
let mut futures = Vec::with_capacity(self.peer_clients.len());
for client in self.peer_clients.iter().cloned() {
let payload = canonical_prepare_payload.clone();
futures.push(async move {
if let Some(client) = client {
notification_peer_result(client.host.to_string(), client.abort_tier_mutation(mutation_id).await)
notification_peer_result(client.host.to_string(), client.abort_tier_mutation(mutation_id, payload).await)
} else {
unreachable_notification_peer_err()
}
@@ -2467,16 +2803,24 @@ fn aggregate_scanner_dirty_usage_acknowledgement_results(
mod tests {
use super::*;
#[test]
fn cross_pool_v2_remains_generic_but_cannot_authorize_v6_journal() {
let peers = BTreeMap::from([("node-b:9000".to_string(), Uuid::new_v4())]);
let (generic_v2, journal_v2) = cross_pool_fence_policy_results(peers.clone(), 2);
assert!(generic_v2.is_ok(), "v2 remains valid for existing cross-pool fencing");
assert!(journal_v2.is_err(), "a mixed v2/v3 fleet must fail closed for journal-v6 deletion");
let (generic_v3, journal_v3) = cross_pool_fence_policy_results(peers, 3);
assert!(generic_v3.is_ok());
assert!(journal_v3.is_ok(), "an all-v3 fleet may authorize journal-v6 deletion");
}
#[test]
fn remote_version_state_fleet_proof_rejects_stale_or_mismatched_membership() {
let now = Instant::now();
let mut peer_epochs = BTreeMap::new();
peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
let proof = FleetCapabilityProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(peer_epochs),
expires_at: now + Duration::from_secs(1),
};
let proof = FleetCapabilityProof::new("topology-a".to_string(), Arc::new(peer_epochs), now + Duration::from_secs(1));
assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now));
assert!(!fleet_capability_proof_valid_at(Some(&proof), "topology-b", now));
@@ -2495,11 +2839,7 @@ mod tests {
#[test]
fn remote_version_state_fleet_proof_accepts_single_node_membership() {
let now = Instant::now();
let proof = FleetCapabilityProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(BTreeMap::new()),
expires_at: now + Duration::from_secs(1),
};
let proof = FleetCapabilityProof::new("topology-a".to_string(), Arc::new(BTreeMap::new()), now + Duration::from_secs(1));
assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now));
}
@@ -2507,21 +2847,97 @@ mod tests {
#[test]
fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
let now = Instant::now();
let proof = FleetCapabilityProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
expires_at: now + Duration::from_secs(1),
};
let proof = FleetCapabilityProof::new(
"topology-a".to_string(),
Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
now + Duration::from_secs(1),
);
let captured = proof.token();
let restarted = FleetCapabilityProof {
topology_fingerprint: proof.topology_fingerprint.clone(),
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
expires_at: proof.expires_at,
};
let restarted = FleetCapabilityProof::new(
proof.topology_fingerprint.clone(),
Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
proof.expires_at,
);
assert!(captured != restarted.token());
}
#[test]
fn tier_delete_journal_generation_is_stable_across_members_and_process_restarts() {
let topology = "topology-a";
let now = Instant::now();
let node_a_view = FleetCapabilityProof::new(
topology.to_string(),
Arc::new(BTreeMap::from([("node-b".to_string(), Uuid::new_v4())])),
now + Duration::from_secs(1),
);
let node_b_view = FleetCapabilityProof::new(
topology.to_string(),
Arc::new(BTreeMap::from([("node-a".to_string(), Uuid::new_v4())])),
now + Duration::from_secs(1),
);
let restarted_node_a_view = FleetCapabilityProof::new(
topology.to_string(),
Arc::new(BTreeMap::from([("node-b".to_string(), Uuid::new_v4())])),
now + Duration::from_secs(1),
);
let generations = [&node_a_view, &node_b_view, &restarted_node_a_view]
.map(|proof| stable_tier_delete_journal_topology_generation(&proof.token().topology_fingerprint));
assert_eq!(generations[0], generations[1]);
assert_eq!(generations[0], generations[2]);
assert_ne!(
generations[0],
stable_tier_delete_journal_topology_generation("topology-b"),
"a real topology change must produce a different durable generation"
);
}
#[test]
fn tier_delete_journal_restart_revokes_old_token_but_fresh_token_recovers_same_generation() {
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
let now = Instant::now();
let original_peers = BTreeMap::from([("node-b".to_string(), Uuid::new_v4())]);
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original_peers), now).is_none());
let original = slot
.read()
.expect("proof slot should not poison")
.proof
.as_ref()
.expect("successful probe should publish proof")
.token();
let original_generation = stable_tier_delete_journal_topology_generation(&original.topology_fingerprint);
let restarted_peers = BTreeMap::from([("node-b".to_string(), Uuid::new_v4())]);
assert!(
publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted_peers), now + Duration::from_millis(1))
.is_none()
);
let state = slot.read().expect("proof slot should not poison");
let fresh = state
.proof
.as_ref()
.expect("restart probe should publish a fresh proof")
.token();
assert!(!fleet_capability_proof_matches_at(
&state,
&original,
"topology-a",
now + Duration::from_millis(2)
));
assert!(fleet_capability_proof_matches_at(
&state,
&fresh,
"topology-a",
now + Duration::from_millis(2)
));
assert_eq!(
original_generation,
stable_tier_delete_journal_topology_generation(&fresh.topology_fingerprint)
);
}
#[test]
fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
@@ -2561,15 +2977,106 @@ mod tests {
assert!(!Arc::ptr_eq(&original.peer_epochs, &replaced.peer_epochs));
}
#[test]
fn tier_delete_journal_successor_waits_for_inflight_generation_to_drain() {
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
let now = Instant::now();
let original_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original_peers), now).is_none());
let admitted = {
let state = slot.read().expect("proof slot should not poison");
acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now)
.expect("a fresh proof should admit one journal operation")
};
{
let state = slot.read().expect("proof slot should not poison");
assert!(
tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now),
"a freshly admitted journal proof must remain current"
);
assert!(
acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now + REMOTE_VERSION_STATE_PROOF_TTL,)
.is_none(),
"TTL expiry must stop new admission"
);
assert!(
!tier_delete_journal_fleet_proof_matches_at(
&state,
&admitted,
"topology-a",
now + REMOTE_VERSION_STATE_PROOF_TTL,
),
"TTL expiry must also stop an admitted proof at its next durable fence"
);
assert!(!admitted._permit.generation.is_drained());
}
let restarted_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
let blocked = publish_fleet_capability_probe_result(
&slot,
"topology-a",
Ok(restarted_peers.clone()),
now + Duration::from_millis(1),
)
.expect("a successor proof must wait for the admitted generation");
assert!(blocked.to_string().contains("previous generation to drain"));
{
let state = slot.read().expect("proof slot should not poison");
assert!(state.proof.is_none(), "new operations must remain closed while the predecessor drains");
assert!(state.draining_generation.is_some());
assert!(
!tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now + Duration::from_millis(1),),
"a restarted peer must revoke an admitted proof before its next durable fence"
);
}
drop(admitted);
assert!(
publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted_peers), now + Duration::from_millis(2),)
.is_none(),
"the successor may publish after the in-flight operation releases its permit"
);
let state = slot.read().expect("proof slot should not poison");
assert!(state.proof.is_some());
assert!(state.draining_generation.is_none());
}
#[test]
fn tier_delete_journal_topology_conflict_revokes_admitted_generation() {
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
let now = Instant::now();
let peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now).is_none());
let admitted = {
let state = slot.read().expect("proof slot should not poison");
acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now)
.expect("a fresh proof should admit one journal operation")
};
mark_fleet_capability_topology_conflict(&slot);
let state = slot.read().expect("proof slot should not poison");
assert!(state.topology_conflict);
assert!(state.proof.is_none());
assert!(state.draining_generation.is_some());
assert!(!admitted._permit.generation.is_accepting());
assert!(
!tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now),
"topology conflict must revoke an already admitted journal proof"
);
}
#[test]
fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
let now = Instant::now();
let mut state = FleetCapabilityProofState {
proof: Some(FleetCapabilityProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(BTreeMap::new()),
expires_at: now + Duration::from_secs(1),
}),
proof: Some(FleetCapabilityProof::new(
"topology-a".to_string(),
Arc::new(BTreeMap::new()),
now + Duration::from_secs(1),
)),
draining_generation: None,
topology_conflict: false,
};
assert!(acquire_fleet_capability_proof_from(&state, "topology-a", now).is_some());
@@ -3279,7 +3786,7 @@ mod tests {
assert_eq!(commit.len(), 1);
assert!(commit[0].err.is_some());
let abort = sys.abort_tier_mutation(mutation_id).await;
let abort = sys.abort_tier_mutation(mutation_id, Bytes::from_static(b"prepare")).await;
assert_eq!(abort.len(), 1);
assert!(abort[0].err.is_some());
}
@@ -356,7 +356,7 @@ impl ECStore {
};
run_guard.ensure_held("rebalance version migration")?;
let result = migrate_entry_version(
&RebalanceMigrationBackend::new(set.as_ref(), self.as_ref(), lock_lost_signal.clone()),
&RebalanceMigrationBackend::new(set.as_ref(), self.clone(), lock_lost_signal.clone()),
bucket.clone(),
pool_index,
version,
@@ -1478,6 +1478,7 @@ mod tests {
let (_temp_dirs, store, _unused_store) =
crate::services::rebalance::test_two_pool_stores(Some(active_rebalance_meta(REBALANCE_ID))).await;
prepare_rebalance_test_volumes(store.as_ref()).await;
crate::services::tier::test_util::register_mock_tier(&store.tier_config_mgr(), "WARM").await;
let source_set = store.pools[0].get_disks_by_key(object);
let target_set = store.pools[1].get_disks_by_key(object);
let version_id = uuid::Uuid::new_v4();
@@ -1520,14 +1521,17 @@ mod tests {
let entry = metacache_entry_from_source(source_set.as_ref(), bucket, object).await;
let run_signal_fence = RebalanceRunSignalTestFence::install(REBALANCE_ID);
let barrier = TieredMetadataCommitBarrier::install(bucket, object);
let task = spawn_real_rebalance_entry(
let mut task = spawn_real_rebalance_entry(
Arc::clone(&store),
Arc::clone(&source_set),
entry,
REBALANCE_ID,
Arc::new(RebalanceBucketConfigs::default()),
);
barrier.wait_until_paused().await;
tokio::select! {
_ = barrier.wait_until_paused() => {}
result = &mut task => panic!("rebalance exited before the tiered commit barrier: {result:?}"),
}
run_signal_fence.mark_lost();
barrier.release();
drop(barrier);
@@ -101,14 +101,14 @@ pub(crate) trait MigrationBackend: Send + Sync {
pub(crate) struct RebalanceMigrationBackend<'a> {
source: &'a SetDisks,
store: &'a ECStore,
store: std::sync::Arc<ECStore>,
lock_lost_signal: Option<std::sync::Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
}
impl<'a> RebalanceMigrationBackend<'a> {
pub(crate) fn new(
source: &'a SetDisks,
store: &'a ECStore,
store: std::sync::Arc<ECStore>,
lock_lost_signal: Option<std::sync::Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
) -> Self {
Self {
+1 -1
View File
@@ -134,7 +134,7 @@ pub(crate) async fn test_three_pool_stores_with_isolated_node_contexts(
test_pool_stores_with_contexts(rebalance_meta, true, 3, 2).await
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pub(crate) async fn test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(
rebalance_meta: Option<RebalanceMeta>,
) -> (
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(feature = "test-util")]
#[cfg(any(test, feature = "test-util"))]
pub mod test_util;
pub mod tier;
pub mod tier_admin;
File diff suppressed because it is too large Load Diff
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use std::sync::{Arc, LazyLock};
use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
use serde::{Deserialize, Serialize};
@@ -32,9 +32,34 @@ pub(crate) const TIER_MUTATION_INTENT_SCHEMA: &str = "rustfs-tier-mutation-inten
pub(crate) const MAX_TIER_MUTATION_INTENT_SIZE: usize = rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE;
pub(crate) const TIER_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/records";
pub(crate) const TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/coordinators";
pub(crate) const TIER_MUTATION_MUTEX_SHARDS: usize = 64;
const TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS: usize = 3;
pub(crate) type TierMutationDigest = [u8; 32];
static TIER_MUTATION_MUTEXES: LazyLock<[tokio::sync::Mutex<()>; TIER_MUTATION_MUTEX_SHARDS]> =
LazyLock::new(|| std::array::from_fn(|_| tokio::sync::Mutex::new(())));
/// Serializes every local phase and recovery action for one mutation id while
/// retaining bounded parallelism for unrelated mutations.
pub(crate) async fn acquire_tier_mutation_mutex(mutation_id: Uuid) -> tokio::sync::MutexGuard<'static, ()> {
TIER_MUTATION_MUTEXES[tier_mutation_mutex_shard_index(mutation_id)]
.lock()
.await
}
fn tier_mutation_mutex_shard_index(mutation_id: Uuid) -> usize {
let raw = mutation_id.as_u128();
let mut mixed = (raw as u64) ^ ((raw >> 64) as u64);
// MurmurHash3's 64-bit finalizer gives stable diffusion without allocating
// or relying on RandomState, whose seed differs between processes.
mixed ^= mixed >> 33;
mixed = mixed.wrapping_mul(0xff51_afd7_ed55_8ccd);
mixed ^= mixed >> 33;
mixed = mixed.wrapping_mul(0xc4ce_b9fe_1a85_ec53);
mixed ^= mixed >> 33;
(mixed as usize) & (TIER_MUTATION_MUTEX_SHARDS - 1)
}
pub(crate) type Result<T> = std::result::Result<T, TierMutationIntentError>;
#[derive(Debug, thiserror::Error)]
@@ -265,6 +290,30 @@ impl TierMutationIntent {
&& self.expires_at_unix_nanos == other.expires_at_unix_nanos
}
/// Reconstruct the exact Prepared record that originally produced this
/// intent. Abort RPCs are identity-bound to that payload; serializing an
/// Aborted terminal record would both violate the wire contract and use a
/// different revision if a missing peer has to persist a tombstone.
pub(crate) fn original_prepared(&self) -> Result<Self> {
if self.state == TierMutationIntentState::Prepared {
self.validate()?;
return Ok(self.clone());
}
let mut prepared = self.clone();
prepared.revision =
prepared
.revision
.checked_sub(1)
.filter(|revision| *revision != 0)
.ok_or(TierMutationIntentError::Corrupt(
"terminal intent cannot reconstruct its prepared revision",
))?;
prepared.state = TierMutationIntentState::Prepared;
prepared.committed_config_etag = None;
prepared.validate()?;
Ok(prepared)
}
pub(crate) fn encode(&self) -> Result<Vec<u8>> {
self.validate()?;
let intent_bytes = serde_json::to_vec(self)?;
@@ -439,6 +488,16 @@ where
load_tier_mutation_intent_record_with_etag_at_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
}
pub(crate) async fn load_tier_coordinator_mutation_intent_record_with_etag<S>(
api: Arc<S>,
mutation_id: Uuid,
) -> EcstoreResult<(TierMutationIntent, String)>
where
S: EcstoreObjectIO,
{
load_tier_mutation_intent_record_with_etag_at_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
}
async fn load_tier_mutation_intent_record_with_etag_at_prefix<S>(
api: Arc<S>,
prefix: &str,
@@ -507,6 +566,7 @@ where
.await
}
#[cfg(test)]
pub(crate) async fn delete_tier_mutation_intent_record<S>(api: Arc<S>, mutation_id: Uuid) -> EcstoreResult<()>
where
S: EcstoreObjectOperations,
@@ -514,13 +574,36 @@ where
delete_tier_mutation_intent_record_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
}
pub(crate) async fn delete_tier_coordinator_mutation_intent_record<S>(api: Arc<S>, mutation_id: Uuid) -> EcstoreResult<()>
pub(crate) async fn delete_tier_mutation_intent_record_if_current<S>(
api: Arc<S>,
mutation_id: Uuid,
current_etag: &str,
) -> EcstoreResult<()>
where
S: EcstoreObjectOperations,
{
delete_tier_mutation_intent_record_with_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
delete_tier_mutation_intent_record_if_current_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id, current_etag)
.await
}
pub(crate) async fn delete_tier_coordinator_mutation_intent_record_if_current<S>(
api: Arc<S>,
mutation_id: Uuid,
current_etag: &str,
) -> EcstoreResult<()>
where
S: EcstoreObjectOperations,
{
delete_tier_mutation_intent_record_if_current_with_prefix(
api,
TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX,
mutation_id,
current_etag,
)
.await
}
#[cfg(test)]
async fn delete_tier_mutation_intent_record_with_prefix<S>(api: Arc<S>, prefix: &str, mutation_id: Uuid) -> EcstoreResult<()>
where
S: EcstoreObjectOperations,
@@ -533,6 +616,40 @@ where
}
}
async fn delete_tier_mutation_intent_record_if_current_with_prefix<S>(
api: Arc<S>,
prefix: &str,
mutation_id: Uuid,
current_etag: &str,
) -> EcstoreResult<()>
where
S: EcstoreObjectOperations,
{
if current_etag.trim().is_empty() {
return Err(Error::other("tier mutation intent current ETag is empty"));
}
let object =
tier_mutation_intent_record_object_name_with_prefix(prefix, mutation_id).map_err(tier_mutation_intent_store_error)?;
match api
.delete_object(
RUSTFS_META_BUCKET,
&object,
ObjectOptions {
http_preconditions: Some(HTTPPreconditions {
if_match: Some(current_etag.to_string()),
..Default::default()
}),
..Default::default()
},
)
.await
{
Ok(_) => Ok(()),
Err(err) if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) => Err(Error::ConfigNotFound),
Err(err) => Err(err),
}
}
pub(crate) async fn advance_tier_mutation_intent_record_idempotent<S>(
api: Arc<S>,
mutation_id: Uuid,
@@ -713,6 +830,7 @@ fn digest_is_empty(digest: &TierMutationDigest) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use std::time::Duration;
const OLD_IDENTITY: TierDestinationId = [1; 32];
const NEW_IDENTITY: TierDestinationId = [2; 32];
@@ -736,6 +854,61 @@ mod tests {
}
}
#[test]
fn mutation_mutex_uses_exactly_64_stable_shards() {
assert_eq!(TIER_MUTATION_MUTEX_SHARDS, 64);
assert_eq!(TIER_MUTATION_MUTEXES.len(), TIER_MUTATION_MUTEX_SHARDS);
let mutation_id = Uuid::parse_str("36e2220e-9ad2-495b-b3bc-c4d2caf70a31").expect("fixture uuid should parse");
let shard = tier_mutation_mutex_shard_index(mutation_id);
assert!(shard < TIER_MUTATION_MUTEX_SHARDS);
assert_eq!(shard, tier_mutation_mutex_shard_index(mutation_id));
}
#[tokio::test]
async fn mutation_mutex_serializes_the_same_id() {
let mutation_id = Uuid::new_v4();
let first = acquire_tier_mutation_mutex(mutation_id).await;
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (acquired_tx, mut acquired_rx) = tokio::sync::oneshot::channel();
let waiter = tokio::spawn(async move {
started_tx.send(()).expect("test receiver should remain alive");
let _second = acquire_tier_mutation_mutex(mutation_id).await;
acquired_tx.send(()).expect("test receiver should remain alive");
});
started_rx.await.expect("waiter should start");
assert!(
tokio::time::timeout(Duration::from_millis(25), &mut acquired_rx)
.await
.is_err(),
"the same mutation id must not enter concurrently"
);
drop(first);
tokio::time::timeout(Duration::from_secs(1), &mut acquired_rx)
.await
.expect("waiter should acquire after release")
.expect("waiter should report acquisition");
waiter.await.expect("waiter task should finish");
}
#[tokio::test]
async fn mutation_mutex_allows_different_shards_to_progress() {
let first_id = Uuid::new_v4();
let first_shard = tier_mutation_mutex_shard_index(first_id);
let second_id = (0..1024)
.map(|_| Uuid::new_v4())
.find(|candidate| tier_mutation_mutex_shard_index(*candidate) != first_shard)
.expect("a distinct shard should be easy to find");
let first = acquire_tier_mutation_mutex(first_id).await;
let _second = tokio::time::timeout(Duration::from_secs(1), acquire_tier_mutation_mutex(second_id))
.await
.expect("a different shard must not wait for the first mutation");
drop(first);
}
#[test]
fn intent_round_trip_preserves_committed_state() {
let mut intent = prepared_intent();
@@ -873,6 +1046,37 @@ mod tests {
));
}
#[test]
fn terminal_intent_reconstructs_original_prepared_abort_payload() {
for terminal in [TierMutationIntentState::Aborted, TierMutationIntentState::Committed] {
let original = prepared_intent();
let mut intent = original.clone();
let committed_etag = (terminal == TierMutationIntentState::Committed).then(|| "new-etag".to_string());
intent
.advance(terminal, committed_etag)
.expect("terminal transition should succeed");
let reconstructed = intent
.original_prepared()
.expect("terminal record should recover prepared payload");
assert_eq!(reconstructed, original);
assert!(intent.same_identity_as(&reconstructed));
}
}
#[test]
fn terminal_intent_with_initial_revision_fails_prepared_reconstruction() {
let mut corrupt = prepared_intent();
corrupt.state = TierMutationIntentState::Aborted;
assert!(matches!(
corrupt.original_prepared(),
Err(TierMutationIntentError::Corrupt(
"terminal intent cannot reconstruct its prepared revision"
))
));
}
#[test]
fn intent_validation_rejects_placeholder_identity() {
let mut intent = prepared_intent();
@@ -15,12 +15,13 @@
use std::sync::Arc;
use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase};
use time::OffsetDateTime;
use uuid::Uuid;
use super::tier::{TierConfigMgr, tier_config_abort_matches, tier_config_commit_matches, tier_config_etag_matches};
use super::tier_mutation_intent::{
MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, advance_tier_mutation_intent_record_idempotent,
load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, acquire_tier_mutation_mutex,
advance_tier_mutation_intent_record_idempotent, load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
};
use crate::error::{Error, StorageError};
use crate::store::ECStore;
@@ -57,6 +58,8 @@ pub enum TierMutationPeerError {
CommitProofMismatch,
#[error("tier mutation peer abort proof does not match the persisted tier configuration")]
AbortProofMismatch,
#[error("tier mutation peer prepared intent has expired")]
ExpiredIntent,
#[error("tier mutation peer runtime error: {0}")]
Runtime(#[source] AdminError),
#[error("tier mutation peer store error: {0}")]
@@ -79,6 +82,7 @@ pub async fn handle_tier_mutation_peer_request(
canonical_payload: &[u8],
) -> TierMutationPeerResult<TierMutationPeerOutcome> {
validate_peer_request_envelope(protocol_version, mutation_id, canonical_payload)?;
let _mutation_guard = acquire_tier_mutation_mutex(mutation_id).await;
match phase {
TierMutationRpcPhase::Prepare => handle_prepare(api, mutation_id, canonical_payload).await,
TierMutationRpcPhase::Commit => handle_commit(api, mutation_id, canonical_payload).await,
@@ -103,43 +107,57 @@ async fn handle_prepare(
}
let tier_config_mgr = api.tier_config_mgr();
match save_tier_mutation_intent_record_if_absent(api.clone(), &intent).await {
Ok(()) => {
TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &intent)
.await
.map_err(TierMutationPeerError::Runtime)?;
Ok(TierMutationPeerOutcome {
state: TierMutationPeerState::Prepared,
applied: true,
})
}
Err(Error::PreconditionFailed) => {
let existing = load_tier_mutation_intent_record(api, mutation_id).await?;
if !existing.same_identity_as(&intent) {
return Err(TierMutationPeerError::ConflictingIntent);
for _ in 0..3 {
let (stored, applied) = match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
Ok(existing) => {
if !existing.same_identity_as(&intent) {
return Err(TierMutationPeerError::ConflictingIntent);
}
(existing, false)
}
match existing.state {
TierMutationIntentState::Prepared => {
TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &existing)
.await
.map_err(TierMutationPeerError::Runtime)?;
Err(Error::ConfigNotFound) => {
let now = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).unwrap_or(i64::MAX);
if intent.expires_at_unix_nanos <= now {
return Err(TierMutationPeerError::ExpiredIntent);
}
TierMutationIntentState::Committed => {
TierConfigMgr::apply_committed_mutation_intent_block(&tier_config_mgr, &existing)
.await
.map_err(TierMutationPeerError::Runtime)?;
}
TierMutationIntentState::Aborted => {
TierConfigMgr::request_committed_mutation_refresh(&tier_config_mgr).await;
match save_tier_mutation_intent_record_if_absent(api.clone(), &intent).await {
Ok(()) => (intent.clone(), true),
Err(Error::PreconditionFailed) => continue,
Err(err) => return Err(err.into()),
}
}
Ok(TierMutationPeerOutcome {
state: peer_state_from_intent(existing.state),
applied: false,
})
Err(err) => return Err(err.into()),
};
match stored.state {
TierMutationIntentState::Prepared => {
TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &stored)
.await
.map_err(TierMutationPeerError::Runtime)?;
TierConfigMgr::wait_for_blocked_tier_operation_leases(&tier_config_mgr, &stored)
.await
.map_err(TierMutationPeerError::Runtime)?;
}
TierMutationIntentState::Committed => {
TierConfigMgr::apply_committed_mutation_intent_block(&tier_config_mgr, &stored)
.await
.map_err(TierMutationPeerError::Runtime)?;
}
TierMutationIntentState::Aborted => {
TierConfigMgr::clear_prepared_mutation_intent_block(&tier_config_mgr, mutation_id)
.await
.map_err(TierMutationPeerError::Runtime)?;
TierConfigMgr::request_committed_mutation_refresh(&tier_config_mgr).await;
}
}
Err(err) => Err(err.into()),
return Ok(TierMutationPeerOutcome {
state: peer_state_from_intent(stored.state),
applied,
});
}
Err(TierMutationPeerError::Store(Error::other(
"tier mutation prepare raced repeatedly with another decision",
)))
}
async fn handle_commit(
@@ -201,26 +219,106 @@ async fn handle_abort(
mutation_id: Uuid,
canonical_payload: &[u8],
) -> TierMutationPeerResult<TierMutationPeerOutcome> {
if !canonical_payload.is_empty() {
return Err(TierMutationPeerError::InvalidPayload("abort payload must be empty".to_string()));
let prepared = TierMutationIntent::decode(mutation_id, canonical_payload)
.map_err(|err| TierMutationPeerError::InvalidPayload(err.to_string()))?;
if prepared.state != TierMutationIntentState::Prepared {
return Err(TierMutationPeerError::InvalidPayload(
"abort payload must carry the original prepared intent".to_string(),
));
}
let existing = load_tier_mutation_intent_record(api.clone(), mutation_id).await?;
if existing.state == TierMutationIntentState::Prepared
&& !tier_config_abort_matches(api.clone(), &existing)
.await
.map_err(Error::other)?
{
return Err(TierMutationPeerError::AbortProofMismatch);
let mut tombstone = prepared.clone();
tombstone
.advance(TierMutationIntentState::Aborted, None)
.map_err(|err| TierMutationPeerError::InvalidPayload(err.to_string()))?;
for _ in 0..3 {
match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
Ok(existing) => {
if !existing.same_identity_as(&prepared) {
return Err(TierMutationPeerError::ConflictingIntent);
}
match existing.state {
TierMutationIntentState::Committed => {
return Ok(TierMutationPeerOutcome {
state: TierMutationPeerState::Committed,
applied: false,
});
}
TierMutationIntentState::Aborted => {
TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id)
.await
.map_err(TierMutationPeerError::Runtime)?;
TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
return Ok(TierMutationPeerOutcome {
state: TierMutationPeerState::Aborted,
applied: false,
});
}
TierMutationIntentState::Prepared => {}
}
if !tier_config_abort_matches(api.clone(), &prepared)
.await
.map_err(Error::other)?
{
return Err(TierMutationPeerError::AbortProofMismatch);
}
let advanced = advance_tier_mutation_intent_record_idempotent(
api.clone(),
mutation_id,
TierMutationIntentState::Aborted,
None,
)
.await;
let (intent, applied) = match advanced {
Ok(result) => result,
Err(err) => match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
Ok(current)
if current.same_identity_as(&prepared) && current.state != TierMutationIntentState::Prepared =>
{
(current, false)
}
_ => return Err(err.into()),
},
};
if intent.state == TierMutationIntentState::Aborted {
TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id)
.await
.map_err(TierMutationPeerError::Runtime)?;
}
return Ok(TierMutationPeerOutcome {
state: peer_state_from_intent(intent.state),
applied,
});
}
Err(Error::ConfigNotFound) => {
if !tier_config_abort_matches(api.clone(), &prepared)
.await
.map_err(Error::other)?
{
return Err(TierMutationPeerError::AbortProofMismatch);
}
match save_tier_mutation_intent_record_if_absent(api.clone(), &tombstone).await {
Ok(()) => {
TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id)
.await
.map_err(TierMutationPeerError::Runtime)?;
TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
return Ok(TierMutationPeerOutcome {
state: TierMutationPeerState::Aborted,
applied: true,
});
}
Err(Error::PreconditionFailed) => continue,
Err(err) => return Err(err.into()),
}
}
Err(err) => return Err(err.into()),
}
}
let (intent, applied) =
advance_tier_mutation_intent_record_idempotent(api.clone(), mutation_id, TierMutationIntentState::Aborted, None).await?;
if intent.state == TierMutationIntentState::Aborted {
TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
}
Ok(TierMutationPeerOutcome {
state: peer_state_from_intent(intent.state),
applied,
})
Err(TierMutationPeerError::Store(Error::other(
"tier mutation abort raced repeatedly with prepare",
)))
}
fn validate_peer_request_envelope(
@@ -228,7 +326,10 @@ fn validate_peer_request_envelope(
mutation_id: Uuid,
canonical_payload: &[u8],
) -> TierMutationPeerResult<()> {
if protocol_version != TIER_MUTATION_RPC_PROTOCOL_VERSION {
if !matches!(
protocol_version,
rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION | TIER_MUTATION_RPC_PROTOCOL_VERSION
) {
return Err(TierMutationPeerError::UnsupportedProtocolVersion(protocol_version));
}
if mutation_id.is_nil() {
@@ -276,6 +377,8 @@ mod tests {
#[test]
fn peer_request_envelope_fails_closed_on_old_version_nil_id_and_large_payload() {
let mutation_id = Uuid::new_v4();
validate_peer_request_envelope(rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION, mutation_id, b"payload")
.expect("v3 must remain accepted during the v4 rollout");
assert!(matches!(
validate_peer_request_envelope(TIER_MUTATION_RPC_PROTOCOL_VERSION + 1, mutation_id, b"payload"),
Err(TierMutationPeerError::UnsupportedProtocolVersion(_))
+4 -2
View File
@@ -859,7 +859,9 @@ static OBJECT_LOCK_DIAG_ENABLED: OnceLock<bool> = OnceLock::new();
mod core;
#[cfg(test)]
pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, disk_call_counters, rename_fanout_barrier};
pub(crate) use core::io_primitives::disk_call_counters;
#[cfg(all(test, feature = "test-util"))]
pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
mod ctx;
mod metadata;
mod ops;
@@ -872,7 +874,7 @@ pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
#[cfg(test)]
pub(crate) use ops::object::DeleteObjectCommitBarrier;
#[cfg(feature = "test-util")]
#[cfg(any(test, feature = "test-util"))]
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
pub(crate) use ops::object::body_cache_plaintext_len;
#[cfg(all(test, feature = "test-util"))]
+1 -1
View File
@@ -244,7 +244,7 @@ impl StaleMultipartCleanupGuard {
}
#[cfg(any(test, feature = "test-util"))]
#[derive(Clone, Copy, PartialEq, Eq)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MultipartCommitPause {
NewUploadBeforeLockLost,
PutPartBeforeLockAcquire,
File diff suppressed because it is too large Load Diff
+301 -8
View File
@@ -29,6 +29,25 @@ use std::future::Future;
const DELETED_BUCKETS_PREFIX: &str = ".deleted";
const SCANNER_BUCKET_LIST_SET_CONCURRENCY: usize = 4;
const EVENT_BUCKET_DELETE_BLOCKED: &str = "bucket_delete_blocked";
fn record_bucket_delete_blocker(bucket: &str, kind: BucketDeleteBlockerKind, residue: &BucketMetadataLessResidue) {
metrics::counter!("rustfs_bucket_delete_blockers_total", "kind" => kind.as_str()).increment(1);
debug!(
event = EVENT_BUCKET_DELETE_BLOCKED,
component = "ecstore",
subsystem = "bucket",
bucket,
blocker = kind.as_str(),
files = residue.files,
uuid_data_dirs = residue.uuid_data_dirs,
entries_scanned = residue.entries_scanned,
diagnostic_bytes_read = residue.diagnostic_bytes_read,
diagnostic_truncated = residue.diagnostic_truncated,
sample = residue.sample.as_deref().unwrap_or("<none>"),
"Bucket deletion was blocked by durable local state"
);
}
fn scanner_bucket_list_set_concurrency(set_count: usize) -> usize {
set_count.clamp(1, SCANNER_BUCKET_LIST_SET_CONCURRENCY)
@@ -156,6 +175,7 @@ where
async fn bucket_delete_local_blocker(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
budget: &mut BucketDeleteDiagnosticBudget,
) -> Result<Option<StorageError>> {
let local_disks = runtime_sources::local_disks_in(ctx).await;
let mut residue = BucketMetadataLessResidue::default();
@@ -164,18 +184,30 @@ async fn bucket_delete_local_blocker(
let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else {
continue;
};
let scan = scan_metadata_less_residue(&bucket_path?).await?;
let scan = scan_metadata_less_residue_with_budget(&bucket_path?, budget).await?;
if scan.xlmeta_found {
record_bucket_delete_blocker(bucket, scan.xlmeta_blocker.unwrap_or(BucketDeleteBlockerKind::UnknownXlMeta), &scan);
return Ok(Some(StorageError::BucketNotEmpty(bucket.to_string())));
}
residue.files = residue.files.saturating_add(scan.files);
residue.uuid_data_dirs = residue.uuid_data_dirs.saturating_add(scan.uuid_data_dirs);
residue.entries_scanned = residue.entries_scanned.saturating_add(scan.entries_scanned);
residue.diagnostic_bytes_read = residue.diagnostic_bytes_read.saturating_add(scan.diagnostic_bytes_read);
if residue.sample.is_none() {
residue.sample = scan.sample;
residue.sample = scan.sample.clone();
}
if scan.diagnostic_truncated {
residue.diagnostic_truncated = true;
record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::DiagnosticBudgetExceeded, &residue);
return Ok(Some(StorageError::BucketNotEmptyWithDetails {
bucket: bucket.to_string(),
details: residue.describe(),
}));
}
}
if residue.has_residue_without_xlmeta() {
record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::OrphanDirectory, &residue);
return Ok(Some(StorageError::BucketNotEmptyWithDetails {
bucket: bucket.to_string(),
details: residue.describe(),
@@ -783,12 +815,14 @@ impl ECStore {
}
}
};
let mut diagnostic_budget = None;
if bucket_exists {
validate_table_bucket_delete_guard(&self.ctx, bucket).await?;
if !opts.force {
if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket).await? {
let budget = diagnostic_budget.get_or_insert_with(BucketDeleteDiagnosticBudget::new);
if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket, budget).await? {
return Err(blocker);
}
delete_opts.force_if_empty = true;
@@ -827,7 +861,12 @@ impl ECStore {
{
if delete_opts.force_if_empty
&& matches!(&err, StorageError::BucketNotEmpty(_))
&& let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket).await?
&& let Some(blocker) = bucket_delete_local_blocker(
&self.ctx,
bucket,
diagnostic_budget.get_or_insert_with(BucketDeleteDiagnosticBudget::new),
)
.await?
{
return Err(blocker);
}
@@ -856,15 +895,17 @@ impl ECStore {
#[cfg(test)]
mod tests {
use super::{
SCANNER_BUCKET_LIST_SET_CONCURRENCY, await_bucket_namespace_operation, bucket_delete_metadata_cleanup_prefixes,
bucket_deleted_marker_prefix, bucket_deleted_marker_volume, run_bucket_usage_cleanup, run_physical_bucket_deletion,
scan_metadata_less_residue, scanner_bucket_list_set_concurrency, should_override_created_from_metadata,
BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES, BucketDeleteBlockerKind,
BucketDeleteDiagnosticBudget, SCANNER_BUCKET_LIST_SET_CONCURRENCY, await_bucket_namespace_operation,
bucket_delete_metadata_cleanup_prefixes, bucket_deleted_marker_prefix, bucket_deleted_marker_volume,
run_bucket_usage_cleanup, run_physical_bucket_deletion, scan_metadata_less_residue,
scan_metadata_less_residue_with_budget, scanner_bucket_list_set_concurrency, should_override_created_from_metadata,
validate_table_bucket_delete_allowed,
};
use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
use crate::bucket::metadata_sys;
use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE};
use crate::error::StorageError;
use crate::object_api::{ObjectOptions, PutObjReader};
use crate::runtime::instance::InstanceContext;
@@ -879,6 +920,7 @@ mod tests {
layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
};
use rustfs_data_usage::{BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageInfo};
use rustfs_filemeta::{FileInfo, FileMeta, TRANSITION_COMPLETE};
use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
use serial_test::serial;
use std::path::{Path, PathBuf};
@@ -892,6 +934,88 @@ mod tests {
static BUCKET_DELETE_TEST_ENV: OnceCell<(Vec<PathBuf>, Arc<ECStore>)> = OnceCell::const_new();
#[tokio::test(start_paused = true)]
async fn bucket_delete_diagnostic_budget_starts_with_first_scan_io_and_latches_once() {
let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
tokio::time::advance(Duration::from_secs(10)).await;
let first_polled = Arc::new(AtomicBool::new(false));
let first_polled_for_io = first_polled.clone();
let first = budget
.run_io(async move {
first_polled_for_io.store(true, Ordering::SeqCst);
Ok::<_, std::io::Error>(7_u8)
})
.await
.expect("the first diagnostic IO should succeed");
assert_eq!(first, Some(7));
assert!(first_polled.load(Ordering::SeqCst));
tokio::time::advance(Duration::from_millis(101)).await;
let expired_polled = Arc::new(AtomicBool::new(false));
let expired_polled_for_io = expired_polled.clone();
let expired = budget
.run_io(async move {
expired_polled_for_io.store(true, Ordering::SeqCst);
Ok::<_, std::io::Error>(9_u8)
})
.await
.expect("an expired diagnostic budget should not become an IO error");
assert_eq!(expired, None);
assert!(
!expired_polled.load(Ordering::SeqCst),
"the deadline must remain latched after the first scan IO"
);
}
#[tokio::test(start_paused = true)]
async fn bucket_delete_diagnostic_budget_times_out_its_first_pending_io() {
let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
let io_polled = Arc::new(AtomicBool::new(false));
let io_polled_for_future = io_polled.clone();
let result = budget
.run_io(std::future::poll_fn(move |_cx| {
io_polled_for_future.store(true, Ordering::SeqCst);
std::task::Poll::<std::io::Result<()>>::Pending
}))
.await
.expect("a diagnostic timeout should fail closed without an IO error");
assert_eq!(result, None);
assert!(
io_polled.load(Ordering::SeqCst),
"the first diagnostic IO must be polled before its timeout"
);
}
#[tokio::test(start_paused = true)]
async fn delayed_metadata_less_scans_still_detect_orphans_and_xlmeta() {
let root = tempfile::tempdir().expect("temporary delayed-scan roots should be created");
let orphan_root = root.path().join("orphan-root");
let xlmeta_root = root.path().join("xlmeta-root");
std::fs::create_dir_all(&orphan_root).expect("orphan root should be created");
std::fs::create_dir_all(&xlmeta_root).expect("xlmeta root should be created");
std::fs::write(orphan_root.join("orphan-part"), b"orphan").expect("orphan fixture should be written");
std::fs::write(xlmeta_root.join(STORAGE_FORMAT_FILE), b"invalid-xlmeta").expect("xl.meta fixture should be written");
let mut orphan_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
let mut xlmeta_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
tokio::time::advance(Duration::from_secs(60)).await;
let orphan = scan_metadata_less_residue_with_budget(&orphan_root, &mut orphan_budget)
.await
.expect("delayed orphan scan should complete");
assert!(orphan.has_residue_without_xlmeta());
assert!(!orphan.diagnostic_truncated);
let xlmeta = scan_metadata_less_residue_with_budget(&xlmeta_root, &mut xlmeta_budget)
.await
.expect("delayed xl.meta scan should complete");
assert!(xlmeta.xlmeta_found);
assert!(!xlmeta.diagnostic_truncated);
}
#[tokio::test(start_paused = true)]
async fn bucket_namespace_operation_fails_closed_after_lease_expiry() {
let ttl = Duration::from_millis(20);
@@ -1313,6 +1437,175 @@ mod tests {
assert!(sample.ends_with("/part.1"));
}
#[tokio::test]
async fn metadata_less_residue_scan_shares_one_entry_budget_across_roots() {
let root = tempfile::tempdir().expect("temporary diagnostic roots should be created");
let first_root = root.path().join("disk-a");
let second_root = root.path().join("disk-b");
tokio::fs::create_dir_all(&first_root)
.await
.expect("first diagnostic root should be created");
tokio::fs::create_dir_all(&second_root)
.await
.expect("second diagnostic root should be created");
for index in 0..3 {
std::fs::write(first_root.join(format!("first-{index}")), b"").expect("first-root fixture should be written");
std::fs::write(second_root.join(format!("second-{index}")), b"").expect("second-root fixture should be written");
}
let mut budget = BucketDeleteDiagnosticBudget::with_limits(4, Duration::from_secs(5));
let first = scan_metadata_less_residue_with_budget(&first_root, &mut budget)
.await
.expect("first root should fit the shared budget");
assert!(!first.diagnostic_truncated);
let second = scan_metadata_less_residue_with_budget(&second_root, &mut budget)
.await
.expect("second root should stop at the remaining shared budget");
assert!(second.diagnostic_truncated);
assert!(first.entries_scanned + second.entries_scanned <= 4);
}
#[tokio::test]
async fn metadata_less_residue_scan_honors_an_expired_request_deadline() {
let root = tempfile::tempdir().expect("temporary diagnostic root should be created");
std::fs::write(root.path().join("orphan"), b"").expect("deadline fixture should be written");
let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::ZERO);
let scan = scan_metadata_less_residue_with_budget(root.path(), &mut budget)
.await
.expect("an expired diagnostic budget should fail closed without an IO error");
assert!(scan.diagnostic_truncated);
assert_eq!(scan.entries_scanned, 0);
assert_eq!(scan.diagnostic_bytes_read, 0);
}
#[tokio::test]
async fn metadata_less_residue_scan_stops_at_diagnostic_budget() {
let root = tempfile::tempdir().expect("temporary bucket root should be created");
let bucket_path = root.path().join("bucket");
tokio::fs::create_dir_all(&bucket_path)
.await
.expect("budget fixture directory should be created");
for index in 0..(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES + 32) {
std::fs::write(bucket_path.join(format!("orphan-{index:05}")), b"").expect("budget fixture file should be created");
}
let residue = scan_metadata_less_residue(&bucket_path)
.await
.expect("budgeted residue scan should fail closed without an IO error");
assert!(residue.diagnostic_truncated);
assert!(residue.has_residue_without_xlmeta());
assert!(!residue.xlmeta_found);
assert!(residue.entries_scanned <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
assert!(residue.files <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
assert_eq!(residue.diagnostic_bytes_read, 0);
}
#[tokio::test]
async fn bucket_residue_scan_distinguishes_visible_and_tier_free_xlmeta() {
let root = tempfile::tempdir().expect("temporary bucket root should be created");
let bucket_path = root.path().join("bucket");
let visible_path = bucket_path.join("visible").join(STORAGE_FORMAT_FILE);
tokio::fs::create_dir_all(visible_path.parent().expect("visible xl.meta should have a parent"))
.await
.expect("visible object directory should be created");
let mut visible = FileMeta::new();
visible
.add_version(FileInfo {
version_id: Some(Uuid::new_v4()),
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
})
.expect("visible version should encode");
tokio::fs::write(&visible_path, visible.marshal_msg().expect("visible xl.meta should marshal"))
.await
.expect("visible xl.meta should be written");
let visible_scan = scan_metadata_less_residue(&bucket_path)
.await
.expect("visible xl.meta scan should succeed");
assert_eq!(visible_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::VisibleVersion));
tokio::fs::remove_dir_all(bucket_path.join("visible"))
.await
.expect("visible fixture should be removed");
let free_path = bucket_path.join("free").join(STORAGE_FORMAT_FILE);
tokio::fs::create_dir_all(free_path.parent().expect("free xl.meta should have a parent"))
.await
.expect("free-version object directory should be created");
let source_version_id = Uuid::new_v4();
let mut free = FileMeta::new();
free.add_version(FileInfo {
version_id: Some(source_version_id),
transition_status: TRANSITION_COMPLETE.to_string(),
transitioned_objname: "remote/object".to_string(),
transition_version_id: Some(Uuid::new_v4()),
transition_tier: "WARM".to_string(),
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
})
.expect("transitioned source should encode");
let mut delete = FileInfo {
version_id: Some(source_version_id),
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
};
delete.set_tier_free_version_id(&Uuid::new_v4().to_string());
free.delete_version(&delete)
.expect("transitioned source delete should create a free-version");
tokio::fs::write(&free_path, free.marshal_msg().expect("free-version xl.meta should marshal"))
.await
.expect("free-version xl.meta should be written");
let free_scan = scan_metadata_less_residue(&bucket_path)
.await
.expect("free-version xl.meta scan should succeed");
assert_eq!(free_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::TierFreeVersion));
tokio::fs::remove_dir_all(bucket_path.join("free"))
.await
.expect("free-version fixture should be removed");
let exact_limit_path = bucket_path.join("exact-limit").join(STORAGE_FORMAT_FILE);
tokio::fs::create_dir_all(exact_limit_path.parent().expect("exact-limit xl.meta should have a parent"))
.await
.expect("exact-limit object directory should be created");
let exact_limit = tokio::fs::File::create(&exact_limit_path)
.await
.expect("exact-limit xl.meta should be created");
exact_limit
.set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES)
.await
.expect("exact-limit xl.meta should be extended without allocating its contents");
let exact_limit_scan = scan_metadata_less_residue(&bucket_path)
.await
.expect("exact-limit xl.meta scan should remain fail closed");
assert_eq!(exact_limit_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
assert_eq!(exact_limit_scan.diagnostic_bytes_read, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
tokio::fs::remove_dir_all(bucket_path.join("exact-limit"))
.await
.expect("exact-limit fixture should be removed");
let oversized_path = bucket_path.join("oversized").join(STORAGE_FORMAT_FILE);
tokio::fs::create_dir_all(oversized_path.parent().expect("oversized xl.meta should have a parent"))
.await
.expect("oversized object directory should be created");
let oversized = tokio::fs::File::create(&oversized_path)
.await
.expect("oversized xl.meta should be created");
oversized
.set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES + 1)
.await
.expect("oversized xl.meta should be extended without allocating its contents");
let oversized_scan = scan_metadata_less_residue(&bucket_path)
.await
.expect("oversized xl.meta scan should remain fail closed");
assert_eq!(oversized_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
assert_eq!(oversized_scan.diagnostic_bytes_read, 0);
assert!(oversized_scan.diagnostic_bytes_read <= BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
}
#[tokio::test]
#[serial]
async fn scanner_bucket_listing_unions_every_erasure_set() {
File diff suppressed because it is too large Load Diff
+177 -10
View File
@@ -65,7 +65,7 @@ use http::HeaderMap;
use lazy_static::lazy_static;
use rand::RngExt as _;
use rustfs_config::server_config::Config;
use rustfs_filemeta::FileInfo;
use rustfs_filemeta::{FileInfo, FileMeta};
use rustfs_heal_contracts::heal_channel::{HealItemType, HealOpts};
use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
use rustfs_madmin::heal_commands::HealResultItem;
@@ -88,25 +88,105 @@ type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
pub const SCANNER_PUBLICATION_LEASE_TTL_MS: u64 = 60_000;
pub(crate) const BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES: u64 = 1024 * 1024;
pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES: usize = 4_096;
pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED: Duration = Duration::from_millis(100);
#[derive(Debug)]
pub(crate) struct BucketDeleteDiagnosticBudget {
deadline: Option<tokio::time::Instant>,
max_elapsed: Duration,
entries_remaining: usize,
}
impl BucketDeleteDiagnosticBudget {
pub(crate) fn new() -> Self {
Self::with_limits(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED)
}
fn with_limits(entries: usize, elapsed: Duration) -> Self {
Self {
deadline: None,
max_elapsed: elapsed,
entries_remaining: entries,
}
}
fn deadline(&mut self) -> tokio::time::Instant {
let max_elapsed = self.max_elapsed;
*self.deadline.get_or_insert_with(|| tokio::time::Instant::now() + max_elapsed)
}
fn claim_entry(&mut self) -> bool {
if self.entries_remaining == 0 {
return false;
}
let deadline = self.deadline();
if tokio::time::Instant::now() >= deadline {
return false;
}
self.entries_remaining -= 1;
true
}
async fn run_io<T, F>(&mut self, future: F) -> std::io::Result<Option<T>>
where
F: std::future::Future<Output = std::io::Result<T>>,
{
let deadline = self.deadline();
if tokio::time::Instant::now() >= deadline {
return Ok(None);
}
match tokio::time::timeout_at(deadline, future).await {
Ok(result) => result.map(Some),
Err(_) => Ok(None),
}
}
}
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub(crate) struct BucketMetadataLessResidue {
pub(crate) xlmeta_found: bool,
pub(crate) xlmeta_blocker: Option<BucketDeleteBlockerKind>,
pub(crate) files: usize,
pub(crate) uuid_data_dirs: usize,
pub(crate) entries_scanned: usize,
pub(crate) diagnostic_bytes_read: u64,
pub(crate) diagnostic_truncated: bool,
pub(crate) sample: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum BucketDeleteBlockerKind {
VisibleVersion,
TierFreeVersion,
UnknownXlMeta,
OrphanDirectory,
DiagnosticBudgetExceeded,
}
impl BucketDeleteBlockerKind {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::VisibleVersion => "visible_version",
Self::TierFreeVersion => "tier_free_version",
Self::UnknownXlMeta => "unknown_xlmeta",
Self::OrphanDirectory => "orphan_directory",
Self::DiagnosticBudgetExceeded => "diagnostic_budget_exceeded",
}
}
}
impl BucketMetadataLessResidue {
pub(crate) fn has_residue_without_xlmeta(&self) -> bool {
!self.xlmeta_found && self.files > 0
!self.xlmeta_found && (self.files > 0 || self.diagnostic_truncated)
}
pub(crate) fn describe(&self) -> String {
let sample = self.sample.as_deref().unwrap_or("<none>");
format!(
"metadata-less on-disk residue remains after empty-bucket verification: files={}, uuid_data_dirs={}, sample={sample}",
self.files, self.uuid_data_dirs
"metadata-less on-disk residue remains after empty-bucket verification: files={}, uuid_data_dirs={}, entries_scanned={}, diagnostic_bytes_read={}, diagnostic_truncated={}, sample={sample}",
self.files, self.uuid_data_dirs, self.entries_scanned, self.diagnostic_bytes_read, self.diagnostic_truncated,
)
}
}
@@ -145,28 +225,112 @@ pub(crate) async fn has_xlmeta_files(path: &std::path::Path) -> std::io::Result<
Ok(false)
}
#[cfg(test)]
pub(crate) async fn scan_metadata_less_residue(path: &std::path::Path) -> std::io::Result<BucketMetadataLessResidue> {
let mut budget = BucketDeleteDiagnosticBudget::new();
scan_metadata_less_residue_with_budget(path, &mut budget).await
}
async fn scan_metadata_less_residue_with_budget(
path: &std::path::Path,
budget: &mut BucketDeleteDiagnosticBudget,
) -> std::io::Result<BucketMetadataLessResidue> {
use crate::disk::STORAGE_FORMAT_FILE;
use tokio::fs;
use tokio::io::AsyncReadExt as _;
let mut scan = BucketMetadataLessResidue::default();
let mut stack = vec![path.to_path_buf()];
let mark_budget_exhausted = |scan: &mut BucketMetadataLessResidue| {
scan.diagnostic_truncated = true;
scan.sample.get_or_insert_with(|| "<diagnostic-budget-exceeded>".to_string());
};
while let Some(current_path) = stack.pop() {
let mut entries = match fs::read_dir(&current_path).await {
Ok(entries) => entries,
let mut entries = match budget.run_io(fs::read_dir(&current_path)).await {
Ok(Some(entries)) => entries,
Ok(None) => {
mark_budget_exhausted(&mut scan);
return Ok(scan);
}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
Err(err) => return Err(err),
};
while let Some(entry) = entries.next_entry().await? {
let file_type = entry.file_type().await?;
loop {
let entry = match budget.run_io(entries.next_entry()).await? {
Some(Some(entry)) => entry,
Some(None) => break,
None => {
mark_budget_exhausted(&mut scan);
return Ok(scan);
}
};
if !budget.claim_entry() {
mark_budget_exhausted(&mut scan);
return Ok(scan);
}
scan.entries_scanned = scan.entries_scanned.saturating_add(1);
let Some(file_type) = budget.run_io(entry.file_type()).await? else {
mark_budget_exhausted(&mut scan);
return Ok(scan);
};
let file_name = entry.file_name();
let file_name_str = file_name.to_string_lossy();
if file_name_str == STORAGE_FORMAT_FILE {
scan.xlmeta_found = true;
continue;
if scan.xlmeta_blocker.is_none() {
let entry_path = entry.path();
let Some(metadata) = budget.run_io(fs::metadata(&entry_path)).await? else {
mark_budget_exhausted(&mut scan);
scan.xlmeta_blocker = Some(BucketDeleteBlockerKind::DiagnosticBudgetExceeded);
return Ok(scan);
};
scan.xlmeta_blocker = Some(if metadata.len() > BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES {
BucketDeleteBlockerKind::UnknownXlMeta
} else {
match budget.run_io(fs::File::open(&entry_path)).await {
Ok(Some(file)) => {
let mut data = Vec::new();
let read = budget
.run_io(file.take(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES).read_to_end(&mut data))
.await;
scan.diagnostic_bytes_read = data.len() as u64;
match read {
Ok(Some(_)) => match FileMeta::load(&data) {
Ok(meta)
if !meta.versions.is_empty()
&& meta.versions.iter().all(|version| version.header.free_version()) =>
{
BucketDeleteBlockerKind::TierFreeVersion
}
Ok(meta) if !meta.versions.is_empty() => BucketDeleteBlockerKind::VisibleVersion,
Ok(_) | Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
},
Ok(None) => {
mark_budget_exhausted(&mut scan);
BucketDeleteBlockerKind::DiagnosticBudgetExceeded
}
Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
}
}
Ok(None) => {
mark_budget_exhausted(&mut scan);
BucketDeleteBlockerKind::DiagnosticBudgetExceeded
}
Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
}
});
let sample = entry_path
.strip_prefix(path)
.unwrap_or(entry_path.as_path())
.to_string_lossy()
.replace(std::path::MAIN_SEPARATOR, "/");
scan.sample = Some(sample);
}
return Ok(scan);
}
if file_type.is_dir() {
@@ -222,7 +386,10 @@ pub(crate) mod init_format;
pub(crate) mod list_objects;
mod multipart;
mod object;
pub(crate) use object::{ObjectLockDiagGuard, SourceCleanupMutationFence, tiered_data_movement_source_matches};
pub(crate) use object::{
DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence,
SourceCleanupMutationFence, tiered_data_movement_source_matches,
};
pub use object::{
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
SnapshotConsistencyError,
+69 -1
View File
@@ -846,6 +846,7 @@ impl ECStore {
.await
}
#[cfg(all(test, feature = "test-util"))]
pub(crate) async fn complete_multipart_upload_for_data_movement(
self: Arc<Self>,
target: (usize, Option<&ObjectLockDiagGuard>),
@@ -854,6 +855,44 @@ impl ECStore {
upload_id: &str,
uploaded_parts: Vec<CompletePart>,
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
self.complete_multipart_upload_for_data_movement_inner(target, bucket, object, upload_id, uploaded_parts, opts, None)
.await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn complete_multipart_upload_for_data_movement_with_publication_fence(
self: Arc<Self>,
target_pool_idx: usize,
bucket: &str,
object: &str,
upload_id: &str,
uploaded_parts: Vec<CompletePart>,
opts: &ObjectOptions,
publication_fence: RemoteTuplePublicationFence,
) -> Result<ObjectInfo> {
self.complete_multipart_upload_for_data_movement_inner(
(target_pool_idx, None),
bucket,
object,
upload_id,
uploaded_parts,
opts,
Some(publication_fence),
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn complete_multipart_upload_for_data_movement_inner(
self: Arc<Self>,
target: (usize, Option<&ObjectLockDiagGuard>),
bucket: &str,
object: &str,
upload_id: &str,
uploaded_parts: Vec<CompletePart>,
opts: &ObjectOptions,
publication_fence: Option<RemoteTuplePublicationFence>,
) -> Result<ObjectInfo> {
let (target_pool_idx, mutation_fence) = target;
check_complete_multipart_args(bucket, object, upload_id)?;
@@ -885,8 +924,36 @@ impl ECStore {
snapshot.add_lock_fences(&mut opts);
opts.object_lock_config_snapshot = Some(snapshot);
}
self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence)
let fixed_read_anchor = publication_fence
.as_ref()
.and_then(RemoteTuplePublicationFence::fixed_read_anchor_guard);
self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence.or(fixed_read_anchor))
.await;
// NewMultipart/UploadPart are staging only. Acquire and consume the
// non-cloneable publication capability immediately before Complete,
// then retain its guards until Complete has drained the commit path.
let publication_object = encode_dir_object(object);
let publication_guard = match publication_fence {
Some(publication_fence) => {
let guard = publication_fence
.into_commit_guard(target_pool_idx, bucket, &publication_object)
.await?;
guard.add_namespace_lock_fence(&mut opts);
opts.no_lock = true;
Some(guard)
}
None => {
if rustfs_utils::http::metadata_compat::contains_key_str(
&opts.user_defined,
rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
) {
return Err(Error::other(
"data movement multipart completion cannot publish transition ownership without a publication capability",
));
}
None
}
};
#[cfg(test)]
pause_data_movement_multipart_before_selected_completion(bucket).await;
let pool = self
@@ -911,6 +978,7 @@ impl ECStore {
},
)
.await;
drop(publication_guard);
let result = enqueue_transition_after_write(result, LcEventSrc::S3CompleteMultipartUpload).await;
if result.is_ok() {
list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await;
File diff suppressed because it is too large Load Diff
+32
View File
@@ -309,6 +309,17 @@ impl ECStore {
}
pub(super) async fn delete_prefix(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
let dispatch_scope = if opts.tier_delete_journal_api.is_some() {
let incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
let authorization = opts
.tier_delete_dispatch_authorization
.as_ref()
.ok_or_else(|| Error::other("prefix mutation is missing its dispatched-journal authorization"))?;
authorization.ensure_current(bucket, incarnation, object)?;
Some((authorization, incarnation))
} else {
None
};
if opts.lifecycle_delete_all.is_some() {
let mut preflight_opts = opts.clone();
preflight_opts
@@ -317,6 +328,9 @@ impl ECStore {
.ok_or(StorageError::PreconditionFailed)?
.phase = crate::object_api::LifecycleDeleteAllPhase::Preflight;
for pool in &self.pools {
if let Some((authorization, incarnation)) = dispatch_scope {
authorization.ensure_current(bucket, incarnation, object)?;
}
#[cfg(test)]
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Preflight, pool.pool_idx)?;
pool.delete_object(bucket, object, preflight_opts.clone()).await?;
@@ -326,6 +340,9 @@ impl ECStore {
.ok_or(StorageError::PreconditionFailed)?
.lock()
.mark_mutation_started();
if let Some((authorization, incarnation)) = dispatch_scope {
authorization.mark_mutation_started(bucket, incarnation, object)?;
}
let mut non_trigger_opts = opts.clone();
non_trigger_opts
.lifecycle_delete_all
@@ -333,6 +350,9 @@ impl ECStore {
.ok_or(StorageError::PreconditionFailed)?
.phase = crate::object_api::LifecycleDeleteAllPhase::History;
for pool in &self.pools {
if let Some((authorization, incarnation)) = dispatch_scope {
authorization.ensure_current(bucket, incarnation, object)?;
}
#[cfg(test)]
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::History, pool.pool_idx)?;
let mut pool_opts = non_trigger_opts.clone();
@@ -348,6 +368,9 @@ impl ECStore {
.phase = crate::object_api::LifecycleDeleteAllPhase::FinalPreflight;
let mut trigger_pools = Vec::new();
for (pool_index, pool) in self.pools.iter().enumerate() {
if let Some((authorization, incarnation)) = dispatch_scope {
authorization.ensure_current(bucket, incarnation, object)?;
}
#[cfg(test)]
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::FinalPreflight, pool.pool_idx)?;
let result = pool.delete_object(bucket, object, final_preflight_opts.clone()).await?;
@@ -366,6 +389,9 @@ impl ECStore {
.ok_or(StorageError::PreconditionFailed)?
.phase = crate::object_api::LifecycleDeleteAllPhase::Trigger;
for pool_index in trigger_pools {
if let Some((authorization, incarnation)) = dispatch_scope {
authorization.ensure_current(bucket, incarnation, object)?;
}
#[cfg(test)]
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Trigger, pool_index)?;
let mut pool_opts = trigger_opts.clone();
@@ -378,7 +404,13 @@ impl ECStore {
let mut first_error = None;
let mut first_volume_error = None;
let mut has_success = false;
if let Some((authorization, incarnation)) = dispatch_scope {
authorization.mark_mutation_started(bucket, incarnation, object)?;
}
for pool in &self.pools {
if let Some((authorization, incarnation)) = dispatch_scope {
authorization.ensure_current(bucket, incarnation, object)?;
}
let mut opts = opts.clone();
opts.delete_prefix = true;
match pool.delete_object(bucket, object, opts).await {
+23 -1
View File
@@ -697,6 +697,11 @@ impl HealChannelProcessor {
| HealRequestSource::Mrf => true,
});
// `no_lock` is an internal coordination hint. An admin request can
// carry the legacy field over the wire, but cannot use it as ambient
// authority to bypass storage namespace locking.
let no_lock = request.no_lock.unwrap_or(false) && request.source != HealRequestSource::Admin;
// Build HealOptions with all available fields
let options = HealOptions {
scan_mode: request.scan_mode.unwrap_or(HealScanMode::Normal),
@@ -705,7 +710,7 @@ impl HealChannelProcessor {
update_parity: request.update_parity.unwrap_or(true),
recursive,
dry_run: request.dry_run.unwrap_or(false),
no_lock: request.no_lock.unwrap_or(false),
no_lock,
timeout: request.timeout_seconds.map(std::time::Duration::from_secs),
pool_index: request.pool_index,
set_index: request.set_index,
@@ -990,6 +995,23 @@ mod tests {
assert!(heal_request.options.no_lock);
}
#[tokio::test]
async fn test_convert_to_heal_request_admin_cannot_bypass_object_lock() {
let heal_manager = create_test_heal_manager();
let processor = HealChannelProcessor::new(heal_manager);
let channel_request = HealChannelRequest {
id: "admin-no-lock".to_string(),
bucket: "test-bucket".to_string(),
object_prefix: Some("test-object".to_string()),
no_lock: Some(true),
source: HealRequestSource::Admin,
..Default::default()
};
let heal_request = processor.convert_to_heal_request(channel_request).unwrap();
assert!(!heal_request.options.no_lock, "admin nolock must not become storage lock authority");
}
#[tokio::test]
async fn test_convert_to_heal_request_scanner_defaults_recreate_missing_false() {
let heal_manager = create_test_heal_manager();
@@ -1521,6 +1521,8 @@ pub struct TierMutationControlResponse {
pub error_info: ::core::option::Option<::prost::alloc::string::String>,
#[prost(bytes = "bytes", tag = "5")]
pub response_proof: ::prost::bytes::Bytes,
#[prost(enumeration = "TierMutationFailureClass", tag = "6")]
pub failure_class: i32,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct GetLiveEventsRequest {
@@ -1611,6 +1613,35 @@ impl TierMutationPeerState {
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
#[repr(i32)]
pub enum TierMutationFailureClass {
Unspecified = 0,
PreDispatchRejected = 1,
Ambiguous = 2,
}
impl TierMutationFailureClass {
/// String value of the enum field names used in the ProtoBuf definition.
///
/// The values are not transformed in any way and thus are considered stable
/// (if the ProtoBuf definition does not change) and safe for programmatic use.
pub fn as_str_name(&self) -> &'static str {
match self {
Self::Unspecified => "TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED",
Self::PreDispatchRejected => "TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED",
Self::Ambiguous => "TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS",
}
}
/// Creates an enum from field names used in the ProtoBuf definition.
pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
match value {
"TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED" => Some(Self::Unspecified),
"TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED" => Some(Self::PreDispatchRejected),
"TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS" => Some(Self::Ambiguous),
_ => None,
}
}
}
/// Generated client implementations.
pub mod node_service_client {
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
+146 -55
View File
@@ -177,6 +177,7 @@ pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-re
pub const CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-cross-pool-fence-capability-v1\0";
pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024;
pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024;
pub const TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE;
pub const TIER_MUTATION_RPC_MAX_MESSAGE_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE + 4096;
pub fn heal_control_coordinator_epoch(topology_fingerprint: &str) -> Result<u64, &'static str> {
@@ -342,7 +343,16 @@ impl TierMutationRpcPhase {
}
}
pub const TIER_MUTATION_RPC_PROTOCOL_VERSION: u32 = 1;
// Version 2 required peer Prepare to block new tier-reference creators and
// drain their in-flight operation leases. Version 3 additionally binds Abort
// to the canonical Prepare intent so a missing-record Abort can persist an
// identity-bound tombstone and linearize against a delayed Prepare. Version 4
// signs a typed failure classification while retaining the exact v3 proof
// bytes for rolling compatibility.
pub const TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION: u32 = 3;
pub const TIER_MUTATION_RPC_PROTOCOL_VERSION: u32 = 4;
pub const TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE: usize = 1024;
pub const TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE: usize = 4096;
pub fn canonical_tier_mutation_rpc_body(
version: u32,
@@ -374,19 +384,23 @@ pub struct TierMutationRpcResponseProofInput<'a> {
pub state: i32,
pub applied: bool,
pub error_info: Option<&'a str>,
pub failure_class: i32,
}
pub fn canonical_tier_mutation_rpc_response_body(
input: TierMutationRpcResponseProofInput<'_>,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
const DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v1\0";
const V3_DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v1\0";
const V4_DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v2\0";
let phase = input.phase.as_wire_str().as_bytes();
let mutation_id = input.mutation_id.as_bytes();
let error_info = input.error_info.map(str::as_bytes);
let error_info_len = error_info.map_or(0, <[u8]>::len);
let is_v4 = input.version >= TIER_MUTATION_RPC_PROTOCOL_VERSION;
let domain = if is_v4 { V4_DOMAIN } else { V3_DOMAIN };
let mut body = Vec::with_capacity(
DOMAIN.len()
domain.len()
+ 4
+ 8
+ phase.len()
@@ -398,9 +412,10 @@ pub fn canonical_tier_mutation_rpc_response_body(
+ 1
+ 1
+ 8
+ error_info_len,
+ error_info_len
+ if is_v4 { 4 } else { 0 },
);
body.extend_from_slice(DOMAIN);
body.extend_from_slice(domain);
body.extend_from_slice(&input.version.to_be_bytes());
body.extend_from_slice(&u64::try_from(phase.len())?.to_be_bytes());
body.extend_from_slice(phase);
@@ -415,6 +430,9 @@ pub fn canonical_tier_mutation_rpc_response_body(
if let Some(error_info) = error_info {
body.extend_from_slice(error_info);
}
if is_v4 {
body.extend_from_slice(&input.failure_class.to_be_bytes());
}
Ok(body)
}
@@ -2137,10 +2155,10 @@ mod heal_control_tests {
#[cfg(test)]
mod tier_mutation_rpc_tests {
use super::{
TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase, TierMutationRpcResponseProofInput,
canonical_tier_mutation_rpc_body, canonical_tier_mutation_rpc_response_body,
TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION, TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase,
TierMutationRpcResponseProofInput, canonical_tier_mutation_rpc_body, canonical_tier_mutation_rpc_response_body,
};
use crate::proto_gen::node_service::TierMutationPeerState;
use crate::proto_gen::node_service::{TierMutationFailureClass, TierMutationPeerState};
use uuid::uuid;
#[test]
@@ -2155,7 +2173,7 @@ mod tier_mutation_rpc_tests {
)
.expect("small mutation body should encode");
let mut golden = b"rustfs-tier-mutation-rpc-v1\0".to_vec();
golden.extend_from_slice(&1_u32.to_be_bytes());
golden.extend_from_slice(&TIER_MUTATION_RPC_PROTOCOL_VERSION.to_be_bytes());
golden.extend_from_slice(&7_u64.to_be_bytes());
golden.extend_from_slice(b"prepare");
golden.extend_from_slice(mutation_id.as_bytes());
@@ -2165,8 +2183,13 @@ mod tier_mutation_rpc_tests {
assert_ne!(
baseline,
canonical_tier_mutation_rpc_body(2, TierMutationRpcPhase::Prepare, mutation_id, payload)
.expect("small mutation body should encode")
canonical_tier_mutation_rpc_body(
TIER_MUTATION_RPC_PROTOCOL_VERSION + 1,
TierMutationRpcPhase::Prepare,
mutation_id,
payload,
)
.expect("small mutation body should encode")
);
assert_ne!(
baseline,
@@ -2201,11 +2224,27 @@ mod tier_mutation_rpc_tests {
}
#[test]
fn canonical_tier_mutation_response_binds_request_state_and_error() {
fn tier_mutation_v3_request_and_response_golden_bytes_are_unchanged() {
let mutation_id = uuid!("12345678-1234-5678-9abc-def012345678");
let payload = b"canonical-intent-record";
let baseline = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
let request = canonical_tier_mutation_rpc_body(
TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
TierMutationRpcPhase::Prepare,
mutation_id,
payload,
)
.expect("v3 request should encode");
let mut request_golden = b"rustfs-tier-mutation-rpc-v1\0".to_vec();
request_golden.extend_from_slice(&TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION.to_be_bytes());
request_golden.extend_from_slice(&7_u64.to_be_bytes());
request_golden.extend_from_slice(b"prepare");
request_golden.extend_from_slice(mutation_id.as_bytes());
request_golden.extend_from_slice(&u64::try_from(payload.len()).expect("payload length should fit").to_be_bytes());
request_golden.extend_from_slice(payload);
assert_eq!(request, request_golden);
let response = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: payload,
@@ -2213,49 +2252,97 @@ mod tier_mutation_rpc_tests {
state: TierMutationPeerState::Prepared as i32,
applied: true,
error_info: None,
// v3 must ignore the field so its authenticated bytes stay exact.
failure_class: TierMutationFailureClass::PreDispatchRejected as i32,
})
.expect("small mutation response should encode");
.expect("v3 response should encode");
let mut response_golden = b"rustfs-tier-mutation-rpc-response-v1\0".to_vec();
response_golden.extend_from_slice(&TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION.to_be_bytes());
response_golden.extend_from_slice(&7_u64.to_be_bytes());
response_golden.extend_from_slice(b"prepare");
response_golden.extend_from_slice(mutation_id.as_bytes());
response_golden.extend_from_slice(&u64::try_from(payload.len()).expect("payload length should fit").to_be_bytes());
response_golden.extend_from_slice(payload);
response_golden.push(1);
response_golden.extend_from_slice(&(TierMutationPeerState::Prepared as i32).to_be_bytes());
response_golden.push(1);
response_golden.push(0);
response_golden.extend_from_slice(&0_u64.to_be_bytes());
assert_eq!(response, response_golden);
}
#[test]
fn canonical_tier_mutation_v4_response_binds_request_result_and_failure_class() {
let mutation_id = uuid!("12345678-1234-5678-9abc-def012345678");
let payload = b"canonical-intent-record";
let baseline = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: payload,
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("failure"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
})
.expect("small v4 mutation response should encode");
let cases = [
TierMutationRpcResponseProofInput {
version: 2,
version: TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: payload,
success: true,
state: TierMutationPeerState::Prepared as i32,
applied: true,
error_info: None,
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("failure"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Commit,
mutation_id,
canonical_payload: payload,
success: true,
state: TierMutationPeerState::Prepared as i32,
applied: true,
error_info: None,
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("failure"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id: uuid!("22345678-1234-5678-9abc-def012345678"),
canonical_payload: payload,
success: true,
state: TierMutationPeerState::Prepared as i32,
applied: true,
error_info: None,
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("failure"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: b"tampered-intent-record",
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("failure"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: payload,
success: true,
state: TierMutationPeerState::Prepared as i32,
applied: true,
error_info: None,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("failure"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
@@ -2263,39 +2350,43 @@ mod tier_mutation_rpc_tests {
mutation_id,
canonical_payload: payload,
success: false,
state: TierMutationPeerState::Prepared as i32,
applied: true,
error_info: None,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: payload,
success: true,
state: TierMutationPeerState::Committed as i32,
applied: true,
error_info: None,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: payload,
success: true,
state: TierMutationPeerState::Prepared as i32,
applied: false,
error_info: None,
error_info: Some("failure"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: payload,
success: true,
state: TierMutationPeerState::Prepared as i32,
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: true,
error_info: Some("error"),
error_info: Some("failure"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: payload,
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("other failure"),
failure_class: TierMutationFailureClass::Ambiguous as i32,
},
TierMutationRpcResponseProofInput {
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
phase: TierMutationRpcPhase::Prepare,
mutation_id,
canonical_payload: payload,
success: false,
state: TierMutationPeerState::Unspecified as i32,
applied: false,
error_info: Some("failure"),
failure_class: TierMutationFailureClass::PreDispatchRejected as i32,
},
];
for case in cases {
+7
View File
@@ -1068,12 +1068,19 @@ enum TierMutationPeerState {
TIER_MUTATION_PEER_STATE_ABORTED = 3;
}
enum TierMutationFailureClass {
TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED = 0;
TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED = 1;
TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS = 2;
}
message TierMutationControlResponse {
bool success = 1;
TierMutationPeerState state = 2;
bool applied = 3;
optional string error_info = 4;
bytes response_proof = 5;
TierMutationFailureClass failure_class = 6;
}
message GetLiveEventsRequest {
@@ -46,7 +46,7 @@ use storage_api::lifecycle::{
STORAGE_FORMAT_FILE, TRANSITION_PENDING, TransitionCleanupStoreBarrier, TransitionOptions, assert_transition_meta_consistent,
enqueue_transition_for_existing_objects, expire_transitioned_object, free_version_count, get_bucket_metadata,
get_global_tier_config_mgr, init_background_expiry, init_bucket_metadata_sys, init_local_disks, is_err_object_not_found,
is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_tier_delete_journal_entries,
is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_transition_transaction_records,
register_mock_tier_util, update_bucket_metadata, wait_for_free_version_absence,
};
@@ -562,6 +562,29 @@ where
}
}
// Deep transition futures can overflow libtest's default stack before their
// first assertion, so the serial ILM cases use one dedicated test thread.
fn run_large_stack_async_test<F, Fut>(thread_name: &'static str, test_fn: F)
where
F: FnOnce() -> Fut + Send + 'static,
Fut: std::future::Future<Output = ()> + 'static,
{
let handle = std::thread::Builder::new()
.name(thread_name.to_string())
.stack_size(32 * 1024 * 1024)
.spawn(move || {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("large-stack scanner test runtime should build");
runtime.block_on(test_fn());
})
.expect("large-stack scanner test thread should spawn");
if let Err(payload) = handle.join() {
std::panic::resume_unwind(payload);
}
}
mod serial_tests {
use super::*;
@@ -737,10 +760,17 @@ mod serial_tests {
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[test]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
async fn rejected_transition_candidate_is_recovered_from_persisted_delete_journal() {
fn rejected_transition_candidate_is_recovered_from_persisted_transaction() {
run_large_stack_async_test(
"scanner-rejected-transition-transaction",
rejected_transition_candidate_is_recovered_from_persisted_transaction_case,
);
}
async fn rejected_transition_candidate_is_recovered_from_persisted_transaction_case() {
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
@@ -790,21 +820,20 @@ mod serial_tests {
"no cleanup path may delete the candidate while remove failures are enabled"
);
let retained = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let retained = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("tier delete journal recovery should scan the persisted candidate");
assert_eq!(retained.scanned, 1);
assert_eq!(retained.deleted, 0);
assert_eq!(retained.failed, 1);
.expect("transition transaction recovery should scan the persisted candidate");
assert_eq!((retained.scanned, retained.recovered, retained.retained, retained.failed), (1, 0, 0, 1));
assert_eq!(backend.object_count().await, 1, "failed recovery must retain the remote candidate");
backend.set_remove_failure(false);
let recovered = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("tier delete journal recovery should delete the retained candidate");
assert_eq!(recovered.scanned, 1);
assert_eq!(recovered.deleted, 1);
assert_eq!(recovered.failed, 0);
.expect("transition transaction recovery should delete the retained candidate");
assert_eq!(
(recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
(1, 1, 0, 0)
);
let removed_versions = backend.remove_versions().await;
assert!(!removed_versions.is_empty(), "recovery must issue at least one successful delete");
assert!(
@@ -813,10 +842,10 @@ mod serial_tests {
);
assert_eq!(backend.object_count().await, 0, "recovery should remove the rejected remote candidate");
let empty = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("a removed tier delete journal entry should no longer be listed");
assert_eq!(empty.scanned, 0, "successful recovery must remove the persisted journal entry");
.expect("a removed transition transaction should no longer be listed");
assert_eq!(empty.scanned, 0, "successful recovery must remove the persisted transaction");
assert_eq!(
read_object_fully(&ecstore, bucket_name.as_str(), object_name).await,
payload,
@@ -824,10 +853,17 @@ mod serial_tests {
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[test]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
async fn cancelled_before_cleanup_store_resolution_persists_journal() {
fn cancelled_before_cleanup_store_resolution_persists_transaction() {
run_large_stack_async_test(
"scanner-cancelled-transition-transaction",
cancelled_before_cleanup_store_resolution_persists_transaction_case,
);
}
async fn cancelled_before_cleanup_store_resolution_persists_transaction_case() {
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&tier_name).await;
@@ -876,9 +912,9 @@ mod serial_tests {
let retained = tokio::time::timeout(Duration::from_secs(30), async {
loop {
let recovery = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let recovery = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("the cancelled transition journal should be readable");
.expect("the cancelled transition transaction should be readable");
if recovery.scanned > 0 {
break recovery;
}
@@ -887,29 +923,36 @@ mod serial_tests {
})
.await
.expect("Drop should persist the rejected candidate through the saved instance context");
assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 0, 1));
assert_eq!((retained.scanned, retained.recovered, retained.retained, retained.failed), (1, 0, 0, 1));
tokio::time::timeout(Duration::from_secs(5), async {
while backend.exact_remove_count() < 2 {
tokio::task::yield_now().await;
}
})
.await
.expect("Drop cleanup and failed journal recovery must both preserve the exact version constraint");
.expect("Drop cleanup and failed transaction recovery must both preserve the exact version constraint");
let failed_exact_attempts = backend.exact_remove_count();
assert_eq!(
failed_exact_attempts, 2,
"the cancelled task and the first failed recovery must each preserve the exact delete constraint"
);
assert_eq!(backend.object_count().await, 1);
assert!(backend.remove_versions().await.is_empty());
backend.set_remove_failure(false);
let recovered = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("recovery should delete the candidate retained by the cancelled transition");
assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (1, 1, 0));
assert_eq!(
(recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
(1, 1, 0, 0)
);
assert_eq!(backend.remove_versions().await, backend.put_versions().await);
assert_eq!(backend.exact_remove_count(), failed_exact_attempts + 1);
assert_eq!(backend.object_count().await, 0);
let empty = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("successful recovery should remove the cancellation journal");
.expect("successful recovery should remove the cancellation transaction");
assert_eq!(empty.scanned, 0);
assert_eq!(
read_object_fully(&ecstore, bucket_name.as_str(), object_name).await,
@@ -918,10 +961,14 @@ mod serial_tests {
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[test]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
async fn rejected_transition_cleanup_durability_matrix() {
fn rejected_transition_cleanup_durability_matrix() {
run_large_stack_async_test("scanner-transition-cleanup-matrix", rejected_transition_cleanup_durability_matrix_case);
}
async fn rejected_transition_cleanup_durability_matrix_case() {
#[derive(Clone, Copy)]
enum CleanupCase {
Persisted,
@@ -1010,44 +1057,67 @@ mod serial_tests {
CleanupCase::Persisted | CleanupCase::DeleteFallback => {
assert_eq!(backend.remove_versions().await, backend.put_versions().await);
assert_eq!(backend.object_count().await, 0, "cleanup must remove the exact candidate");
let recovery = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let recovery = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("successful cleanup must not retain a journal entry");
assert_eq!(recovery.scanned, 0);
.expect("successful cleanup must leave no failed transition transaction");
assert_eq!(recovery.failed, 0);
assert_eq!(recovery.retained, 0);
assert_eq!(recovery.recovered, recovery.scanned);
let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("successful reconciliation must remove every transition transaction");
assert_eq!(empty.scanned, 0);
}
CleanupCase::RetryPersisted => {
assert!(
!err.to_string().contains("journal retry error"),
"a successful journal retry must preserve the original version-constraint error"
"the transaction path must not surface the removed journal fallback error"
);
assert_eq!(backend.object_count().await, 1);
let retained = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let retained = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("the retried journal should be recoverable");
assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 0, 1));
.expect("the retained transaction should be recoverable");
assert_eq!((retained.scanned, retained.recovered, retained.retained, retained.failed), (1, 0, 0, 1));
backend.set_remove_failure(false);
let recovered = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("recovery should delete the exact retried candidate");
assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (1, 1, 0));
.expect("recovery should delete the exact transaction candidate");
assert_eq!(
(recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
(1, 1, 0, 0)
);
assert_eq!(backend.remove_versions().await, backend.put_versions().await);
assert_eq!(backend.object_count().await, 0);
let empty = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("successful recovery must remove the retried journal");
.expect("successful recovery must remove the retained transaction");
assert_eq!(empty.scanned, 0);
}
CleanupCase::FullyFailed => {
let message = err.to_string();
assert!(message.contains("initial journal error"), "{message}");
assert!(message.contains("cleanup error"), "{message}");
assert!(message.contains("journal retry error"), "{message}");
assert_eq!(backend.object_count().await, 1, "both failed safeguards must leave the candidate visible");
assert!(backend.remove_versions().await.is_empty());
let recovery = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
let retained = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("failed journal writes must not create partial recovery entries");
assert_eq!(recovery.scanned, 0);
.expect("the pre-upload transaction must retain ownership after cleanup failure");
assert_eq!(retained.scanned, 1, "the failed cleanup must keep one durable transaction owner");
assert_eq!(retained.recovered, 0);
assert_eq!(retained.retained + retained.failed, 1);
backend.set_remove_failure(false);
let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("recovery should delete the candidate after the backend becomes available");
assert_eq!(
(recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
(1, 1, 0, 0)
);
assert_eq!(backend.remove_versions().await, backend.put_versions().await);
assert_eq!(backend.object_count().await, 0);
let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
.await
.expect("successful recovery must remove the failed cleanup transaction");
assert_eq!(empty.scanned, 0);
}
}
assert_eq!(
@@ -1062,20 +1132,7 @@ mod serial_tests {
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
fn test_transition_and_restore_flows() {
std::thread::Builder::new()
.name("scanner-transition-restore-flows".to_string())
.stack_size(32 * 1024 * 1024)
.spawn(|| {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("transition and restore test runtime should build");
runtime.block_on(test_transition_and_restore_flows_inner());
})
.expect("transition and restore test thread should spawn")
.join()
.expect("transition and restore test thread should finish");
run_large_stack_async_test("scanner-transition-restore-flows", test_transition_and_restore_flows_inner);
}
async fn test_transition_and_restore_flows_inner() {
+2 -2
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::tier_delete_journal::recover_tier_delete_journal_entries;
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::transition_transaction::recover_transition_transaction_records;
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::{
bucket_lifecycle_audit::LcEventSrc,
bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, expire_transitioned_object, init_background_expiry},
@@ -50,7 +50,7 @@ pub(crate) mod lifecycle {
TransitionCleanupStoreBarrier, TransitionOptions, assert_transition_meta_consistent,
enqueue_transition_for_existing_objects, expire_transitioned_object, free_version_count, get_bucket_metadata,
get_global_tier_config_mgr, init_background_expiry, init_bucket_metadata_sys, init_local_disks, is_err_object_not_found,
is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_tier_delete_journal_entries,
is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_transition_transaction_records,
register_mock_tier_util, update_bucket_metadata, wait_for_free_version_absence,
};
}