fix(ecstore): make transitioned deletes durable (#5644)

* fix(ecstore): make transitioned deletes durable

* fix(ecstore): journal force deletes

* fix(ecstore): journal force deletes
This commit is contained in:
cxymds
2026-08-03 02:00:11 +08:00
committed by GitHub
parent 8a65017f36
commit 2ce670837c
13 changed files with 823 additions and 264 deletions
@@ -5928,6 +5928,8 @@ mod tests {
backend_identity: Some([1; 32]),
version_id_exact: true,
version_state: rustfs_filemeta::TransitionVersionState::Exact,
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
source: None,
};
let err = state
@@ -6040,6 +6042,8 @@ mod tests {
backend_identity: Some([1; 32]),
version_id_exact: true,
version_state: rustfs_filemeta::TransitionVersionState::Exact,
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
source: None,
};
state
@@ -10146,6 +10150,8 @@ mod tests {
backend_identity: Some(identity),
version_id_exact: false,
version_state: rustfs_filemeta::TransitionVersionState::Unknown,
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
source: None,
};
let err = crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
@@ -10185,6 +10191,8 @@ mod tests {
backend_identity: Some(identity),
version_id_exact: true,
version_state: rustfs_filemeta::TransitionVersionState::Exact,
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
source: None,
};
crate::set_disk::cleanup_rejected_transition_upload_durably(
@@ -20,8 +20,10 @@ use tokio_util::sync::CancellationToken;
use tracing::{debug, warn};
use crate::bucket::lifecycle::config_boundary;
use crate::bucket::lifecycle::runtime_boundary;
use crate::bucket::lifecycle::tier_sweeper::{
Jentry, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
Jentry, TierDeleteJournalState, TierDeleteSourceIdentity,
delete_confirmed_transition_candidate_exact_with_manager_and_identity,
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
};
use crate::disk::RUSTFS_META_BUCKET;
@@ -30,7 +32,7 @@ use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader
use crate::services::tier::tier::tier_destination_id_from_metadata;
use crate::storage_api_contracts::{
list::ListOperations as _,
object::{DeletedObject, ObjectIO, ObjectOperations, ObjectToDelete},
object::{DeletedObject, HTTPPreconditions, ObjectIO, ObjectOperations, ObjectToDelete},
range::HTTPRangeSpec,
};
use crate::store::ECStore;
@@ -46,6 +48,7 @@ const TIER_DELETE_JOURNAL_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5;
pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/";
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
@@ -61,13 +64,22 @@ struct PersistedTierDeleteJournalEntry {
version_id_exact: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
version_state: Option<rustfs_filemeta::TransitionVersionState>,
#[serde(default, skip_serializing_if = "Option::is_none")]
state: Option<TierDeleteJournalState>,
#[serde(default, skip_serializing_if = "Option::is_none")]
source: Option<TierDeleteSourceIdentity>,
}
impl PersistedTierDeleteJournalEntry {
fn from_jentry(je: &Jentry) -> Result<Self> {
validate_version_state(je.version_state, &je.version_id, je.version_id_exact)?;
let legacy_unknown = je.version_state == rustfs_filemeta::TransitionVersionState::Unknown;
let version = if legacy_unknown {
let version = if je.source.is_some() || je.state == TierDeleteJournalState::Prepared {
if je.backend_identity.is_none() {
return Err(Error::other("tier delete transaction is missing its backend identity"));
}
TIER_DELETE_JOURNAL_TRANSACTION_VERSION
} else if legacy_unknown {
if je.backend_identity.is_some() {
TIER_DELETE_JOURNAL_VERSION
} else {
@@ -87,6 +99,10 @@ impl PersistedTierDeleteJournalEntry {
backend_identity: je.backend_identity,
version_id_exact: je.version_id_exact.then_some(true),
version_state: (!legacy_unknown).then_some(je.version_state),
state: (version == TIER_DELETE_JOURNAL_TRANSACTION_VERSION).then_some(je.state),
source: (version == TIER_DELETE_JOURNAL_TRANSACTION_VERSION)
.then(|| je.source.clone())
.flatten(),
})
}
@@ -101,14 +117,21 @@ impl PersistedTierDeleteJournalEntry {
}
if self.version != TIER_DELETE_JOURNAL_EXACT_VERSION
&& self.version != TIER_DELETE_JOURNAL_STATE_VERSION
&& self.version != TIER_DELETE_JOURNAL_TRANSACTION_VERSION
&& self.version_id_exact.unwrap_or(false)
{
return Err(Error::other(
"legacy tier delete journal entry has an unsupported exact version constraint",
));
}
let (backend_identity, version_id_exact, version_state) = match self.version {
1 => (None, false, rustfs_filemeta::TransitionVersionState::Unknown),
let (backend_identity, version_id_exact, version_state, state, source) = match self.version {
1 => (
None,
false,
rustfs_filemeta::TransitionVersionState::Unknown,
TierDeleteJournalState::Committed,
None,
),
TIER_DELETE_JOURNAL_VERSION => (
Some(
self.backend_identity
@@ -116,6 +139,8 @@ impl PersistedTierDeleteJournalEntry {
),
false,
rustfs_filemeta::TransitionVersionState::Unknown,
TierDeleteJournalState::Committed,
None,
),
TIER_DELETE_JOURNAL_EXACT_VERSION => {
if self.version_id.is_empty() || self.version_id_exact != Some(true) {
@@ -128,6 +153,8 @@ impl PersistedTierDeleteJournalEntry {
),
true,
rustfs_filemeta::TransitionVersionState::Exact,
TierDeleteJournalState::Committed,
None,
)
}
TIER_DELETE_JOURNAL_STATE_VERSION => {
@@ -143,6 +170,31 @@ impl PersistedTierDeleteJournalEntry {
),
exact,
state,
TierDeleteJournalState::Committed,
None,
)
}
TIER_DELETE_JOURNAL_TRANSACTION_VERSION => {
let state = self
.state
.ok_or_else(|| Error::other("tier delete journal v5 entry is missing its state"))?;
let source = self
.source
.ok_or_else(|| Error::other("tier delete journal v5 entry is missing its source identity"))?;
let exact = self.version_id_exact.unwrap_or(false);
let version_state = self
.version_state
.ok_or_else(|| Error::other("tier delete journal v5 entry is missing its version state"))?;
validate_version_state(version_state, &self.version_id, exact)?;
(
Some(
self.backend_identity
.ok_or_else(|| Error::other("tier delete journal v5 entry is missing its backend identity"))?,
),
exact,
version_state,
state,
Some(source),
)
}
version => return Err(Error::other(format!("unsupported tier delete journal version {version}"))),
@@ -154,6 +206,8 @@ impl PersistedTierDeleteJournalEntry {
backend_identity,
version_id_exact,
version_state,
state,
source,
})
}
}
@@ -201,6 +255,20 @@ pub(crate) fn tier_delete_journal_object_name(je: &Jentry) -> String {
hasher.update([0]);
hasher.update(b"exact-version-id");
}
if let Some(source) = &je.source {
hasher.update([0]);
hasher.update(source.bucket.as_bytes());
hasher.update([0]);
hasher.update(source.object.as_bytes());
hasher.update([0]);
hasher.update(source.version_id.as_deref().unwrap_or_default().as_bytes());
hasher.update([0]);
hasher.update(source.data_dir.as_deref().unwrap_or_default().as_bytes());
hasher.update([0]);
hasher.update(source.etag.as_deref().unwrap_or_default().as_bytes());
hasher.update([0]);
hasher.update(source.mod_time.as_deref().unwrap_or_default().as_bytes());
}
format!(
"{TIER_DELETE_JOURNAL_PREFIX}{}.json",
rustfs_utils::crypto::hex(hasher.finalize().as_slice())
@@ -246,6 +314,66 @@ where
.map_err(std::io::Error::other)
}
pub async fn commit_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = http::HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let mut committed = je.clone();
committed.state = TierDeleteJournalState::Committed;
persist_tier_delete_journal_entry(api, &committed).await
}
pub async fn abort_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
where
S: ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
remove_tier_delete_journal_entry(api, je).await
}
pub async fn abort_prepared_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
let name = tier_delete_journal_object_name(je);
let (data, metadata) = match config_boundary::read_config_with_metadata(api.clone(), &name, &ObjectOptions::default()).await {
Ok(result) => result,
Err(Error::ConfigNotFound) | Err(Error::FileNotFound) => return Ok(()),
Err(err) => return Err(std::io::Error::other(err)),
};
let current = decode_tier_delete_journal_entry(&data).map_err(std::io::Error::other)?;
if current.state != TierDeleteJournalState::Prepared {
return Ok(());
}
let etag = metadata
.etag
.ok_or_else(|| std::io::Error::other("prepared tier delete journal has no entity tag"))?;
match config_boundary::delete_config_if_match(api, &name, &etag).await {
Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
Err(Error::PreconditionFailed) => Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"prepared tier delete journal changed before abort",
)),
Err(err) => Err(std::io::Error::other(err)),
}
}
pub(crate) async fn enqueue_committed_tier_delete_journal_entry(je: &Jentry) -> std::io::Result<()> {
let expiry_state = runtime_boundary::expiry_state_handle();
expiry_state.write().await.enqueue_tier_journal_entry(je)
}
pub async fn remove_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
where
S: ObjectOperations<
@@ -264,6 +392,13 @@ where
}
pub async fn process_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
if je.state == TierDeleteJournalState::Prepared {
return reconcile_prepared_tier_delete_journal_entry(api, je).await;
}
process_committed_tier_delete_journal_entry(api, je).await
}
async fn process_committed_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
if je.version_state == rustfs_filemeta::TransitionVersionState::Unknown {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
@@ -296,6 +431,87 @@ pub async fn process_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -
remove_tier_delete_journal_entry(api, je).await
}
async fn reconcile_prepared_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
let (data, metadata) =
config_boundary::read_config_with_metadata(api.clone(), &tier_delete_journal_object_name(je), &ObjectOptions::default())
.await
.map_err(std::io::Error::other)?;
let current = decode_tier_delete_journal_entry(&data).map_err(std::io::Error::other)?;
if current.state != TierDeleteJournalState::Prepared {
return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"prepared tier delete journal changed before reconciliation",
));
}
let Some(etag) = metadata.etag else {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"prepared tier delete journal has no entity tag",
));
};
let source = je
.source
.as_ref()
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "prepared tier delete journal has no source"))?;
match api
.get_object_info(&source.bucket, &source.object, &source.lookup_options())
.await
{
Ok(info) if source.matches(&info) => {
match config_boundary::delete_config_if_match(api, &tier_delete_journal_object_name(&current), &etag).await {
Ok(()) => Ok(()),
Err(Error::PreconditionFailed) => Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"prepared tier delete journal changed before abort",
)),
Err(err) => Err(std::io::Error::other(err)),
}
}
Ok(_info) if source.has_stable_identity() => {
commit_prepared_tier_delete_journal_entry_if_current(api, current, etag).await
}
Ok(_) => Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"prepared tier delete journal source identity is not sufficient to confirm deletion",
)),
Err(Error::ObjectNotFound(_, _)) | Err(Error::FileNotFound) | Err(Error::FileVersionNotFound) => {
commit_prepared_tier_delete_journal_entry_if_current(api, current, etag).await
}
Err(err) => Err(std::io::Error::other(err)),
}
}
async fn commit_prepared_tier_delete_journal_entry_if_current(
api: Arc<ECStore>,
mut committed: Jentry,
etag: String,
) -> std::io::Result<()> {
committed.state = TierDeleteJournalState::Committed;
let data = encode_tier_delete_journal_entry(&committed).map_err(std::io::Error::other)?;
match config_boundary::save_config_with_opts(
api.clone(),
&tier_delete_journal_object_name(&committed),
data,
&ObjectOptions {
max_parity: true,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(etag),
..Default::default()
}),
..Default::default()
},
)
.await
{
Ok(()) => process_committed_tier_delete_journal_entry(api, &committed).await,
Err(Error::PreconditionFailed) => Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"prepared tier delete journal changed before commit",
)),
Err(err) => Err(std::io::Error::other(err)),
}
}
pub async fn recover_tier_delete_journal_entries(
api: Arc<ECStore>,
limit: usize,
@@ -482,10 +698,13 @@ mod tests {
decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
tier_delete_journal_object_name,
};
use crate::bucket::lifecycle::tier_sweeper::Jentry;
use crate::bucket::lifecycle::tier_sweeper::{Jentry, TierDeleteJournalState, TierDeleteSourceIdentity};
use crate::error::Result;
use crate::object_api::ObjectInfo;
use std::time::Duration;
use time::OffsetDateTime;
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
fn journal_entry() -> Jentry {
Jentry {
@@ -495,6 +714,8 @@ mod tests {
backend_identity: Some([7; 32]),
version_id_exact: true,
version_state: rustfs_filemeta::TransitionVersionState::Exact,
state: TierDeleteJournalState::Committed,
source: None,
}
}
@@ -513,6 +734,55 @@ mod tests {
assert_eq!(decoded.version_state, je.version_state);
}
#[test]
fn tier_delete_transaction_roundtrips_prepared_source_identity() {
let mut je = journal_entry();
je.state = TierDeleteJournalState::Prepared;
je.source = Some(TierDeleteSourceIdentity {
bucket: "bucket".to_string(),
object: "object".to_string(),
version_id: Some("version".to_string()),
versioned: true,
version_suspended: false,
data_dir: Some("data-dir".to_string()),
etag: Some("etag".to_string()),
mod_time: Some("mod-time".to_string()),
});
let encoded = encode_tier_delete_journal_entry(&je).expect("prepared transaction should encode");
let value: serde_json::Value = serde_json::from_slice(&encoded).expect("transaction should be JSON");
assert_eq!(value["version"], serde_json::json!(5));
assert_eq!(value["state"], serde_json::json!("Prepared"));
assert!(value["source"].is_object());
let decoded = decode_tier_delete_journal_entry(&encoded).expect("prepared transaction should decode");
assert_eq!(decoded.state, TierDeleteJournalState::Prepared);
assert_eq!(decoded.source, je.source);
}
#[test]
fn tier_delete_source_identity_rejects_recreated_object() {
let version_id = Uuid::from_u128(1);
let data_dir = Uuid::from_u128(2);
let mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1);
let info = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
version_id: Some(version_id),
data_dir: Some(data_dir),
mod_time: Some(mod_time),
..Default::default()
};
let source = TierDeleteSourceIdentity::from_object_info("bucket", "object", &info, true, false);
assert!(source.matches(&info));
let recreated = ObjectInfo {
data_dir: Some(Uuid::from_u128(3)),
..info
};
assert!(!source.matches(&recreated));
}
#[test]
fn tier_delete_journal_roundtrips_exact_put_response_constraint() {
let mut exact = journal_entry();
@@ -23,10 +23,12 @@ use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryOp;
use crate::bucket::lifecycle::lifecycle::{self, ObjectOpts};
use crate::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry;
use crate::client::signer_error::error_chain_contains_signer_header_marker;
use crate::object_api::ObjectInfo;
use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease};
use crate::storage_api_contracts::lifecycle::TransitionedObject;
use crate::store::ECStore;
use rustfs_utils::get_env_usize;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::any::Any;
use std::collections::VecDeque;
@@ -257,6 +259,8 @@ impl ObjSweeper {
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact
),
version_state: self.transition_version_state,
state: TierDeleteJournalState::Committed,
source: None,
});
}
None
@@ -285,6 +289,76 @@ impl ObjSweeper {
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
pub(crate) enum TierDeleteJournalState {
Prepared,
Committed,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub(crate) struct TierDeleteSourceIdentity {
pub(crate) bucket: String,
pub(crate) object: String,
pub(crate) version_id: Option<String>,
pub(crate) versioned: bool,
pub(crate) version_suspended: bool,
pub(crate) data_dir: Option<String>,
pub(crate) etag: Option<String>,
pub(crate) mod_time: Option<String>,
}
impl TierDeleteSourceIdentity {
pub(crate) fn from_object_info(
bucket: &str,
object: &str,
info: &ObjectInfo,
versioned: bool,
version_suspended: bool,
) -> Self {
Self {
bucket: bucket.to_string(),
object: object.to_string(),
version_id: info.version_id.map(|id| id.to_string()),
versioned,
version_suspended,
data_dir: info.data_dir.map(|id| id.to_string()),
etag: info.etag.clone(),
mod_time: info.mod_time.map(|time| time.to_string()),
}
}
pub(crate) fn lookup_options(&self) -> crate::object_api::ObjectOptions {
crate::object_api::ObjectOptions {
version_id: self.version_id.clone(),
versioned: self.versioned,
version_suspended: self.version_suspended,
..Default::default()
}
}
pub(crate) fn matches(&self, info: &ObjectInfo) -> bool {
if self.bucket != info.bucket {
return false;
}
if let Some(version_id) = &self.version_id {
return info.version_id.map(|id| id.to_string()).as_deref() == Some(version_id.as_str())
&& self.data_dir == info.data_dir.map(|id| id.to_string());
}
if self.data_dir.is_some() {
return self.data_dir == info.data_dir.map(|id| id.to_string());
}
self.etag.is_some()
&& self.etag == info.etag
&& self.mod_time.is_some()
&& self.mod_time == info.mod_time.map(|time| time.to_string())
}
pub(crate) fn has_stable_identity(&self) -> bool {
self.version_id.is_some() || self.data_dir.is_some() || (self.etag.is_some() && self.mod_time.is_some())
}
}
#[derive(Debug, Clone)]
#[allow(unused_assignments)]
pub struct Jentry {
@@ -294,6 +368,8 @@ pub struct Jentry {
pub(crate) backend_identity: Option<TierDestinationId>,
pub(crate) version_id_exact: bool,
pub(crate) version_state: rustfs_filemeta::TransitionVersionState,
pub(crate) state: TierDeleteJournalState,
pub(crate) source: Option<TierDeleteSourceIdentity>,
}
impl ExpiryOp for Jentry {
@@ -554,9 +630,48 @@ pub fn transitioned_force_delete_journal_entry(
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact
),
version_state: transition_version_state,
state: TierDeleteJournalState::Committed,
source: None,
})
}
pub(crate) fn attach_tier_delete_source(
je: &mut Jentry,
bucket: &str,
object: &str,
info: &ObjectInfo,
versioned: bool,
version_suspended: bool,
) {
je.state = TierDeleteJournalState::Prepared;
je.source = Some(TierDeleteSourceIdentity::from_object_info(
bucket,
object,
info,
versioned,
version_suspended,
));
}
pub(crate) fn transitioned_delete_journal_entry_for_source(
version_id: Option<Uuid>,
versioned: bool,
suspended: bool,
bucket: &str,
object: &str,
source: &ObjectInfo,
) -> Option<Jentry> {
let mut je = transitioned_delete_journal_entry(
version_id,
versioned,
suspended,
&source.transitioned_object,
source.transition_version_state,
)?;
attach_tier_delete_source(&mut je, bucket, object, source, versioned, suspended);
Some(je)
}
#[cfg(test)]
mod test {
use crate::client::signer_error::invalid_utf8_header_error;