mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-22 12:26:37 +00:00
fix(lifecycle): safely expire all object versions (#6291)
* fix(lifecycle): safely expire all object versions * fix(lifecycle): preserve delete-all replication purges * fix(lifecycle): remove dead replication journal * fix(ci): avoid lifecycle transition test stack overflow * fix(lifecycle): release recovery locks before tier IO * test(lifecycle): align object-lock error assertions * test(lifecycle): avoid scanner restore stack overflow * test(scanner): avoid stack overflow in transition and restore flow test (#6300) * refactor(scanner): split remote_scanner.rs into stream child module (#6289) Split the 3080-line remote_scanner.rs (47% inline tests) into a canonical foo.rs + foo/ module tree with zero behavior change: - remote_scanner.rs (~320): protocol constants, process statics, and the request decode/validate/admit/preflight/claim API plus root re-exports - remote_scanner/stream.rs (~1340): wire/frame types, replay cache, FrameAuthenticator, serve path, local bucket scan + persist, client scan, and the bounded stream plumbing - remote_scanner/stream/tests.rs (~1470): the inline test module as a child module of stream so it can reach both parents' private items All crate paths are unchanged: lib.rs re-exports (serve_remote_scanner_request, RemoteScannerRequest, ...) resolve through root re-exports, and scanner_io's crate::remote_scanner:: {scan_remote_bucket, RemoteScannerScanSpec, RemoteScannerOutcome} paths resolve through pub(crate) re-exports. Cross-module items gain pub(super), whose scope equals the old single-module privacy domain; no item's effective visibility widens. Code is moved verbatim apart from those markers, per-module import headers, and rustfmt line re-wraps. Co-authored-by: heihutu <heihutu@gmail.com> * refactor(heal): split resume.rs into focused child modules (#6290) Split the 4242-line resume.rs (46% inline tests) into a canonical foo.rs + foo/ module tree with zero behavior change: - resume.rs (~1020): state file constants, PersistThrottle, ResumeState, ResumeManager core (constructors, load/discovery, progress mutators, ordinary persistence) plus root re-exports - resume/replacement.rs (~690): replacement-intent/proof types and the ResumeManager replacement-lifecycle methods - resume/checkpoint.rs (~350): ResumeCheckpoint + CheckpointManager - resume/utils.rs (~310): ResumeUtils statics - resume/tests.rs (~1980): the inline test module as a child module All module paths are unchanged (heal::resume::CheckpointManager and friends resolve through root re-exports), so no consumer inside or outside the crate changes. Items defined in child modules keep module-private visibility; only the ten cross-module helpers gain pub(super), which is not part of the crate API. Code is moved verbatim apart from those visibility markers, four super::storage_api path fixes, and the new per-module import headers. Co-authored-by: heihutu <heihutu@gmail.com> * refactor(scanner): split scanner_io.rs into child modules (#6294) Split the 5369-line scanner_io.rs (39% inline tests) into a canonical scanner_io.rs + scanner_io/ module tree with zero behavior change: - scanner_io.rs (~660): constants, metadata-error constructors, the bucket scan plan, cycle-status classification helpers, the ScannerIO / ScannerIOCache / ScannerIODisk traits, and ScannerCycleResult - scanner_io/dirty_usage.rs (~300): process-wide dirty-usage statics and the acknowledgment protocol - scanner_io/guards.rs (~270): concurrency gauges and RAII guards - scanner_io/cache.rs (~410): scanner cache locks and the snapshot persist/publish path - scanner_io/io_cycle.rs (~390), io_cache.rs (~1160), io_disk.rs (~230): the ECStore / SetDisks / Disk trait implementations - scanner_io/publish_gate_tests.rs (~750) and tests.rs (~1340): the two inline test modules as child modules All crate paths are unchanged: the lib.rs scanner_io re-exports and every crate::scanner_io:: consumer (scanner.rs, remote_scanner, scanner_folder, and cross-crate rustfs users) resolve through root re-exports with their original visibilities (pub stays pub, pub(crate) stays pub(crate)). Cross-module items gain pub(super), whose scope equals the old single-module privacy domain. Code is moved verbatim apart from those markers, per-module import headers, and rustfmt re-wraps. The logging-guardrail nsscanner_disk skip-set_disks rule now points at scanner_io/io_disk.rs where the function moved; the pattern and thresholds are unchanged. Co-authored-by: heihutu <heihutu@gmail.com> * refactor(scanner): split data_usage_define persistence and tests (#6292) Split the 3655-line data_usage_define.rs (59% inline tests) into a canonical foo.rs + foo/ module tree with zero behavior change: - data_usage_define.rs (~950): cache constants and revision helpers, the data-usage tree types, DataUsageCacheInfo with its hand-written Serialize, the in-memory tree operations, dui, and marshal/unmarshal - data_usage_define/persistence.rs (~580): the load/backup/restore ladder (load, try_load_inner, revision_for_path) and the CAS save path with its retry policy and save metrics - data_usage_define/tests.rs (~2155): the inline test module as a child module All module paths are unchanged (the lib.rs data_usage_define::* glob re-export and every crate::data_usage_define:: consumer resolve as before). The hand-written map-encoded Serialize for DataUsageCacheInfo is moved byte-for-byte per the AGENTS.md cross-cutting invariant; on-disk names and the cache key format const stay in the root. Four persistence helpers used by tests gain pub(super), whose scope equals the old single-module privacy domain. Code is moved verbatim apart from those markers, per-module import headers, and rustfmt re-wraps. Co-authored-by: heihutu <heihutu@gmail.com> * chore(deps): bump datafusion to 55.0.0 (#6288) * refactor(heal): split task.rs per heal kind (#6293) * feat(ecstore): batch small file fdatasync commits (#6297) * feat(ecstore): batch small file fdatasync commits Add a default-off experimental file fdatasync group commit path for small rename_data shard directories. The coordinator batches same-disk waiters into one blocking task while preserving per-directory source fsync after shard contents are durable. Co-Authored-By: heihutu <heihutu@gmail.com> * test(e2e): wait for compression S3 readiness Reuse the shared S3 API readiness probe for compression test servers so multipart requests do not race the startup readiness gate after the TCP port opens. Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com> * fix(tier): recover multi-committed mutation intents (#6296) * fix(tier): recover multi-committed mutation intents * fix(tier): recover committed mutations on standalone nodes * test(scanner): avoid stack overflow in transition test --------- Co-authored-by: heihutu <heihutu@gmail.com> Co-authored-by: cxymds <cxymds@gmail.com> --------- Co-authored-by: houseme <housemecn@gmail.com> Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
@@ -4994,6 +4994,9 @@ pub async fn apply_expiry_on_transitioned_object(
|
||||
src: &LcEventSrc,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> bool {
|
||||
if lc_event.action.delete_all() {
|
||||
return apply_expiry_on_non_transitioned_objects(api, oi, lc_event, src, bucket_incarnation_id).await;
|
||||
}
|
||||
let time_ilm = Metrics::time_ilm(lc_event.action);
|
||||
if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src, bucket_incarnation_id).await {
|
||||
return false;
|
||||
@@ -5047,13 +5050,24 @@ pub async fn apply_expiry_on_non_transitioned_objects(
|
||||
if lc_event.action.delete_all() {
|
||||
opts.delete_prefix = true;
|
||||
opts.delete_prefix_object = true;
|
||||
opts.lifecycle_delete_all = Some(crate::object_api::LifecycleDeleteAllRequest {
|
||||
version_id: oi.version_id.filter(|version_id| !version_id.is_nil()),
|
||||
delete_marker: oi.delete_marker,
|
||||
action: lc_event.action,
|
||||
rule_id: lc_event.rule_id.clone(),
|
||||
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
||||
});
|
||||
opts.ensure_lifecycle_delete_all_journal();
|
||||
}
|
||||
|
||||
let time_ilm = Metrics::time_ilm(lc_event.action);
|
||||
|
||||
//debug!("lc_event.action: {:?}", lc_event.action);
|
||||
debug!("expiry_on_non_transitioned_objects opts: {:?}", opts);
|
||||
let mut dobj = match api.delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts).await {
|
||||
let mut dobj = match api
|
||||
.delete_object_with_tier_delete_journal(&oi.bucket, &encode_dir_object(&oi.name), opts)
|
||||
.await
|
||||
{
|
||||
Ok(dobj) => dobj,
|
||||
Err(e) => {
|
||||
error!(
|
||||
@@ -5283,7 +5297,7 @@ mod tests {
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_last_day_stats::LastDayTierStats;
|
||||
use crate::bucket::lifecycle::tier_sweeper::Jentry;
|
||||
use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
|
||||
use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG};
|
||||
use crate::bucket::metadata_sys;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::client::transition_api::ReaderImpl;
|
||||
@@ -5304,6 +5318,7 @@ mod tests {
|
||||
use crate::storage_api_contracts::{
|
||||
bucket::{BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
|
||||
lifecycle::ExpirationOptions,
|
||||
list::ListOperations as _,
|
||||
multipart::MultipartOperations as _,
|
||||
object::{ObjectIO as _, ObjectOperations as _},
|
||||
};
|
||||
@@ -10917,6 +10932,199 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn queued_delete_all_rechecks_a_same_id_rule_moved_into_the_future() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("stale-delete-all-rule-{}", Uuid::new_v4().simple());
|
||||
let object = "object";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
metadata_sys::update(
|
||||
&bucket,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("bucket versioning should be enabled");
|
||||
let lifecycle_xml = |days| {
|
||||
format!(
|
||||
r#"<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>delete-marker-history</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix></Prefix></Filter>
|
||||
<DelMarkerExpiration><Days>{days}</Days></DelMarkerExpiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#
|
||||
)
|
||||
};
|
||||
metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml(1).into_bytes())
|
||||
.await
|
||||
.expect("initial lifecycle rule should be stored");
|
||||
|
||||
let old_time = OffsetDateTime::now_utc() - time::Duration::days(3);
|
||||
let mut reader = PutObjReader::from_vec(b"old version".to_vec());
|
||||
ecstore
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(old_time - time::Duration::hours(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("old version should be stored");
|
||||
let marker = ecstore
|
||||
.delete_object(
|
||||
&bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(old_time),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("delete marker should be created");
|
||||
|
||||
let queued_event = crate::bucket::lifecycle::lifecycle::Event {
|
||||
action: IlmAction::DelMarkerDeleteAllVersionsAction,
|
||||
rule_id: "delete-marker-history".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml(30).into_bytes())
|
||||
.await
|
||||
.expect("updated lifecycle rule should be stored");
|
||||
let incarnation = ecstore
|
||||
.bucket_incarnation_id_from_disk(&bucket)
|
||||
.await
|
||||
.expect("bucket incarnation should be available");
|
||||
|
||||
let deleted = super::apply_expiry_on_non_transitioned_objects(
|
||||
ecstore.clone(),
|
||||
&marker,
|
||||
&queued_event,
|
||||
&LcEventSrc::Scanner,
|
||||
incarnation,
|
||||
)
|
||||
.await;
|
||||
assert!(!deleted, "the stale queued rule must be rejected");
|
||||
let versions = ecstore
|
||||
.clone()
|
||||
.list_object_versions(&bucket, object, None, None, None, 10)
|
||||
.await
|
||||
.expect("remaining versions should be listable");
|
||||
assert_eq!(versions.objects.iter().filter(|version| version.name == object).count(), 2);
|
||||
|
||||
metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml(1).into_bytes())
|
||||
.await
|
||||
.expect("due lifecycle rule should be restored");
|
||||
let deleted = super::apply_expiry_on_non_transitioned_objects(
|
||||
ecstore.clone(),
|
||||
&marker,
|
||||
&queued_event,
|
||||
&LcEventSrc::Scanner,
|
||||
incarnation,
|
||||
)
|
||||
.await;
|
||||
assert!(deleted, "the current due rule should purge marker and history");
|
||||
let versions = ecstore
|
||||
.clone()
|
||||
.list_object_versions(&bucket, object, None, None, None, 10)
|
||||
.await
|
||||
.expect("purged versions should be listable");
|
||||
assert_eq!(versions.objects.iter().filter(|version| version.name == object).count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn queued_expired_object_all_versions_purges_history_through_transitioned_dispatch() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("expired-all-versions-{}", Uuid::new_v4().simple());
|
||||
let object = "object";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
metadata_sys::update(
|
||||
&bucket,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("bucket versioning should be enabled");
|
||||
metadata_sys::update(
|
||||
&bucket,
|
||||
BUCKET_LIFECYCLE_CONFIG,
|
||||
br#"<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>delete-all-versions</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix></Prefix></Filter>
|
||||
<Expiration><Days>1</Days><ExpiredObjectAllVersions>true</ExpiredObjectAllVersions></Expiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#
|
||||
.to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("delete-all lifecycle rule should be stored");
|
||||
|
||||
let old_time = OffsetDateTime::now_utc() - time::Duration::days(3);
|
||||
let mut old_reader = PutObjReader::from_vec(b"old version".to_vec());
|
||||
ecstore
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut old_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(old_time - time::Duration::hours(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("old version should be stored");
|
||||
let mut current_reader = PutObjReader::from_vec(b"current version".to_vec());
|
||||
let mut current = ecstore
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut current_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(old_time),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("current version should be stored");
|
||||
current.transitioned_object.status = crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string();
|
||||
|
||||
let incarnation = ecstore
|
||||
.bucket_incarnation_id_from_disk(&bucket)
|
||||
.await
|
||||
.expect("bucket incarnation should be available");
|
||||
let deleted = super::apply_expiry_on_transitioned_object(
|
||||
ecstore.clone(),
|
||||
¤t,
|
||||
&crate::bucket::lifecycle::lifecycle::Event {
|
||||
action: IlmAction::DeleteAllVersionsAction,
|
||||
rule_id: "delete-all-versions".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
&LcEventSrc::Scanner,
|
||||
incarnation,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(deleted, "delete-all must not degrade to transitioned single-version expiry");
|
||||
let versions = ecstore
|
||||
.list_object_versions(&bucket, object, None, None, None, 10)
|
||||
.await
|
||||
.expect("purged versions should be listable");
|
||||
assert_eq!(versions.objects.iter().filter(|version| version.name == object).count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_object_lifecycle_skips_current_expiration_for_explicit_legal_hold() {
|
||||
let lc = latest_expiration_lifecycle();
|
||||
|
||||
@@ -435,12 +435,88 @@ async fn process_committed_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jen
|
||||
remove_tier_delete_journal_entry(api, je).await
|
||||
}
|
||||
|
||||
async fn reconcile_prepared_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
|
||||
let (data, metadata) =
|
||||
config_boundary::read_config_with_metadata(api.clone(), &tier_delete_journal_object_name(je), &ObjectOptions::default())
|
||||
fn object_info_references_tier_delete(info: &ObjectInfo, je: &Jentry) -> std::io::Result<bool> {
|
||||
if info.transitioned_object.status != rustfs_filemeta::TRANSITION_COMPLETE
|
||||
|| info.transitioned_object.name != je.obj_name
|
||||
|| info.transitioned_object.tier != je.tier_name
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let source_backend_identity = tier_destination_id_from_metadata(&info.user_defined)?;
|
||||
if source_backend_identity.is_some() && source_backend_identity != je.backend_identity {
|
||||
return Ok(false);
|
||||
}
|
||||
if !je.version_id_exact {
|
||||
return Ok(true);
|
||||
}
|
||||
Ok(match info.transition_version_state {
|
||||
rustfs_filemeta::TransitionVersionState::Unknown => true,
|
||||
rustfs_filemeta::TransitionVersionState::KnownDisabled => false,
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact => {
|
||||
info.transitioned_object.version_id == je.version_id
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async fn prepared_tier_delete_has_live_source(
|
||||
api: &ECStore,
|
||||
source: &TierDeleteSourceIdentity,
|
||||
je: &Jentry,
|
||||
) -> std::io::Result<(bool, Vec<crate::store::ObjectLockDiagGuard>)> {
|
||||
let lock_object = rustfs_utils::path::encode_dir_object(&source.object);
|
||||
let mut lock_opts = ObjectOptions::default();
|
||||
let read_guards = api
|
||||
.acquire_all_object_read_locks("tier_delete_journal_recovery", &source.bucket, &lock_object, &mut lock_opts)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
if api.ctx.lock_manager().is_disabled() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier delete journal recovery requires namespace locking",
|
||||
));
|
||||
}
|
||||
let mut has_live_source = false;
|
||||
for pool in &api.pools {
|
||||
let set = pool.get_disks_by_key(&lock_object);
|
||||
let Some(versions) = set
|
||||
.load_file_info_versions_exact(&source.bucket, &source.object)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
.map_err(std::io::Error::other)?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
for version in versions.versions.iter().filter(|version| !version.tier_free_version()) {
|
||||
let info = ObjectInfo::from_file_info(version, &source.bucket, &source.object, source.versioned);
|
||||
if object_info_references_tier_delete(&info, je)? {
|
||||
has_live_source = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if has_live_source {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if read_guards.iter().any(crate::store::ObjectLockDiagGuard::is_lock_lost) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier delete journal recovery object read lock was lost",
|
||||
));
|
||||
}
|
||||
Ok((has_live_source, read_guards))
|
||||
}
|
||||
|
||||
async fn reconcile_prepared_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
|
||||
let journal_name = tier_delete_journal_object_name(je);
|
||||
let (data, metadata) = config_boundary::read_config_with_metadata(api.clone(), &journal_name, &ObjectOptions::default())
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
let current = decode_tier_delete_journal_entry(&data).map_err(std::io::Error::other)?;
|
||||
if tier_delete_journal_object_name(¤t) != journal_name {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"prepared tier delete journal content does not match its object name",
|
||||
));
|
||||
}
|
||||
if current.state != TierDeleteJournalState::Prepared {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
@@ -453,16 +529,21 @@ async fn reconcile_prepared_tier_delete_journal_entry(api: Arc<ECStore>, je: &Je
|
||||
"prepared tier delete journal has no entity tag",
|
||||
));
|
||||
};
|
||||
let source = je
|
||||
let source = current
|
||||
.source
|
||||
.as_ref()
|
||||
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "prepared tier delete journal has no source"))?;
|
||||
match api
|
||||
.get_object_info(&source.bucket, &source.object, &source.lookup_options())
|
||||
.await
|
||||
{
|
||||
Ok(info) if source.matches(&info) => {
|
||||
match config_boundary::delete_config_if_match(api, &tier_delete_journal_object_name(¤t), &etag).await {
|
||||
match prepared_tier_delete_has_live_source(&api, source, ¤t).await {
|
||||
Ok((true, read_guards)) => {
|
||||
if read_guards.iter().any(crate::store::ObjectLockDiagGuard::is_lock_lost) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier delete journal recovery object read lock was lost before abort",
|
||||
));
|
||||
}
|
||||
let result = config_boundary::delete_config_if_match(api, &tier_delete_journal_object_name(¤t), &etag).await;
|
||||
drop(read_guards);
|
||||
match result {
|
||||
Ok(()) => Ok(()),
|
||||
Err(Error::PreconditionFailed) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
@@ -471,17 +552,32 @@ async fn reconcile_prepared_tier_delete_journal_entry(api: Arc<ECStore>, je: &Je
|
||||
Err(err) => Err(std::io::Error::other(err)),
|
||||
}
|
||||
}
|
||||
Ok(_info) if source.has_stable_identity() => {
|
||||
commit_prepared_tier_delete_journal_entry_if_current(api, current, etag).await
|
||||
Ok((false, read_guards)) if source.has_stable_identity() => {
|
||||
if read_guards.iter().any(crate::store::ObjectLockDiagGuard::is_lock_lost) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier delete journal recovery object read lock was lost before commit",
|
||||
));
|
||||
}
|
||||
let mut commit_opts = ObjectOptions::default();
|
||||
for signal in read_guards
|
||||
.iter()
|
||||
.filter_map(crate::store::ObjectLockDiagGuard::lock_lost_signal)
|
||||
{
|
||||
commit_opts.add_namespace_lock_lost_signal(signal);
|
||||
}
|
||||
let committed =
|
||||
commit_prepared_tier_delete_journal_entry_if_current(api.clone(), current, etag, &commit_opts).await?;
|
||||
// Keep namespace locks only through the journal CAS. Remote-tier IO
|
||||
// must not block writers for the object during recovery.
|
||||
drop(read_guards);
|
||||
process_committed_tier_delete_journal_entry(api, &committed).await
|
||||
}
|
||||
Ok(_) => Err(std::io::Error::new(
|
||||
Ok((false, _read_guards)) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"prepared tier delete journal source identity is not sufficient to confirm deletion",
|
||||
)),
|
||||
Err(Error::ObjectNotFound(_, _)) | Err(Error::FileNotFound) | Err(Error::FileVersionNotFound) => {
|
||||
commit_prepared_tier_delete_journal_entry_if_current(api, current, etag).await
|
||||
}
|
||||
Err(err) => Err(std::io::Error::other(err)),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -489,7 +585,8 @@ async fn commit_prepared_tier_delete_journal_entry_if_current(
|
||||
api: Arc<ECStore>,
|
||||
mut committed: Jentry,
|
||||
etag: String,
|
||||
) -> std::io::Result<()> {
|
||||
lock_opts: &ObjectOptions,
|
||||
) -> std::io::Result<Jentry> {
|
||||
committed.state = TierDeleteJournalState::Committed;
|
||||
let data = encode_tier_delete_journal_entry(&committed).map_err(std::io::Error::other)?;
|
||||
match config_boundary::save_config_with_opts(
|
||||
@@ -502,12 +599,13 @@ async fn commit_prepared_tier_delete_journal_entry_if_current(
|
||||
if_match: Some(etag),
|
||||
..Default::default()
|
||||
}),
|
||||
namespace_lock_fence: lock_opts.namespace_lock_fence.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => process_committed_tier_delete_journal_entry(api, &committed).await,
|
||||
Ok(()) => Ok(committed),
|
||||
Err(Error::PreconditionFailed) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"prepared tier delete journal changed before commit",
|
||||
@@ -582,6 +680,18 @@ pub async fn recover_tier_delete_journal_entries(
|
||||
}
|
||||
};
|
||||
|
||||
if tier_delete_journal_object_name(&je) != object.name {
|
||||
stats.failed += 1;
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
journal_object = %object.name,
|
||||
"Tier delete journal content does not match its object name and will be retained"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if je.backend_identity.is_none() {
|
||||
stats.failed += 1;
|
||||
warn!(
|
||||
@@ -699,16 +809,14 @@ where
|
||||
mod tests {
|
||||
use super::{
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_STATE_VERSION, await_tier_delete_journal_recovery,
|
||||
decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
|
||||
tier_delete_journal_object_name,
|
||||
decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, object_info_references_tier_delete,
|
||||
record_tier_delete_journal_backend_identity, tier_delete_journal_object_name,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::{Jentry, TierDeleteJournalState, TierDeleteSourceIdentity};
|
||||
use crate::error::Result;
|
||||
use crate::object_api::ObjectInfo;
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn journal_entry() -> Jentry {
|
||||
Jentry {
|
||||
@@ -738,6 +846,16 @@ mod tests {
|
||||
assert_eq!(decoded.version_state, je.version_state);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_object_name_binds_persisted_content() {
|
||||
let original = journal_entry();
|
||||
let original_name = tier_delete_journal_object_name(&original);
|
||||
let mut replaced = original;
|
||||
replaced.obj_name = "remote/replaced".to_string();
|
||||
|
||||
assert_ne!(tier_delete_journal_object_name(&replaced), original_name);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_transaction_roundtrips_prepared_source_identity() {
|
||||
let mut je = journal_entry();
|
||||
@@ -765,26 +883,35 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_source_identity_rejects_recreated_object() {
|
||||
let version_id = Uuid::from_u128(1);
|
||||
let data_dir = Uuid::from_u128(2);
|
||||
let mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1);
|
||||
let info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(version_id),
|
||||
data_dir: Some(data_dir),
|
||||
mod_time: Some(mod_time),
|
||||
fn prepared_recovery_blocks_any_live_reference_to_the_remote_version() {
|
||||
let je = journal_entry();
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::crypto::hex(je.backend_identity.expect("test journal should bind a backend")),
|
||||
);
|
||||
let mut info = ObjectInfo {
|
||||
user_defined: std::sync::Arc::new(metadata),
|
||||
transitioned_object: crate::storage_api_contracts::lifecycle::TransitionedObject {
|
||||
name: je.obj_name.clone(),
|
||||
version_id: je.version_id.clone(),
|
||||
tier: je.tier_name.clone(),
|
||||
status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
..Default::default()
|
||||
};
|
||||
let source = TierDeleteSourceIdentity::from_object_info("bucket", "object", &info, true, false);
|
||||
assert!(source.matches(&info));
|
||||
|
||||
let recreated = ObjectInfo {
|
||||
data_dir: Some(Uuid::from_u128(3)),
|
||||
..info
|
||||
};
|
||||
assert!(!source.matches(&recreated));
|
||||
assert!(object_info_references_tier_delete(&info, &je).expect("matching reference should be valid"));
|
||||
info.transitioned_object.version_id = "other-version".to_string();
|
||||
assert!(!object_info_references_tier_delete(&info, &je).expect("different exact version should be valid"));
|
||||
info.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
assert!(
|
||||
object_info_references_tier_delete(&info, &je).expect("legacy unknown reference should fail closed"),
|
||||
"an unknown live source may still reference the journaled remote version"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -334,32 +334,6 @@ impl TierDeleteSourceIdentity {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn lookup_options(&self) -> crate::object_api::ObjectOptions {
|
||||
crate::object_api::ObjectOptions {
|
||||
version_id: self.version_id.clone(),
|
||||
versioned: self.versioned,
|
||||
version_suspended: self.version_suspended,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn matches(&self, info: &ObjectInfo) -> bool {
|
||||
if self.bucket != info.bucket {
|
||||
return false;
|
||||
}
|
||||
if let Some(version_id) = &self.version_id {
|
||||
return info.version_id.map(|id| id.to_string()).as_deref() == Some(version_id.as_str())
|
||||
&& self.data_dir == info.data_dir.map(|id| id.to_string());
|
||||
}
|
||||
if self.data_dir.is_some() {
|
||||
return self.data_dir == info.data_dir.map(|id| id.to_string());
|
||||
}
|
||||
self.etag.is_some()
|
||||
&& self.etag == info.etag
|
||||
&& self.mod_time.is_some()
|
||||
&& self.mod_time == info.mod_time.map(|time| time.to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn has_stable_identity(&self) -> bool {
|
||||
self.version_id.is_some() || self.data_dir.is_some() || (self.etag.is_some() && self.mod_time.is_some())
|
||||
}
|
||||
|
||||
@@ -23,6 +23,8 @@ use super::replication_queue_boundary::DeletedObjectReplicationInfo;
|
||||
use super::replication_storage_boundary::{
|
||||
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, deleted_object_for_replication,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use std::sync::Mutex;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
@@ -32,6 +34,9 @@ pub(crate) type ReplicationLifecycleConfig = ReplicationConfig;
|
||||
|
||||
pub(crate) struct ReplicationLifecycleBridge;
|
||||
|
||||
#[cfg(test)]
|
||||
static SCHEDULED_DELETE_OBJECTS: Mutex<Vec<DeletedObject>> = Mutex::new(Vec::new());
|
||||
|
||||
impl ReplicationLifecycleBridge {
|
||||
#[allow(
|
||||
dead_code,
|
||||
@@ -85,6 +90,13 @@ impl ReplicationLifecycleBridge {
|
||||
}
|
||||
|
||||
pub(crate) async fn schedule_delete(bucket: String, delete_object: DeletedObject) {
|
||||
#[cfg(test)]
|
||||
{
|
||||
SCHEDULED_DELETE_OBJECTS
|
||||
.lock()
|
||||
.expect("scheduled delete test hook lock should not poison")
|
||||
.push(delete_object.clone());
|
||||
}
|
||||
super::replication_pool::schedule_replication_delete(DeletedObjectReplicationInfo {
|
||||
delete_object: deleted_object_for_replication(delete_object),
|
||||
bucket,
|
||||
@@ -93,6 +105,15 @@ impl ReplicationLifecycleBridge {
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn take_scheduled_deletes_for_test() -> Vec<DeletedObject> {
|
||||
std::mem::take(
|
||||
&mut *SCHEDULED_DELETE_OBJECTS
|
||||
.lock()
|
||||
.expect("scheduled delete test hook lock should not poison"),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
Reference in New Issue
Block a user