fix(ecstore): migrate tier free versions during decommission

This commit is contained in:
overtrue
2026-08-23 06:45:57 +08:00
parent a206895fad
commit e1b5c665e8
6 changed files with 394 additions and 118 deletions
+160 -56
View File
@@ -1130,21 +1130,29 @@ fn should_cleanup_decommission_source_entry(decommissioned: usize, total_version
decommissioned.saturating_add(expired) == total_versions decommissioned.saturating_add(expired) == total_versions
} }
/// Disposition reason logged for tier free-version records that decommission const DECOMMISSION_FREE_VERSION_MIGRATED_REASON: &str = "tier_free_version_migrated";
/// skips instead of migrating. const DECOMMISSION_FREE_VERSION_RETAINED_REASON: &str = "tier_free_version_migration_failed";
const DECOMMISSION_FREE_VERSION_SKIP_REASON: &str = "tier_free_version_not_migrated"; const DECOMMISSION_FREE_VERSION_SWEEP_REASON: &str = "tier_free_version_unresolved_after_decommission";
const DECOMMISSION_FREE_VERSION_DISPOSITION_REASON: &str = "tier_free_version_disposition_recorded";
/// Counts the tier free-version records present in a decommission entry #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
/// inventory. The exact loader (`load_file_info_versions_exact`) keeps these struct DecommissionFreeVersionDisposition {
/// records inline in `versions` instead of separating them into migrated: usize,
/// `free_versions`, and the migration loop then routes them through the retained: usize,
/// generic delete-marker path: the free-version flag and its remote-tier }
/// identity are never carried to the target pool, and a lone record is skipped
/// by the empty-delete-marker rule. Accounting for them here keeps the final impl DecommissionFreeVersionDisposition {
/// sweep from silently omitting records whose free-version disposition was fn record_migrated(&mut self) {
/// dropped (see docs/architecture/decommission-compatibility.md). self.migrated += 1;
fn decommission_free_versions_skipped(fivs: &FileInfoVersions) -> usize { }
fivs.versions.iter().filter(|version| version.tier_free_version()).count()
fn record_retained(&mut self) {
self.retained += 1;
}
fn total(self) -> usize {
self.migrated.saturating_add(self.retained)
}
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -2478,6 +2486,7 @@ fn decommission_remote_tiered_opts(
user_defined: version.metadata.clone(), user_defined: version.metadata.clone(),
src_pool_idx, src_pool_idx,
data_movement: true, data_movement: true,
incl_free_versions: version.tier_free_version(),
include_part_checksums: true, include_part_checksums: true,
http_preconditions: Some(crate::data_movement::data_movement_target_precondition()), http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
expected_bucket_incarnation_id, expected_bucket_incarnation_id,
@@ -3114,24 +3123,9 @@ impl ECStore {
fivs.versions fivs.versions
.sort_by_key(|v| (v.mod_time.is_none(), std::cmp::Reverse(v.mod_time))); .sort_by_key(|v| (v.mod_time.is_none(), std::cmp::Reverse(v.mod_time)));
let skipped_free_versions = decommission_free_versions_skipped(&fivs);
if skipped_free_versions > 0 {
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %entry.name,
skipped_free_versions,
reason = DECOMMISSION_FREE_VERSION_SKIP_REASON,
state = "free_versions_skipped",
"Decommission skipped free-version migration"
);
}
let mut decommissioned: usize = 0; let mut decommissioned: usize = 0;
let mut expired: usize = 0; let mut expired: usize = 0;
let mut free_version_disposition = DecommissionFreeVersionDisposition::default();
let mut cleanup_preflight_allowed_missing = Vec::new(); let mut cleanup_preflight_allowed_missing = Vec::new();
for version in fivs.versions.iter() { for version in fivs.versions.iter() {
@@ -3140,6 +3134,80 @@ impl ECStore {
} }
decommission_cancel_signal_result(rx.is_cancelled())?; decommission_cancel_signal_result(rx.is_cancelled())?;
if version.tier_free_version() {
let version_id = version.version_id.map(|v| v.to_string());
let mut migration_error = None;
let mut migrated = false;
for _ in 0..3 {
match self
.decommission_tiered_object(
bucket.as_str(),
&version.name,
version,
&decommission_remote_tiered_opts(version, version_id.clone(), idx, expected_bucket_incarnation_id),
)
.await
{
Ok(()) => {
migrated = true;
migration_error = None;
break;
}
Err(err) if is_decommission_target_capacity_error(&err) => {
return Err(with_decommission_entry_context(
"decommission_tier_free_version",
bucket.as_str(),
version.name.as_str(),
err,
));
}
Err(err) => migration_error = Some(err),
}
}
{
let mut pool_meta = self.pool_meta.write().await;
if let Err(err) = count_decommission_item(&mut pool_meta, idx, 0, !migrated) {
return Err(with_decommission_entry_context(
"count_decommission_item",
bucket.as_str(),
entry.name.as_str(),
err,
));
}
}
if migrated {
decommissioned += 1;
free_version_disposition.record_migrated();
} else {
free_version_disposition.record_retained();
}
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
version_id = ?version_id,
result = ?migration_error,
reason = if migrated {
DECOMMISSION_FREE_VERSION_MIGRATED_REASON
} else {
DECOMMISSION_FREE_VERSION_RETAINED_REASON
},
state = if migrated { "free_version_migrated" } else { "free_version_retained" },
"Decommission free-version disposition recorded"
);
if !migrated {
break;
}
continue;
}
if should_skip_lifecycle_for_data_movement( if should_skip_lifecycle_for_data_movement(
self.clone(), self.clone(),
&bucket, &bucket,
@@ -3427,6 +3495,23 @@ impl ECStore {
} }
} }
if free_version_disposition.total() > 0 {
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %entry.name,
free_versions_migrated = free_version_disposition.migrated,
free_versions_retained = free_version_disposition.retained,
free_versions_total = free_version_disposition.total(),
reason = DECOMMISSION_FREE_VERSION_DISPOSITION_REASON,
state = "free_version_disposition",
"Decommission free-version disposition summary"
);
}
if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) { if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) {
if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) { if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
return Err(Error::other("decommission bucket incarnation fence was lost before source cleanup")); return Err(Error::other("decommission bucket incarnation fence was lost before source cleanup"));
@@ -4393,6 +4478,7 @@ impl ECStore {
let lifecycle_config_cb = lifecycle_config.clone(); let lifecycle_config_cb = lifecycle_config.clone();
let object_lock_config_cb = object_lock_config.clone(); let object_lock_config_cb = object_lock_config.clone();
let store = Arc::clone(self); let store = Arc::clone(self);
let set_cb = Arc::clone(set);
let callback_rx_cb = callback_rx.clone(); let callback_rx_cb = callback_rx.clone();
let callback: ListCallback = Arc::new(move |entry: MetaCacheEntry| { let callback: ListCallback = Arc::new(move |entry: MetaCacheEntry| {
@@ -4402,6 +4488,7 @@ impl ECStore {
let lifecycle_config = lifecycle_config_cb.clone(); let lifecycle_config = lifecycle_config_cb.clone();
let object_lock_config = object_lock_config_cb.clone(); let object_lock_config = object_lock_config_cb.clone();
let store = Arc::clone(&store); let store = Arc::clone(&store);
let set = Arc::clone(&set_cb);
let callback_rx = callback_rx_cb.clone(); let callback_rx = callback_rx_cb.clone();
Box::pin(async move { Box::pin(async move {
if callback_rx.is_cancelled() { if callback_rx.is_cancelled() {
@@ -4416,11 +4503,14 @@ impl ECStore {
return; return;
} }
let fivs = match load_decommission_entry_versions( let fivs = match load_decommission_entry_exact_versions(
&set,
&entry, &entry,
&bucket_name, &bucket_name,
"check_after_decommission.file_info_versions", "check_after_decommission.file_info_versions",
) { )
.await
{
Ok(fivs) => fivs, Ok(fivs) => fivs,
Err(err) => { Err(err) => {
let mut first_err = entry_error.lock().await; let mut first_err = entry_error.lock().await;
@@ -4433,7 +4523,23 @@ impl ECStore {
}; };
let mut remaining = 0; let mut remaining = 0;
for version in &fivs.versions { for version in fivs.versions.iter().chain(fivs.free_versions.iter()) {
if version.tier_free_version() {
remaining += 1;
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket_name,
object = %entry.name,
version_id = ?version.version_id,
reason = DECOMMISSION_FREE_VERSION_SWEEP_REASON,
state = "free_version_retained",
"Decommission final sweep retained a free version"
);
continue;
}
if version.deleted { if version.deleted {
continue; continue;
} }
@@ -4787,6 +4893,12 @@ mod tests {
assert!(opts.include_part_checksums); assert!(opts.include_part_checksums);
assert!(opts.http_preconditions.is_some()); assert!(opts.http_preconditions.is_some());
assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation)); assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
assert!(!opts.incl_free_versions);
let mut free_version = version;
free_version.set_tier_free_version();
let free_opts = decommission_remote_tiered_opts(&free_version, Some("free-version-id".to_string()), 9, Some(incarnation));
assert!(free_opts.incl_free_versions);
} }
#[test] #[test]
@@ -5491,13 +5603,13 @@ pub(crate) fn fallback_free_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usi
#[cfg(test)] #[cfg(test)]
mod pools_tests { mod pools_tests {
use super::{ use super::{
DECOMMISSION_FREE_VERSION_SKIP_REASON, DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DECOMMISSION_FREE_VERSION_MIGRATED_REASON, DECOMMISSION_FREE_VERSION_RETAINED_REASON,
DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF, DecomBucketInfo, DecommissionStartPoolState, DecommissionTerminalState, DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF,
ListCallback, PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, PoolStatus, apply_decommission_status_space_info, DecomBucketInfo, DecommissionFreeVersionDisposition, DecommissionStartPoolState, DecommissionTerminalState, ListCallback,
PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, PoolStatus, apply_decommission_status_space_info,
bind_decommission_cancelers, bind_missing_decommission_cancelers, cancel_decommission_canceler, bind_decommission_cancelers, bind_missing_decommission_cancelers, cancel_decommission_canceler,
classify_decommission_terminal_state, count_decommission_item, decommission_cancel_signal_result, classify_decommission_terminal_state, count_decommission_item, decommission_cancel_signal_result, decommission_item_size,
decommission_free_versions_skipped, decommission_item_size, decommission_meta_bucket_options, decommission_meta_bucket_options, decommission_start_pool_state, dedup_indices, default_decommission_bucket_concurrency,
decommission_start_pool_state, dedup_indices, default_decommission_bucket_concurrency,
ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed, ensure_decommission_listing_disks_available, ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed, ensure_decommission_listing_disks_available,
ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, ensure_decommission_start_keeps_active_pool, ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, ensure_decommission_start_keeps_active_pool,
ensure_decommission_start_local_leader, ensure_decommission_start_pool_states, ensure_decommission_start_local_leader, ensure_decommission_start_pool_states,
@@ -6797,24 +6909,16 @@ mod pools_tests {
} }
#[test] #[test]
fn decommission_free_version_accounting_reports_skipped_records() { fn decommission_free_version_accounting_records_migration_and_retention() {
let mut fivs = FileInfoVersions::default(); let mut disposition = DecommissionFreeVersionDisposition::default();
assert_eq!(decommission_free_versions_skipped(&fivs), 0); disposition.record_migrated();
disposition.record_retained();
fivs.versions.push(FileInfo { assert_eq!(disposition.migrated, 1);
name: "object.txt".to_string(), assert_eq!(disposition.retained, 1);
..Default::default() assert_eq!(disposition.total(), 2);
}); assert_eq!(DECOMMISSION_FREE_VERSION_MIGRATED_REASON, "tier_free_version_migrated");
let mut free_one = FileInfo::default(); assert_eq!(DECOMMISSION_FREE_VERSION_RETAINED_REASON, "tier_free_version_migration_failed");
free_one.set_tier_free_version();
fivs.versions.push(free_one);
let mut free_two = FileInfo::default();
free_two.set_tier_free_version();
free_two.transition_tier = "WARM".to_string();
fivs.versions.push(free_two);
assert_eq!(decommission_free_versions_skipped(&fivs), 2);
assert_eq!(DECOMMISSION_FREE_VERSION_SKIP_REASON, "tier_free_version_not_migrated");
} }
#[test] #[test]
+108
View File
@@ -4642,6 +4642,63 @@ fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_
} }
impl SetDisks { impl SetDisks {
/// Publish an internal tier free-version record without changing its
/// delete-marker shape or remote-tier identity. The caller holds the
/// source and target object locks; a write quorum is required before the
/// source cleanup may remove the original record.
#[tracing::instrument(skip(self, fi, opts))]
pub(crate) async fn decommission_tier_free_version(
&self,
bucket: &str,
object: &str,
fi: &FileInfo,
opts: &ObjectOptions,
) -> Result<()> {
if !fi.deleted || !fi.tier_free_version() {
return Err(Error::other("decommission tier free-version write requires a free version record"));
}
if opts
.namespace_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
|| opts
.bucket_lifecycle_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
{
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "decommission_tier_free_version_commit",
bucket: bucket.to_string(),
object: object.to_string(),
required: 1,
achieved: 0,
});
}
let disks = self.disks.read().await.clone();
let write_quorum = self.default_write_quorum();
let futures = disks.into_iter().map(|disk| {
let file_info = fi.clone();
async move {
if let Some(disk) = disk {
disk.write_metadata("", bucket, object, file_info).await
} else {
Err(DiskError::DiskNotFound)
}
}
});
let mut errs = Vec::new();
for result in join_all(futures).await {
match result {
Ok(_) => errs.push(None),
Err(err) => errs.push(Some(err)),
}
}
resolve_tiered_decommission_write_quorum_result(&errs, write_quorum, bucket, object)
}
#[tracing::instrument(skip(self, fi, opts))] #[tracing::instrument(skip(self, fi, opts))]
pub(crate) async fn decommission_tiered_object( pub(crate) async fn decommission_tiered_object(
&self, &self,
@@ -9864,6 +9921,57 @@ mod tests {
assert_ne!(updated.erasure.distribution, original.erasure.distribution); assert_ne!(updated.erasure.distribution, original.erasure.distribution);
} }
#[tokio::test]
async fn decommission_tier_free_version_preserves_remote_identity() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "free-version-decommission";
let object = "object.txt";
let version_id = Uuid::new_v4();
let mut free_version = FileInfo {
name: object.to_string(),
volume: bucket.to_string(),
version_id: Some(version_id),
mod_time: Some(time::OffsetDateTime::now_utc()),
deleted: true,
transition_tier: "WARM-TIER".to_string(),
transitioned_objname: "remote/object".to_string(),
..Default::default()
};
free_version.set_tier_free_version();
set_disks
.decommission_tier_free_version(bucket, object, &free_version, &ObjectOptions::default())
.await
.expect("free-version metadata should reach the target quorum");
set_disks
.decommission_tier_free_version(bucket, object, &free_version, &ObjectOptions::default())
.await
.expect("replaying the same free-version metadata should be idempotent");
let versions = set_disks
.load_file_info_versions_exact(bucket, object)
.await
.expect("migrated free-version metadata should decode")
.expect("migrated free-version metadata should exist");
let migrated = versions
.versions
.iter()
.find(|version| version.version_id == Some(version_id))
.expect("free version should be present on the target");
assert_eq!(
versions
.versions
.iter()
.filter(|version| version.version_id == Some(version_id))
.count(),
1
);
assert!(migrated.tier_free_version());
assert_eq!(migrated.transition_tier, "WARM-TIER");
assert_eq!(migrated.transitioned_objname, "remote/object");
}
#[test] #[test]
fn test_resolve_tiered_decommission_write_quorum_result_allows_successful_quorum() { fn test_resolve_tiered_decommission_write_quorum_result_allows_successful_quorum() {
let errs = vec![None, None, Some(DiskError::DiskNotFound), None]; let errs = vec![None, None, Some(DiskError::DiskNotFound), None];
+80 -14
View File
@@ -1152,6 +1152,15 @@ fn tiered_data_movement_source_matches(
&& expected_backend == current_backend) && expected_backend == current_backend)
} }
fn decommission_free_version_overwrite_error(bucket: &str, object: &str, version_id: Option<Uuid>) -> Error {
StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
object.to_owned(),
version_id.map(|id| id.to_string()).unwrap_or_default(),
)
.into()
}
fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool { fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
target_pool_idx != src_pool_idx target_pool_idx != src_pool_idx
} }
@@ -1776,6 +1785,43 @@ impl ECStore {
) )
} }
async fn has_equivalent_data_movement_tier_free_version(
&self,
bucket: &str,
object: &str,
source: &rustfs_filemeta::FileInfo,
target_pool_idx: usize,
) -> Result<bool> {
let pool = self
.pools
.get(target_pool_idx)
.ok_or_else(|| Error::other(format!("invalid tiered data movement target pool {target_pool_idx}")))?;
let logical_object = decode_dir_object(object);
let Some(versions) = pool
.get_disks_by_key(object)
.load_file_info_versions_exact(bucket, &logical_object)
.await?
else {
return Ok(false);
};
let Some(target) = versions
.versions
.iter()
.find(|version| version.version_id == source.version_id)
else {
return Ok(false);
};
if !target.tier_free_version() {
return Err(decommission_free_version_overwrite_error(bucket, object, source.version_id));
}
if tiered_data_movement_source_matches(source, target)? {
Ok(true)
} else {
Err(decommission_free_version_overwrite_error(bucket, object, source.version_id))
}
}
fn resolve_decommission_target_pool_idx_result(result: Result<usize>, bucket: &str, object: &str) -> Result<usize> { fn resolve_decommission_target_pool_idx_result(result: Result<usize>, bucket: &str, object: &str) -> Result<usize> {
result.map_err(|err| Error::other(format!("failed to select decommission target pool for {bucket}/{object}: {err}"))) result.map_err(|err| Error::other(format!("failed to select decommission target pool for {bucket}/{object}: {err}")))
} }
@@ -1795,6 +1841,10 @@ impl ECStore {
check_put_object_args(bucket, object)?; check_put_object_args(bucket, object)?;
let mut opts = opts.clone(); let mut opts = opts.clone();
let is_free_version = fi.tier_free_version();
if is_free_version {
opts.incl_free_versions = true;
}
let bucket_incarnation_fence = if is_meta_bucketname(bucket) { let bucket_incarnation_fence = if is_meta_bucketname(bucket) {
None None
} else { } else {
@@ -1849,7 +1899,7 @@ impl ECStore {
versions versions
.versions .versions
.iter() .iter()
.find(|current| current.version_id == fi.version_id && !current.tier_free_version()) .find(|current| current.version_id == fi.version_id && current.tier_free_version() == is_free_version)
}) })
.ok_or_else(|| to_object_err(StorageError::FileNotFound, vec![bucket, object.as_str()]))?; .ok_or_else(|| to_object_err(StorageError::FileNotFound, vec![bucket, object.as_str()]))?;
if !tiered_data_movement_source_matches(fi, current_source)? { if !tiered_data_movement_source_matches(fi, current_source)? {
@@ -1864,24 +1914,40 @@ impl ECStore {
.get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx) .get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
.await; .await;
let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx); let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx);
if is_free_version && target_pool_idx == opts.src_pool_idx {
return Err(Error::DiskFull);
}
let equivalent = if is_free_version {
self.has_equivalent_data_movement_tier_free_version(bucket, &object, &fi, target_pool_idx)
.await?
} else {
self.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, target_pool_idx)
.await?
};
if equivalent {
return Ok(());
}
return Err(decommission_free_version_overwrite_error(bucket, &object, fi.version_id));
}
let result = if is_free_version {
if self if self
.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, target_pool_idx) .has_equivalent_data_movement_tier_free_version(bucket, &object, &fi, idx)
.await? .await?
{ {
return Ok(()); return Ok(());
} }
self.pools[idx]
return Err(StorageError::DataMovementOverwriteErr( .get_disks_by_key(&object)
bucket.to_owned(), .decommission_tier_free_version(bucket, &object, &fi, &opts)
object.to_owned(), .await
opts.version_id.clone().unwrap_or_default(), } else {
)); self.pools[idx]
} .get_disks_by_key(&object)
.decommission_tiered_object(bucket, &object, &fi, &opts)
let result = self.pools[idx] .await
.get_disks_by_key(&object) };
.decommission_tiered_object(bucket, &object, &fi, &opts)
.await;
if matches!(result, Err(Error::PreconditionFailed)) { if matches!(result, Err(Error::PreconditionFailed)) {
if self if self
.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, idx) .has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, idx)
+3 -4
View File
@@ -102,10 +102,9 @@ pub const TRANSITION_PENDING: &str = "pending";
/// delete paths the same obligation is also carried by a committed tier-journal /// delete paths the same obligation is also carried by a committed tier-journal
/// entry; deletes without such an entry (for example a removed version whose /// entry; deletes without such an entry (for example a removed version whose
/// transition state decodes as unknown) rely on this record alone until the /// transition state decodes as unknown) rely on this record alone until the
/// worker removes it after a successful remote delete. Decommission does not /// worker removes it after a successful remote delete. Decommission preserves
/// preserve these semantics: its exact inventory keeps the records inline in /// the record and its remote identity on the target pool before source cleanup
/// `versions` and the migration loop treats them as ordinary delete markers — /// — see docs/architecture/decommission-compatibility.md.
/// see docs/architecture/decommission-compatibility.md.
pub const FREE_VERSION: &str = "free-version"; pub const FREE_VERSION: &str = "free-version";
pub const TRANSITION_STATUS: &str = "transition-status"; pub const TRANSITION_STATUS: &str = "transition-status";
+4 -4
View File
@@ -2730,10 +2730,10 @@ impl MetaObject {
/// identity so the lifecycle worker can issue the idempotent remote delete /// identity so the lifecycle worker can issue the idempotent remote delete
/// and only then remove the record; until then the recovery scan and the /// and only then remove the record; until then the recovery scan and the
/// usage scanner keep re-enqueueing it. S3 and lifecycle deletes also /// usage scanner keep re-enqueueing it. S3 and lifecycle deletes also
/// persist a committed tier-journal entry for the same remote delete, so a /// persist a committed tier-journal entry for the same remote delete. The
/// record destroyed without its remote delete (as decommission does when it /// decommission path copies this record unchanged before source cleanup,
/// treats these records as ordinary delete markers) strands only the /// including when the transition state is unknown — see
/// journal-less cases — see docs/architecture/decommission-compatibility.md. /// docs/architecture/decommission-compatibility.md.
pub fn init_free_version(&self, fi: &FileInfo) -> Result<(FileMetaVersion, bool)> { pub fn init_free_version(&self, fi: &FileInfo) -> Result<(FileMetaVersion, bool)> {
if fi.skip_tier_free_version() { if fi.skip_tier_free_version() {
return Ok((FileMetaVersion::default(), false)); return Ok((FileMetaVersion::default(), false));
+39 -40
View File
@@ -188,55 +188,54 @@ them.
### Decommission Handling ### Decommission Handling
The exact decommission inventory loader (`load_file_info_versions_exact` via The exact decommission inventory loader (`load_file_info_versions_exact` via
`get_all_file_info_versions`) keeps free-version records inline in `versions`; it `get_all_file_info_versions`) keeps free-version records inline in `versions`.
never populates `free_versions`, so the source-cleanup preflight comparison of The migration loop handles them before lifecycle expiry and delete-marker
`free_versions` is vacuous for decommission. The migration loop then routes every shortcuts. It selects a target pool using the free-version-aware lookup, then
record through the generic delete-marker handling: writes the original free record to every target disk with the normal metadata
write quorum. The free-version marker, local version id, transition identity,
transition state, and destination id are preserved at the FileInfo/metadata
boundary.
- a record that is the only remaining version without replication is skipped by the The source record is physically removed only after the target write quorum has
empty-delete-marker rule and counted as done; committed and the source cleanup preflight still matches the exact inventory.
- any other record is copied to the target pool as an ordinary delete marker with the If target capacity, metadata validation, lock fencing, or quorum fails, the
same version id and mod time. source record remains and the entry records `state = "free_version_retained"`
with reason `tier_free_version_migration_failed`; the worker retries the
In both cases the free-version flag and its remote-tier identity are dropped: operation on a later pass. A target record with the same version id is accepted
decommission neither preserves free-version semantics nor performs or reschedules the only when its free-version identity matches; a conflicting ordinary version or
pending remote-tier delete. Source cleanup then removes the original records together different free record is an overwrite error. This makes retries idempotent and
with the source xl.meta. prevents a free record from replacing a user-visible version.
Allowed physical-delete timing: the source record may be removed once the migration
loop has dispositioned it (copied as a plain marker or skipped as lone), which
happens regardless of whether its remote-tier delete was ever performed.
### Reference-Audit Result ### Reference-Audit Result
No cluster-local consumer resolves a free version after decommission finishes: GET, After migration, user-facing GET/list/transition/replication/restore paths still
listing, transition planning, replication, restore, and heal operate either on exclude the record. Recovery, usage scanning, lifecycle tier cleanup, and heal
user-visible versions or while the record still exists. The remote exposure is continue to see it when they request free versions, so an unresolved remote
bounded: delete remains actionable on the target pool. The committed tier journal remains
an independent retry source where one exists; it is not used as a reason to drop
the xl.meta record. In particular, `Unknown` transition state records are
migrated unchanged rather than discarded: the lifecycle worker retains them if
remote identity validation cannot make a delete request.
- On every user-facing delete path the remote-delete obligation is durably carried Each migrated record emits `state = "free_version_migrated"` with reason
by the committed tier-journal entry, which the tier sweeper processes `tier_free_version_migrated`. Each failed record emits the retained state and
independently of xl.meta; the free-version record is an idempotent second failure reason above. The entry also emits a disposition summary with migrated,
pointer, not the only one. Dropping it during decommission therefore does not retained, and total counts. The final decommission sweep uses the exact loader,
orphan the remote object. counts free records still present, and emits one retained record/reason for each
- Residual exposure: for records whose version state decoded as `Unknown` no unresolved free version before failing the sweep. This makes both successful
journal entry exists, so dropping the unconsumed record loses that cleanup hint migration and retained cleanup obligations visible instead of silently omitting
and the remote-tier object is orphaned. The same applies to any future internal free records.
delete path that removes transitioned versions without a journal entry.
Copying a pending record as an ordinary delete marker also adds a user-visible No new S3-visible version or admin response field is needed: free versions remain
tombstone to the target pool's version history that the source never exposed. internal and are never counted as user-visible versions. The structured
`decommission_entry` events are the operational status surface for the
Because of the residual journal-less case, decommission must account for every free-version disposition; the existing decommission item/failed counters still
free-version record instead of omitting it silently: report the enclosing object migration result.
- `decommission_free_versions_skipped` counts the records per decommission entry;
- entries with a non-zero count log `state = "free_versions_skipped"` with reason
`tier_free_version_not_migrated`.
Regression guard: Regression guard:
- `decommission_free_version_accounting_reports_skipped_records` - `decommission_free_version_accounting_records_migration_and_retention`
- `decommission_tier_free_version_preserves_remote_identity`
## Regression Guard ## Regression Guard