mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-29 00:17:11 +00:00
fix(ecstore): migrate tier free versions during decommission
This commit is contained in:
@@ -1130,21 +1130,29 @@ fn should_cleanup_decommission_source_entry(decommissioned: usize, total_version
|
|||||||
decommissioned.saturating_add(expired) == total_versions
|
decommissioned.saturating_add(expired) == total_versions
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Disposition reason logged for tier free-version records that decommission
|
const DECOMMISSION_FREE_VERSION_MIGRATED_REASON: &str = "tier_free_version_migrated";
|
||||||
/// skips instead of migrating.
|
const DECOMMISSION_FREE_VERSION_RETAINED_REASON: &str = "tier_free_version_migration_failed";
|
||||||
const DECOMMISSION_FREE_VERSION_SKIP_REASON: &str = "tier_free_version_not_migrated";
|
const DECOMMISSION_FREE_VERSION_SWEEP_REASON: &str = "tier_free_version_unresolved_after_decommission";
|
||||||
|
const DECOMMISSION_FREE_VERSION_DISPOSITION_REASON: &str = "tier_free_version_disposition_recorded";
|
||||||
|
|
||||||
/// Counts the tier free-version records present in a decommission entry
|
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||||
/// inventory. The exact loader (`load_file_info_versions_exact`) keeps these
|
struct DecommissionFreeVersionDisposition {
|
||||||
/// records inline in `versions` instead of separating them into
|
migrated: usize,
|
||||||
/// `free_versions`, and the migration loop then routes them through the
|
retained: usize,
|
||||||
/// generic delete-marker path: the free-version flag and its remote-tier
|
}
|
||||||
/// identity are never carried to the target pool, and a lone record is skipped
|
|
||||||
/// by the empty-delete-marker rule. Accounting for them here keeps the final
|
impl DecommissionFreeVersionDisposition {
|
||||||
/// sweep from silently omitting records whose free-version disposition was
|
fn record_migrated(&mut self) {
|
||||||
/// dropped (see docs/architecture/decommission-compatibility.md).
|
self.migrated += 1;
|
||||||
fn decommission_free_versions_skipped(fivs: &FileInfoVersions) -> usize {
|
}
|
||||||
fivs.versions.iter().filter(|version| version.tier_free_version()).count()
|
|
||||||
|
fn record_retained(&mut self) {
|
||||||
|
self.retained += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn total(self) -> usize {
|
||||||
|
self.migrated.saturating_add(self.retained)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
@@ -2478,6 +2486,7 @@ fn decommission_remote_tiered_opts(
|
|||||||
user_defined: version.metadata.clone(),
|
user_defined: version.metadata.clone(),
|
||||||
src_pool_idx,
|
src_pool_idx,
|
||||||
data_movement: true,
|
data_movement: true,
|
||||||
|
incl_free_versions: version.tier_free_version(),
|
||||||
include_part_checksums: true,
|
include_part_checksums: true,
|
||||||
http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
|
http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
|
||||||
expected_bucket_incarnation_id,
|
expected_bucket_incarnation_id,
|
||||||
@@ -3114,24 +3123,9 @@ impl ECStore {
|
|||||||
fivs.versions
|
fivs.versions
|
||||||
.sort_by_key(|v| (v.mod_time.is_none(), std::cmp::Reverse(v.mod_time)));
|
.sort_by_key(|v| (v.mod_time.is_none(), std::cmp::Reverse(v.mod_time)));
|
||||||
|
|
||||||
let skipped_free_versions = decommission_free_versions_skipped(&fivs);
|
|
||||||
if skipped_free_versions > 0 {
|
|
||||||
debug!(
|
|
||||||
event = EVENT_DECOMMISSION_ENTRY,
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
|
||||||
pool_index = idx,
|
|
||||||
bucket = %bucket,
|
|
||||||
object = %entry.name,
|
|
||||||
skipped_free_versions,
|
|
||||||
reason = DECOMMISSION_FREE_VERSION_SKIP_REASON,
|
|
||||||
state = "free_versions_skipped",
|
|
||||||
"Decommission skipped free-version migration"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut decommissioned: usize = 0;
|
let mut decommissioned: usize = 0;
|
||||||
let mut expired: usize = 0;
|
let mut expired: usize = 0;
|
||||||
|
let mut free_version_disposition = DecommissionFreeVersionDisposition::default();
|
||||||
let mut cleanup_preflight_allowed_missing = Vec::new();
|
let mut cleanup_preflight_allowed_missing = Vec::new();
|
||||||
|
|
||||||
for version in fivs.versions.iter() {
|
for version in fivs.versions.iter() {
|
||||||
@@ -3140,6 +3134,80 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
decommission_cancel_signal_result(rx.is_cancelled())?;
|
decommission_cancel_signal_result(rx.is_cancelled())?;
|
||||||
|
|
||||||
|
if version.tier_free_version() {
|
||||||
|
let version_id = version.version_id.map(|v| v.to_string());
|
||||||
|
let mut migration_error = None;
|
||||||
|
let mut migrated = false;
|
||||||
|
for _ in 0..3 {
|
||||||
|
match self
|
||||||
|
.decommission_tiered_object(
|
||||||
|
bucket.as_str(),
|
||||||
|
&version.name,
|
||||||
|
version,
|
||||||
|
&decommission_remote_tiered_opts(version, version_id.clone(), idx, expected_bucket_incarnation_id),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(()) => {
|
||||||
|
migrated = true;
|
||||||
|
migration_error = None;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Err(err) if is_decommission_target_capacity_error(&err) => {
|
||||||
|
return Err(with_decommission_entry_context(
|
||||||
|
"decommission_tier_free_version",
|
||||||
|
bucket.as_str(),
|
||||||
|
version.name.as_str(),
|
||||||
|
err,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Err(err) => migration_error = Some(err),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
let mut pool_meta = self.pool_meta.write().await;
|
||||||
|
if let Err(err) = count_decommission_item(&mut pool_meta, idx, 0, !migrated) {
|
||||||
|
return Err(with_decommission_entry_context(
|
||||||
|
"count_decommission_item",
|
||||||
|
bucket.as_str(),
|
||||||
|
entry.name.as_str(),
|
||||||
|
err,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if migrated {
|
||||||
|
decommissioned += 1;
|
||||||
|
free_version_disposition.record_migrated();
|
||||||
|
} else {
|
||||||
|
free_version_disposition.record_retained();
|
||||||
|
}
|
||||||
|
|
||||||
|
debug!(
|
||||||
|
event = EVENT_DECOMMISSION_ENTRY,
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||||
|
pool_index = idx,
|
||||||
|
bucket = %bucket,
|
||||||
|
object = %version.name,
|
||||||
|
version_id = ?version_id,
|
||||||
|
result = ?migration_error,
|
||||||
|
reason = if migrated {
|
||||||
|
DECOMMISSION_FREE_VERSION_MIGRATED_REASON
|
||||||
|
} else {
|
||||||
|
DECOMMISSION_FREE_VERSION_RETAINED_REASON
|
||||||
|
},
|
||||||
|
state = if migrated { "free_version_migrated" } else { "free_version_retained" },
|
||||||
|
"Decommission free-version disposition recorded"
|
||||||
|
);
|
||||||
|
|
||||||
|
if !migrated {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
if should_skip_lifecycle_for_data_movement(
|
if should_skip_lifecycle_for_data_movement(
|
||||||
self.clone(),
|
self.clone(),
|
||||||
&bucket,
|
&bucket,
|
||||||
@@ -3427,6 +3495,23 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if free_version_disposition.total() > 0 {
|
||||||
|
debug!(
|
||||||
|
event = EVENT_DECOMMISSION_ENTRY,
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||||
|
pool_index = idx,
|
||||||
|
bucket = %bucket,
|
||||||
|
object = %entry.name,
|
||||||
|
free_versions_migrated = free_version_disposition.migrated,
|
||||||
|
free_versions_retained = free_version_disposition.retained,
|
||||||
|
free_versions_total = free_version_disposition.total(),
|
||||||
|
reason = DECOMMISSION_FREE_VERSION_DISPOSITION_REASON,
|
||||||
|
state = "free_version_disposition",
|
||||||
|
"Decommission free-version disposition summary"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) {
|
if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) {
|
||||||
if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||||
return Err(Error::other("decommission bucket incarnation fence was lost before source cleanup"));
|
return Err(Error::other("decommission bucket incarnation fence was lost before source cleanup"));
|
||||||
@@ -4393,6 +4478,7 @@ impl ECStore {
|
|||||||
let lifecycle_config_cb = lifecycle_config.clone();
|
let lifecycle_config_cb = lifecycle_config.clone();
|
||||||
let object_lock_config_cb = object_lock_config.clone();
|
let object_lock_config_cb = object_lock_config.clone();
|
||||||
let store = Arc::clone(self);
|
let store = Arc::clone(self);
|
||||||
|
let set_cb = Arc::clone(set);
|
||||||
let callback_rx_cb = callback_rx.clone();
|
let callback_rx_cb = callback_rx.clone();
|
||||||
|
|
||||||
let callback: ListCallback = Arc::new(move |entry: MetaCacheEntry| {
|
let callback: ListCallback = Arc::new(move |entry: MetaCacheEntry| {
|
||||||
@@ -4402,6 +4488,7 @@ impl ECStore {
|
|||||||
let lifecycle_config = lifecycle_config_cb.clone();
|
let lifecycle_config = lifecycle_config_cb.clone();
|
||||||
let object_lock_config = object_lock_config_cb.clone();
|
let object_lock_config = object_lock_config_cb.clone();
|
||||||
let store = Arc::clone(&store);
|
let store = Arc::clone(&store);
|
||||||
|
let set = Arc::clone(&set_cb);
|
||||||
let callback_rx = callback_rx_cb.clone();
|
let callback_rx = callback_rx_cb.clone();
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
if callback_rx.is_cancelled() {
|
if callback_rx.is_cancelled() {
|
||||||
@@ -4416,11 +4503,14 @@ impl ECStore {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let fivs = match load_decommission_entry_versions(
|
let fivs = match load_decommission_entry_exact_versions(
|
||||||
|
&set,
|
||||||
&entry,
|
&entry,
|
||||||
&bucket_name,
|
&bucket_name,
|
||||||
"check_after_decommission.file_info_versions",
|
"check_after_decommission.file_info_versions",
|
||||||
) {
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(fivs) => fivs,
|
Ok(fivs) => fivs,
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
let mut first_err = entry_error.lock().await;
|
let mut first_err = entry_error.lock().await;
|
||||||
@@ -4433,7 +4523,23 @@ impl ECStore {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let mut remaining = 0;
|
let mut remaining = 0;
|
||||||
for version in &fivs.versions {
|
for version in fivs.versions.iter().chain(fivs.free_versions.iter()) {
|
||||||
|
if version.tier_free_version() {
|
||||||
|
remaining += 1;
|
||||||
|
debug!(
|
||||||
|
event = EVENT_DECOMMISSION_ENTRY,
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||||
|
pool_index = idx,
|
||||||
|
bucket = %bucket_name,
|
||||||
|
object = %entry.name,
|
||||||
|
version_id = ?version.version_id,
|
||||||
|
reason = DECOMMISSION_FREE_VERSION_SWEEP_REASON,
|
||||||
|
state = "free_version_retained",
|
||||||
|
"Decommission final sweep retained a free version"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if version.deleted {
|
if version.deleted {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -4787,6 +4893,12 @@ mod tests {
|
|||||||
assert!(opts.include_part_checksums);
|
assert!(opts.include_part_checksums);
|
||||||
assert!(opts.http_preconditions.is_some());
|
assert!(opts.http_preconditions.is_some());
|
||||||
assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
|
assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
|
||||||
|
assert!(!opts.incl_free_versions);
|
||||||
|
|
||||||
|
let mut free_version = version;
|
||||||
|
free_version.set_tier_free_version();
|
||||||
|
let free_opts = decommission_remote_tiered_opts(&free_version, Some("free-version-id".to_string()), 9, Some(incarnation));
|
||||||
|
assert!(free_opts.incl_free_versions);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -5491,13 +5603,13 @@ pub(crate) fn fallback_free_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usi
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod pools_tests {
|
mod pools_tests {
|
||||||
use super::{
|
use super::{
|
||||||
DECOMMISSION_FREE_VERSION_SKIP_REASON, DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
|
DECOMMISSION_FREE_VERSION_MIGRATED_REASON, DECOMMISSION_FREE_VERSION_RETAINED_REASON,
|
||||||
DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF, DecomBucketInfo, DecommissionStartPoolState, DecommissionTerminalState,
|
DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF,
|
||||||
ListCallback, PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, PoolStatus, apply_decommission_status_space_info,
|
DecomBucketInfo, DecommissionFreeVersionDisposition, DecommissionStartPoolState, DecommissionTerminalState, ListCallback,
|
||||||
|
PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, PoolStatus, apply_decommission_status_space_info,
|
||||||
bind_decommission_cancelers, bind_missing_decommission_cancelers, cancel_decommission_canceler,
|
bind_decommission_cancelers, bind_missing_decommission_cancelers, cancel_decommission_canceler,
|
||||||
classify_decommission_terminal_state, count_decommission_item, decommission_cancel_signal_result,
|
classify_decommission_terminal_state, count_decommission_item, decommission_cancel_signal_result, decommission_item_size,
|
||||||
decommission_free_versions_skipped, decommission_item_size, decommission_meta_bucket_options,
|
decommission_meta_bucket_options, decommission_start_pool_state, dedup_indices, default_decommission_bucket_concurrency,
|
||||||
decommission_start_pool_state, dedup_indices, default_decommission_bucket_concurrency,
|
|
||||||
ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed, ensure_decommission_listing_disks_available,
|
ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed, ensure_decommission_listing_disks_available,
|
||||||
ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, ensure_decommission_start_keeps_active_pool,
|
ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, ensure_decommission_start_keeps_active_pool,
|
||||||
ensure_decommission_start_local_leader, ensure_decommission_start_pool_states,
|
ensure_decommission_start_local_leader, ensure_decommission_start_pool_states,
|
||||||
@@ -6797,24 +6909,16 @@ mod pools_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn decommission_free_version_accounting_reports_skipped_records() {
|
fn decommission_free_version_accounting_records_migration_and_retention() {
|
||||||
let mut fivs = FileInfoVersions::default();
|
let mut disposition = DecommissionFreeVersionDisposition::default();
|
||||||
assert_eq!(decommission_free_versions_skipped(&fivs), 0);
|
disposition.record_migrated();
|
||||||
|
disposition.record_retained();
|
||||||
|
|
||||||
fivs.versions.push(FileInfo {
|
assert_eq!(disposition.migrated, 1);
|
||||||
name: "object.txt".to_string(),
|
assert_eq!(disposition.retained, 1);
|
||||||
..Default::default()
|
assert_eq!(disposition.total(), 2);
|
||||||
});
|
assert_eq!(DECOMMISSION_FREE_VERSION_MIGRATED_REASON, "tier_free_version_migrated");
|
||||||
let mut free_one = FileInfo::default();
|
assert_eq!(DECOMMISSION_FREE_VERSION_RETAINED_REASON, "tier_free_version_migration_failed");
|
||||||
free_one.set_tier_free_version();
|
|
||||||
fivs.versions.push(free_one);
|
|
||||||
let mut free_two = FileInfo::default();
|
|
||||||
free_two.set_tier_free_version();
|
|
||||||
free_two.transition_tier = "WARM".to_string();
|
|
||||||
fivs.versions.push(free_two);
|
|
||||||
|
|
||||||
assert_eq!(decommission_free_versions_skipped(&fivs), 2);
|
|
||||||
assert_eq!(DECOMMISSION_FREE_VERSION_SKIP_REASON, "tier_free_version_not_migrated");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -4642,6 +4642,63 @@ fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl SetDisks {
|
impl SetDisks {
|
||||||
|
/// Publish an internal tier free-version record without changing its
|
||||||
|
/// delete-marker shape or remote-tier identity. The caller holds the
|
||||||
|
/// source and target object locks; a write quorum is required before the
|
||||||
|
/// source cleanup may remove the original record.
|
||||||
|
#[tracing::instrument(skip(self, fi, opts))]
|
||||||
|
pub(crate) async fn decommission_tier_free_version(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
fi: &FileInfo,
|
||||||
|
opts: &ObjectOptions,
|
||||||
|
) -> Result<()> {
|
||||||
|
if !fi.deleted || !fi.tier_free_version() {
|
||||||
|
return Err(Error::other("decommission tier free-version write requires a free version record"));
|
||||||
|
}
|
||||||
|
if opts
|
||||||
|
.namespace_lock_fence
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||||
|
|| opts
|
||||||
|
.bucket_lifecycle_lock_fence
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||||
|
{
|
||||||
|
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||||
|
mode: "decommission_tier_free_version_commit",
|
||||||
|
bucket: bucket.to_string(),
|
||||||
|
object: object.to_string(),
|
||||||
|
required: 1,
|
||||||
|
achieved: 0,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let disks = self.disks.read().await.clone();
|
||||||
|
let write_quorum = self.default_write_quorum();
|
||||||
|
let futures = disks.into_iter().map(|disk| {
|
||||||
|
let file_info = fi.clone();
|
||||||
|
async move {
|
||||||
|
if let Some(disk) = disk {
|
||||||
|
disk.write_metadata("", bucket, object, file_info).await
|
||||||
|
} else {
|
||||||
|
Err(DiskError::DiskNotFound)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let mut errs = Vec::new();
|
||||||
|
for result in join_all(futures).await {
|
||||||
|
match result {
|
||||||
|
Ok(_) => errs.push(None),
|
||||||
|
Err(err) => errs.push(Some(err)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resolve_tiered_decommission_write_quorum_result(&errs, write_quorum, bucket, object)
|
||||||
|
}
|
||||||
|
|
||||||
#[tracing::instrument(skip(self, fi, opts))]
|
#[tracing::instrument(skip(self, fi, opts))]
|
||||||
pub(crate) async fn decommission_tiered_object(
|
pub(crate) async fn decommission_tiered_object(
|
||||||
&self,
|
&self,
|
||||||
@@ -9864,6 +9921,57 @@ mod tests {
|
|||||||
assert_ne!(updated.erasure.distribution, original.erasure.distribution);
|
assert_ne!(updated.erasure.distribution, original.erasure.distribution);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decommission_tier_free_version_preserves_remote_identity() {
|
||||||
|
let set_disks = make_local_bucket_test_set_disks().await;
|
||||||
|
let bucket = "free-version-decommission";
|
||||||
|
let object = "object.txt";
|
||||||
|
let version_id = Uuid::new_v4();
|
||||||
|
let mut free_version = FileInfo {
|
||||||
|
name: object.to_string(),
|
||||||
|
volume: bucket.to_string(),
|
||||||
|
version_id: Some(version_id),
|
||||||
|
mod_time: Some(time::OffsetDateTime::now_utc()),
|
||||||
|
deleted: true,
|
||||||
|
transition_tier: "WARM-TIER".to_string(),
|
||||||
|
transitioned_objname: "remote/object".to_string(),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
free_version.set_tier_free_version();
|
||||||
|
|
||||||
|
set_disks
|
||||||
|
.decommission_tier_free_version(bucket, object, &free_version, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect("free-version metadata should reach the target quorum");
|
||||||
|
set_disks
|
||||||
|
.decommission_tier_free_version(bucket, object, &free_version, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect("replaying the same free-version metadata should be idempotent");
|
||||||
|
|
||||||
|
let versions = set_disks
|
||||||
|
.load_file_info_versions_exact(bucket, object)
|
||||||
|
.await
|
||||||
|
.expect("migrated free-version metadata should decode")
|
||||||
|
.expect("migrated free-version metadata should exist");
|
||||||
|
let migrated = versions
|
||||||
|
.versions
|
||||||
|
.iter()
|
||||||
|
.find(|version| version.version_id == Some(version_id))
|
||||||
|
.expect("free version should be present on the target");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
versions
|
||||||
|
.versions
|
||||||
|
.iter()
|
||||||
|
.filter(|version| version.version_id == Some(version_id))
|
||||||
|
.count(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert!(migrated.tier_free_version());
|
||||||
|
assert_eq!(migrated.transition_tier, "WARM-TIER");
|
||||||
|
assert_eq!(migrated.transitioned_objname, "remote/object");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_resolve_tiered_decommission_write_quorum_result_allows_successful_quorum() {
|
fn test_resolve_tiered_decommission_write_quorum_result_allows_successful_quorum() {
|
||||||
let errs = vec![None, None, Some(DiskError::DiskNotFound), None];
|
let errs = vec![None, None, Some(DiskError::DiskNotFound), None];
|
||||||
|
|||||||
@@ -1152,6 +1152,15 @@ fn tiered_data_movement_source_matches(
|
|||||||
&& expected_backend == current_backend)
|
&& expected_backend == current_backend)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn decommission_free_version_overwrite_error(bucket: &str, object: &str, version_id: Option<Uuid>) -> Error {
|
||||||
|
StorageError::DataMovementOverwriteErr(
|
||||||
|
bucket.to_owned(),
|
||||||
|
object.to_owned(),
|
||||||
|
version_id.map(|id| id.to_string()).unwrap_or_default(),
|
||||||
|
)
|
||||||
|
.into()
|
||||||
|
}
|
||||||
|
|
||||||
fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
|
fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
|
||||||
target_pool_idx != src_pool_idx
|
target_pool_idx != src_pool_idx
|
||||||
}
|
}
|
||||||
@@ -1776,6 +1785,43 @@ impl ECStore {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn has_equivalent_data_movement_tier_free_version(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
source: &rustfs_filemeta::FileInfo,
|
||||||
|
target_pool_idx: usize,
|
||||||
|
) -> Result<bool> {
|
||||||
|
let pool = self
|
||||||
|
.pools
|
||||||
|
.get(target_pool_idx)
|
||||||
|
.ok_or_else(|| Error::other(format!("invalid tiered data movement target pool {target_pool_idx}")))?;
|
||||||
|
let logical_object = decode_dir_object(object);
|
||||||
|
let Some(versions) = pool
|
||||||
|
.get_disks_by_key(object)
|
||||||
|
.load_file_info_versions_exact(bucket, &logical_object)
|
||||||
|
.await?
|
||||||
|
else {
|
||||||
|
return Ok(false);
|
||||||
|
};
|
||||||
|
|
||||||
|
let Some(target) = versions
|
||||||
|
.versions
|
||||||
|
.iter()
|
||||||
|
.find(|version| version.version_id == source.version_id)
|
||||||
|
else {
|
||||||
|
return Ok(false);
|
||||||
|
};
|
||||||
|
if !target.tier_free_version() {
|
||||||
|
return Err(decommission_free_version_overwrite_error(bucket, object, source.version_id));
|
||||||
|
}
|
||||||
|
if tiered_data_movement_source_matches(source, target)? {
|
||||||
|
Ok(true)
|
||||||
|
} else {
|
||||||
|
Err(decommission_free_version_overwrite_error(bucket, object, source.version_id))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn resolve_decommission_target_pool_idx_result(result: Result<usize>, bucket: &str, object: &str) -> Result<usize> {
|
fn resolve_decommission_target_pool_idx_result(result: Result<usize>, bucket: &str, object: &str) -> Result<usize> {
|
||||||
result.map_err(|err| Error::other(format!("failed to select decommission target pool for {bucket}/{object}: {err}")))
|
result.map_err(|err| Error::other(format!("failed to select decommission target pool for {bucket}/{object}: {err}")))
|
||||||
}
|
}
|
||||||
@@ -1795,6 +1841,10 @@ impl ECStore {
|
|||||||
check_put_object_args(bucket, object)?;
|
check_put_object_args(bucket, object)?;
|
||||||
|
|
||||||
let mut opts = opts.clone();
|
let mut opts = opts.clone();
|
||||||
|
let is_free_version = fi.tier_free_version();
|
||||||
|
if is_free_version {
|
||||||
|
opts.incl_free_versions = true;
|
||||||
|
}
|
||||||
let bucket_incarnation_fence = if is_meta_bucketname(bucket) {
|
let bucket_incarnation_fence = if is_meta_bucketname(bucket) {
|
||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
@@ -1849,7 +1899,7 @@ impl ECStore {
|
|||||||
versions
|
versions
|
||||||
.versions
|
.versions
|
||||||
.iter()
|
.iter()
|
||||||
.find(|current| current.version_id == fi.version_id && !current.tier_free_version())
|
.find(|current| current.version_id == fi.version_id && current.tier_free_version() == is_free_version)
|
||||||
})
|
})
|
||||||
.ok_or_else(|| to_object_err(StorageError::FileNotFound, vec![bucket, object.as_str()]))?;
|
.ok_or_else(|| to_object_err(StorageError::FileNotFound, vec![bucket, object.as_str()]))?;
|
||||||
if !tiered_data_movement_source_matches(fi, current_source)? {
|
if !tiered_data_movement_source_matches(fi, current_source)? {
|
||||||
@@ -1864,24 +1914,40 @@ impl ECStore {
|
|||||||
.get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
|
.get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
|
||||||
.await;
|
.await;
|
||||||
let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx);
|
let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx);
|
||||||
|
if is_free_version && target_pool_idx == opts.src_pool_idx {
|
||||||
|
return Err(Error::DiskFull);
|
||||||
|
}
|
||||||
|
let equivalent = if is_free_version {
|
||||||
|
self.has_equivalent_data_movement_tier_free_version(bucket, &object, &fi, target_pool_idx)
|
||||||
|
.await?
|
||||||
|
} else {
|
||||||
|
self.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, target_pool_idx)
|
||||||
|
.await?
|
||||||
|
};
|
||||||
|
if equivalent {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
return Err(decommission_free_version_overwrite_error(bucket, &object, fi.version_id));
|
||||||
|
}
|
||||||
|
|
||||||
|
let result = if is_free_version {
|
||||||
if self
|
if self
|
||||||
.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, target_pool_idx)
|
.has_equivalent_data_movement_tier_free_version(bucket, &object, &fi, idx)
|
||||||
.await?
|
.await?
|
||||||
{
|
{
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
self.pools[idx]
|
||||||
return Err(StorageError::DataMovementOverwriteErr(
|
.get_disks_by_key(&object)
|
||||||
bucket.to_owned(),
|
.decommission_tier_free_version(bucket, &object, &fi, &opts)
|
||||||
object.to_owned(),
|
.await
|
||||||
opts.version_id.clone().unwrap_or_default(),
|
} else {
|
||||||
));
|
self.pools[idx]
|
||||||
}
|
.get_disks_by_key(&object)
|
||||||
|
.decommission_tiered_object(bucket, &object, &fi, &opts)
|
||||||
let result = self.pools[idx]
|
.await
|
||||||
.get_disks_by_key(&object)
|
};
|
||||||
.decommission_tiered_object(bucket, &object, &fi, &opts)
|
|
||||||
.await;
|
|
||||||
if matches!(result, Err(Error::PreconditionFailed)) {
|
if matches!(result, Err(Error::PreconditionFailed)) {
|
||||||
if self
|
if self
|
||||||
.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, idx)
|
.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, idx)
|
||||||
|
|||||||
@@ -102,10 +102,9 @@ pub const TRANSITION_PENDING: &str = "pending";
|
|||||||
/// delete paths the same obligation is also carried by a committed tier-journal
|
/// delete paths the same obligation is also carried by a committed tier-journal
|
||||||
/// entry; deletes without such an entry (for example a removed version whose
|
/// entry; deletes without such an entry (for example a removed version whose
|
||||||
/// transition state decodes as unknown) rely on this record alone until the
|
/// transition state decodes as unknown) rely on this record alone until the
|
||||||
/// worker removes it after a successful remote delete. Decommission does not
|
/// worker removes it after a successful remote delete. Decommission preserves
|
||||||
/// preserve these semantics: its exact inventory keeps the records inline in
|
/// the record and its remote identity on the target pool before source cleanup
|
||||||
/// `versions` and the migration loop treats them as ordinary delete markers —
|
/// — see docs/architecture/decommission-compatibility.md.
|
||||||
/// see docs/architecture/decommission-compatibility.md.
|
|
||||||
pub const FREE_VERSION: &str = "free-version";
|
pub const FREE_VERSION: &str = "free-version";
|
||||||
|
|
||||||
pub const TRANSITION_STATUS: &str = "transition-status";
|
pub const TRANSITION_STATUS: &str = "transition-status";
|
||||||
|
|||||||
@@ -2730,10 +2730,10 @@ impl MetaObject {
|
|||||||
/// identity so the lifecycle worker can issue the idempotent remote delete
|
/// identity so the lifecycle worker can issue the idempotent remote delete
|
||||||
/// and only then remove the record; until then the recovery scan and the
|
/// and only then remove the record; until then the recovery scan and the
|
||||||
/// usage scanner keep re-enqueueing it. S3 and lifecycle deletes also
|
/// usage scanner keep re-enqueueing it. S3 and lifecycle deletes also
|
||||||
/// persist a committed tier-journal entry for the same remote delete, so a
|
/// persist a committed tier-journal entry for the same remote delete. The
|
||||||
/// record destroyed without its remote delete (as decommission does when it
|
/// decommission path copies this record unchanged before source cleanup,
|
||||||
/// treats these records as ordinary delete markers) strands only the
|
/// including when the transition state is unknown — see
|
||||||
/// journal-less cases — see docs/architecture/decommission-compatibility.md.
|
/// docs/architecture/decommission-compatibility.md.
|
||||||
pub fn init_free_version(&self, fi: &FileInfo) -> Result<(FileMetaVersion, bool)> {
|
pub fn init_free_version(&self, fi: &FileInfo) -> Result<(FileMetaVersion, bool)> {
|
||||||
if fi.skip_tier_free_version() {
|
if fi.skip_tier_free_version() {
|
||||||
return Ok((FileMetaVersion::default(), false));
|
return Ok((FileMetaVersion::default(), false));
|
||||||
|
|||||||
@@ -188,55 +188,54 @@ them.
|
|||||||
### Decommission Handling
|
### Decommission Handling
|
||||||
|
|
||||||
The exact decommission inventory loader (`load_file_info_versions_exact` via
|
The exact decommission inventory loader (`load_file_info_versions_exact` via
|
||||||
`get_all_file_info_versions`) keeps free-version records inline in `versions`; it
|
`get_all_file_info_versions`) keeps free-version records inline in `versions`.
|
||||||
never populates `free_versions`, so the source-cleanup preflight comparison of
|
The migration loop handles them before lifecycle expiry and delete-marker
|
||||||
`free_versions` is vacuous for decommission. The migration loop then routes every
|
shortcuts. It selects a target pool using the free-version-aware lookup, then
|
||||||
record through the generic delete-marker handling:
|
writes the original free record to every target disk with the normal metadata
|
||||||
|
write quorum. The free-version marker, local version id, transition identity,
|
||||||
|
transition state, and destination id are preserved at the FileInfo/metadata
|
||||||
|
boundary.
|
||||||
|
|
||||||
- a record that is the only remaining version without replication is skipped by the
|
The source record is physically removed only after the target write quorum has
|
||||||
empty-delete-marker rule and counted as done;
|
committed and the source cleanup preflight still matches the exact inventory.
|
||||||
- any other record is copied to the target pool as an ordinary delete marker with the
|
If target capacity, metadata validation, lock fencing, or quorum fails, the
|
||||||
same version id and mod time.
|
source record remains and the entry records `state = "free_version_retained"`
|
||||||
|
with reason `tier_free_version_migration_failed`; the worker retries the
|
||||||
In both cases the free-version flag and its remote-tier identity are dropped:
|
operation on a later pass. A target record with the same version id is accepted
|
||||||
decommission neither preserves free-version semantics nor performs or reschedules the
|
only when its free-version identity matches; a conflicting ordinary version or
|
||||||
pending remote-tier delete. Source cleanup then removes the original records together
|
different free record is an overwrite error. This makes retries idempotent and
|
||||||
with the source xl.meta.
|
prevents a free record from replacing a user-visible version.
|
||||||
|
|
||||||
Allowed physical-delete timing: the source record may be removed once the migration
|
|
||||||
loop has dispositioned it (copied as a plain marker or skipped as lone), which
|
|
||||||
happens regardless of whether its remote-tier delete was ever performed.
|
|
||||||
|
|
||||||
### Reference-Audit Result
|
### Reference-Audit Result
|
||||||
|
|
||||||
No cluster-local consumer resolves a free version after decommission finishes: GET,
|
After migration, user-facing GET/list/transition/replication/restore paths still
|
||||||
listing, transition planning, replication, restore, and heal operate either on
|
exclude the record. Recovery, usage scanning, lifecycle tier cleanup, and heal
|
||||||
user-visible versions or while the record still exists. The remote exposure is
|
continue to see it when they request free versions, so an unresolved remote
|
||||||
bounded:
|
delete remains actionable on the target pool. The committed tier journal remains
|
||||||
|
an independent retry source where one exists; it is not used as a reason to drop
|
||||||
|
the xl.meta record. In particular, `Unknown` transition state records are
|
||||||
|
migrated unchanged rather than discarded: the lifecycle worker retains them if
|
||||||
|
remote identity validation cannot make a delete request.
|
||||||
|
|
||||||
- On every user-facing delete path the remote-delete obligation is durably carried
|
Each migrated record emits `state = "free_version_migrated"` with reason
|
||||||
by the committed tier-journal entry, which the tier sweeper processes
|
`tier_free_version_migrated`. Each failed record emits the retained state and
|
||||||
independently of xl.meta; the free-version record is an idempotent second
|
failure reason above. The entry also emits a disposition summary with migrated,
|
||||||
pointer, not the only one. Dropping it during decommission therefore does not
|
retained, and total counts. The final decommission sweep uses the exact loader,
|
||||||
orphan the remote object.
|
counts free records still present, and emits one retained record/reason for each
|
||||||
- Residual exposure: for records whose version state decoded as `Unknown` no
|
unresolved free version before failing the sweep. This makes both successful
|
||||||
journal entry exists, so dropping the unconsumed record loses that cleanup hint
|
migration and retained cleanup obligations visible instead of silently omitting
|
||||||
and the remote-tier object is orphaned. The same applies to any future internal
|
free records.
|
||||||
delete path that removes transitioned versions without a journal entry.
|
|
||||||
|
|
||||||
Copying a pending record as an ordinary delete marker also adds a user-visible
|
No new S3-visible version or admin response field is needed: free versions remain
|
||||||
tombstone to the target pool's version history that the source never exposed.
|
internal and are never counted as user-visible versions. The structured
|
||||||
|
`decommission_entry` events are the operational status surface for the
|
||||||
Because of the residual journal-less case, decommission must account for every
|
free-version disposition; the existing decommission item/failed counters still
|
||||||
free-version record instead of omitting it silently:
|
report the enclosing object migration result.
|
||||||
|
|
||||||
- `decommission_free_versions_skipped` counts the records per decommission entry;
|
|
||||||
- entries with a non-zero count log `state = "free_versions_skipped"` with reason
|
|
||||||
`tier_free_version_not_migrated`.
|
|
||||||
|
|
||||||
Regression guard:
|
Regression guard:
|
||||||
|
|
||||||
- `decommission_free_version_accounting_reports_skipped_records`
|
- `decommission_free_version_accounting_records_migration_and_retention`
|
||||||
|
- `decommission_tier_free_version_preserves_remote_identity`
|
||||||
|
|
||||||
## Regression Guard
|
## Regression Guard
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user