From 3cee88f313a8894bc8d96c75fff865e278445104 Mon Sep 17 00:00:00 2001 From: overtrue Date: Sat, 22 Aug 2026 18:37:54 +0800 Subject: [PATCH] feat(ecstore): account for tier free versions in decommission sweep Tier free versions (xl.meta cleanup records for deleted transitioned versions) are not migrated as free versions during decommission: the exact inventory keeps them inline in versions and the migration loop routes them through the generic delete-marker path, dropping the flag and remote-tier identity. Reference audit across GET, heal, ILM, transition, replication, and restore found no cluster-local consumer that resolves a free version after decommission; on user-facing delete paths the remote-delete obligation is also carried by a committed tier-journal entry, leaving only journal-less records (transition state unknown) exposed to remote orphaning. - count and log skipped free versions per decommission entry with disposition reason tier_free_version_not_migrated instead of omitting them silently - document free-version lifecycle, non-migration invariant, allowed physical-delete timing, and the reference-audit result in docs/architecture/decommission-compatibility.md - state the invariant in doc comments at the filemeta free-version sites - guard the accounting with decommission_free_version_accounting_reports_skipped_records Closes rustfs/backlog#1923 --- crates/ecstore/src/core/pools.rs | 65 ++++++++++++-- crates/filemeta/src/filemeta.rs | 16 ++++ crates/filemeta/src/filemeta/version.rs | 9 ++ .../decommission-compatibility.md | 85 +++++++++++++++++++ 4 files changed, 170 insertions(+), 5 deletions(-) diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 04a94cc05..d5886dee1 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -1130,6 +1130,23 @@ fn should_cleanup_decommission_source_entry(decommissioned: usize, total_version decommissioned.saturating_add(expired) == total_versions } +/// Disposition reason logged for tier free-version records that decommission +/// skips instead of migrating. +const DECOMMISSION_FREE_VERSION_SKIP_REASON: &str = "tier_free_version_not_migrated"; + +/// Counts the tier free-version records present in a decommission entry +/// inventory. The exact loader (`load_file_info_versions_exact`) keeps these +/// records inline in `versions` instead of separating them into +/// `free_versions`, and the migration loop then routes them through the +/// generic delete-marker path: the free-version flag and its remote-tier +/// identity are never carried to the target pool, and a lone record is skipped +/// by the empty-delete-marker rule. Accounting for them here keeps the final +/// sweep from silently omitting records whose free-version disposition was +/// dropped (see docs/architecture/decommission-compatibility.md). +fn decommission_free_versions_skipped(fivs: &FileInfoVersions) -> usize { + fivs.versions.iter().filter(|version| version.tier_free_version()).count() +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[allow( dead_code, @@ -3097,6 +3114,22 @@ impl ECStore { fivs.versions .sort_by_key(|v| (v.mod_time.is_none(), std::cmp::Reverse(v.mod_time))); + let skipped_free_versions = decommission_free_versions_skipped(&fivs); + if skipped_free_versions > 0 { + debug!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + bucket = %bucket, + object = %entry.name, + skipped_free_versions, + reason = DECOMMISSION_FREE_VERSION_SKIP_REASON, + state = "free_versions_skipped", + "Decommission skipped free-version migration" + ); + } + let mut decommissioned: usize = 0; let mut expired: usize = 0; let mut cleanup_preflight_allowed_missing = Vec::new(); @@ -5458,11 +5491,12 @@ pub(crate) fn fallback_free_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usi #[cfg(test)] mod pools_tests { use super::{ - DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF, - DecomBucketInfo, DecommissionStartPoolState, DecommissionTerminalState, ListCallback, PoolDecommissionInfo, PoolMeta, - PoolSpaceInfo, PoolStatus, apply_decommission_status_space_info, bind_decommission_cancelers, - bind_missing_decommission_cancelers, cancel_decommission_canceler, classify_decommission_terminal_state, - count_decommission_item, decommission_cancel_signal_result, decommission_item_size, decommission_meta_bucket_options, + DECOMMISSION_FREE_VERSION_SKIP_REASON, DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, + DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF, DecomBucketInfo, DecommissionStartPoolState, DecommissionTerminalState, + ListCallback, PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, PoolStatus, apply_decommission_status_space_info, + bind_decommission_cancelers, bind_missing_decommission_cancelers, cancel_decommission_canceler, + classify_decommission_terminal_state, count_decommission_item, decommission_cancel_signal_result, + decommission_free_versions_skipped, decommission_item_size, decommission_meta_bucket_options, decommission_start_pool_state, dedup_indices, default_decommission_bucket_concurrency, ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed, ensure_decommission_listing_disks_available, ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, ensure_decommission_start_keeps_active_pool, @@ -6762,6 +6796,27 @@ mod pools_tests { assert!(!should_cleanup_decommission_source_entry(2, 2, 1)); } + #[test] + fn decommission_free_version_accounting_reports_skipped_records() { + let mut fivs = FileInfoVersions::default(); + assert_eq!(decommission_free_versions_skipped(&fivs), 0); + + fivs.versions.push(FileInfo { + name: "object.txt".to_string(), + ..Default::default() + }); + let mut free_one = FileInfo::default(); + free_one.set_tier_free_version(); + fivs.versions.push(free_one); + let mut free_two = FileInfo::default(); + free_two.set_tier_free_version(); + free_two.transition_tier = "WARM".to_string(); + fivs.versions.push(free_two); + + assert_eq!(decommission_free_versions_skipped(&fivs), 2); + assert_eq!(DECOMMISSION_FREE_VERSION_SKIP_REASON, "tier_free_version_not_migrated"); + } + #[test] fn test_pool_meta_update_after_rejects_out_of_range_index() { let mut meta = PoolMeta::default(); diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index d2569447c..44bd60014 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -90,6 +90,22 @@ fn legacy_data_key_for_version(version_id: Option) -> Option { pub const TRANSITION_COMPLETE: &str = "complete"; pub const TRANSITION_PENDING: &str = "pending"; +/// xl.meta key marking a tier free-version record. +/// +/// A free version is a delete-marker-shaped cleanup hint appended by +/// [`MetaObject::delete_version`] when a version whose remote transition +/// completed is removed from xl.meta; it carries the remote tier identity for +/// an idempotent remote delete and is never a user-visible version +/// (`num_versions` excludes it). While the record exists it is consumed by the +/// lifecycle free-version recovery scan and the usage scanner, which re-enqueue +/// the pending remote delete, and by heal metadata walks. On S3 and lifecycle +/// delete paths the same obligation is also carried by a committed tier-journal +/// entry; deletes without such an entry (for example a removed version whose +/// transition state decodes as unknown) rely on this record alone until the +/// worker removes it after a successful remote delete. Decommission does not +/// preserve these semantics: its exact inventory keeps the records inline in +/// `versions` and the migration loop treats them as ordinary delete markers — +/// see docs/architecture/decommission-compatibility.md. pub const FREE_VERSION: &str = "free-version"; pub const TRANSITION_STATUS: &str = "transition-status"; diff --git a/crates/filemeta/src/filemeta/version.rs b/crates/filemeta/src/filemeta/version.rs index 37c22b125..967936c3b 100644 --- a/crates/filemeta/src/filemeta/version.rs +++ b/crates/filemeta/src/filemeta/version.rs @@ -2725,6 +2725,15 @@ impl MetaObject { self.meta_sys.retain(|k, _| !k.starts_with("X-Amz-Restore")); } + /// Builds the free-version cleanup record appended when a transitioned + /// version is removed from xl.meta. The record keeps the remote tier + /// identity so the lifecycle worker can issue the idempotent remote delete + /// and only then remove the record; until then the recovery scan and the + /// usage scanner keep re-enqueueing it. S3 and lifecycle deletes also + /// persist a committed tier-journal entry for the same remote delete, so a + /// record destroyed without its remote delete (as decommission does when it + /// treats these records as ordinary delete markers) strands only the + /// journal-less cases — see docs/architecture/decommission-compatibility.md. pub fn init_free_version(&self, fi: &FileInfo) -> Result<(FileMetaVersion, bool)> { if fi.skip_tier_free_version() { return Ok((FileMetaVersion::default(), false)); diff --git a/docs/architecture/decommission-compatibility.md b/docs/architecture/decommission-compatibility.md index e84f90b0e..924e772da 100644 --- a/docs/architecture/decommission-compatibility.md +++ b/docs/architecture/decommission-compatibility.md @@ -153,6 +153,91 @@ No migration step is required for these decisions because this note documents th current RustFS behavior. Changing either decision later requires an operator compatibility note and updated characterization tests. +## Tier Free Versions During Decommission + +A tier free version is an internal xl.meta record (`rustfs_filemeta::FREE_VERSION`, +flagged `XL_FLAG_FREE_VERSION`) shaped like a delete marker. It is created by +`MetaObject::init_free_version` when a version whose remote transition completed is +deleted locally: the visible version is removed and the record keeps the remote-tier +identity (tier, object name, version id, state, destination id) needed for an +idempotent remote delete. Free versions are not user-visible versions; `num_versions` +and all listing/GET paths exclude them. + +### Lifecycle And Consumers + +Creation: any local delete that removes a version whose transition status is +`complete` appends the record via `MetaObject::delete_version` → +`init_free_version` (skipped only when `skip_tier_free_version` is set, as on +data-movement copies). The same deletes also persist a durable tier-journal +entry on every user-facing path: S3 single deletes (`execute_delete_object` → +`delete_object_with_tier_delete_journal`), S3 batch deletes, lifecycle expiry, +and lifecycle delete-all all prepare and commit a journal entry around the +delete. A journal entry is omitted when the removed version's transition state +decodes as `TransitionVersionState::Unknown`, or on internal journal-less +delete paths that never touch transitioned user objects. + +Consumption while the record exists: the background recovery loop started by +`init_background_expiry` (spawned by `spawn_tier_free_version_recovery_once`, +enabled by default) scans disks for pending records and re-enqueues them; the +usage scanner does the same; the lifecycle worker then deletes the remote tier +object idempotently and only afterwards removes the local record. Heal walks +include free-version records in metadata healing. Transition planning, +replication, restore, GET, listings, and usage aggregation never depend on +them. + +### Decommission Handling + +The exact decommission inventory loader (`load_file_info_versions_exact` via +`get_all_file_info_versions`) keeps free-version records inline in `versions`; it +never populates `free_versions`, so the source-cleanup preflight comparison of +`free_versions` is vacuous for decommission. The migration loop then routes every +record through the generic delete-marker handling: + +- a record that is the only remaining version without replication is skipped by the + empty-delete-marker rule and counted as done; +- any other record is copied to the target pool as an ordinary delete marker with the + same version id and mod time. + +In both cases the free-version flag and its remote-tier identity are dropped: +decommission neither preserves free-version semantics nor performs or reschedules the +pending remote-tier delete. Source cleanup then removes the original records together +with the source xl.meta. + +Allowed physical-delete timing: the source record may be removed once the migration +loop has dispositioned it (copied as a plain marker or skipped as lone), which +happens regardless of whether its remote-tier delete was ever performed. + +### Reference-Audit Result + +No cluster-local consumer resolves a free version after decommission finishes: GET, +listing, transition planning, replication, restore, and heal operate either on +user-visible versions or while the record still exists. The remote exposure is +bounded: + +- On every user-facing delete path the remote-delete obligation is durably carried + by the committed tier-journal entry, which the tier sweeper processes + independently of xl.meta; the free-version record is an idempotent second + pointer, not the only one. Dropping it during decommission therefore does not + orphan the remote object. +- Residual exposure: for records whose version state decoded as `Unknown` no + journal entry exists, so dropping the unconsumed record loses that cleanup hint + and the remote-tier object is orphaned. The same applies to any future internal + delete path that removes transitioned versions without a journal entry. + +Copying a pending record as an ordinary delete marker also adds a user-visible +tombstone to the target pool's version history that the source never exposed. + +Because of the residual journal-less case, decommission must account for every +free-version record instead of omitting it silently: + +- `decommission_free_versions_skipped` counts the records per decommission entry; +- entries with a non-zero count log `state = "free_versions_skipped"` with reason + `tier_free_version_not_migrated`. + +Regression guard: + +- `decommission_free_version_accounting_reports_skipped_records` + ## Regression Guard The queued multi-pool contract is guarded by: