fix(storage): integrate main and preserve recovery progress

.github/workflows/rustfs-upgrade-test.yml

@@ -18,7 +18,7 @@ on:
   workflow_dispatch:
     inputs:
       from_version:
-        description: 'OLD RustFS release tag (must ship a .deb asset, e.g. 1.0.0-rc.3)'
+        description: 'OLD RustFS release tag, e.g. 1.0.0-rc.3 (its release must ship a .deb asset). Leave empty for the default.'
         required: false
         default: '1.0.0-rc.3'
       from_url:
@@ -26,7 +26,7 @@ on:
         required: false
         type: string
       to_version:
-        description: 'NEW RustFS release tag (leave empty for latest nightly)'
+        description: 'NEW RustFS release tag, e.g. 1.0.0-rc.5 (any version with a .deb asset). Leave empty for latest nightly.'
         required: false
       to_url:
         description: 'NEW .deb URL. Overrides to_version / nightly default.'
@@ -145,6 +145,7 @@ jobs:
         continue-on-error: true
         env:
           LOG_FILE: /tmp/rustfs-upgrade.log
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-upgrade-test.sh
@@ -175,6 +176,29 @@ jobs:
           else
             ARGS+=(--to-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
           fi
+          # Fail fast with a clear message when a requested release tag has
+          # no .deb asset (e.g. 1.0.0-rc.4 ships only zips), instead of
+          # letting the suite die mid-run on a 404.
+          check_release_asset() {
+            local version="$1" tag asset url
+            [ -n "${version}" ] && [ "${version}" != "null" ] || return 0
+            tag="${version#v}"
+            asset="rustfs_${tag//-/.}_amd64.deb"
+            url="https://github.com/rustfs/rustfs/releases/download/${tag}/${asset}"
+            if ! gh api "repos/rustfs/rustfs/releases/tags/${tag}" --jq '.assets[].name' 2>/dev/null | grep -qxF "${asset}"; then
+              echo "ERROR: release ${tag} has no downloadable asset ${asset}:" >&2
+              echo "       ${url}" >&2
+              echo "Pick a tag whose release ships a .deb (check its release assets)." >&2
+              exit 1
+            fi
+            echo "resolved ${tag} -> ${url}"
+          }
+          if [ -z "${FROM_URL}" ]; then
+            check_release_asset "${FROM_VERSION}"
+          fi
+          if [ -z "${TO_URL}" ]; then
+            check_release_asset "${TO_VERSION}"
+          fi
           ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
 
       - name: Generate report

.gitignore

@@ -33,6 +33,7 @@ profile.json
 *.zst
 .secrets
 *.go
+!crates/zip/tests/fixtures/snowball/**/generate/*.go
 *.pb
 *.svg
 deploy/logs/*.log.*

Cargo.toml

@@ -168,7 +168,7 @@ reqwest = "0.13.4"
 rustfs-kafka-async = { version = "1.3.1" }
 socket2 = { version = "0.6.5" }
 tokio = { version = "1.53.1" }
-tokio-rustls = { default-features = false, version = "0.26.4" }
+tokio-rustls = { default-features = false, version = "0.26.5" }
 tokio-stream = { version = "0.1.19" }
 tokio-test = "0.4.5"
 tokio-util = { version = "0.7.19" }
@@ -234,15 +234,19 @@ tokio-postgres-rustls = "0.14.0"
 # Utilities and Tools
 anyhow = "1.0.104"
 arc-swap = "1.9.2"
-# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until every parser hardening used by Snowball is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published release includes extension, physical-entry, and sparse limits, cancellation-safe sparse parsing, and error-fused entry streams.
+# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin while Snowball and Swift still depend on it. Remove after Snowball uses a released tar-codec/tar-framing API that exposes precedence-resolved MinIO vendor records, RustFS preserves cancellation-safe ownership of large streamed members, footerless minio-go input is accepted only at an authenticated complete request boundary, the existing resource-limit, cancellation, and error-fuse regressions pass, and Swift no longer needs this fork.
 astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" }
+# Candidate Snowball parser versions exercised by rustfs-zip compatibility fixtures.
+tar-codec = "0.0.14"
+tar-framing = "0.0.14"
 atoi = "3.1.0"
 atomic_enum = "0.3.0"
-aws-config = { version = "1.11.0" }
+aws-config = { version = "1.12.0" }
 aws-credential-types = { version = "1.3.0" }
-aws-sdk-kms = { default-features = false, version = "1.117.0" }
+aws-sdk-kms = { default-features = false, version = "1.118.0" }
-aws-sdk-s3 = { default-features = false, version = "1.144.0" }
+aws-sdk-s3 = { default-features = false, version = "1.145.0" }
-aws-sdk-sts = { default-features = false, version = "1.113.0" }
+aws-sdk-sts = { default-features = false, version = "1.114.0" }
+aws-smithy-async = { version = "1.3.0" }
 aws-smithy-http-client = { default-features = false, version = "1.4.0" }
 aws-smithy-runtime-api = { version = "1.16.0" }
 aws-smithy-types = { version = "1.6.3" }

crates/ecstore/Cargo.toml

@@ -244,6 +244,7 @@ windows-sys = { workspace = true, features = [
 windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
 
 [dev-dependencies]
+aws-smithy-async.workspace = true
 tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
 criterion = { workspace = true, features = ["html_reports"] }
 temp-env = { workspace = true, features = ["async_closure"] }

crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs

@@ -1170,6 +1170,7 @@ pub async fn save_manual_transition_job_record_if_current(
         data.clone(),
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_match: Some(current_etag.to_string()),
                 ..Default::default()
@@ -1242,6 +1243,7 @@ pub(crate) async fn save_manual_transition_worker_result_if_absent(
         data,
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_none_match: Some("*".to_string()),
                 ..Default::default()
@@ -1270,6 +1272,7 @@ pub(crate) async fn save_manual_transition_task_if_absent(
         data,
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_none_match: Some("*".to_string()),
                 ..Default::default()
@@ -1621,6 +1624,7 @@ pub async fn save_manual_transition_scope_admission_if_absent(
         data.clone(),
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_none_match: Some("*".to_string()),
                 ..Default::default()
@@ -1672,6 +1676,7 @@ pub async fn save_manual_transition_scope_admission_if_current(
         data.clone(),
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_match: Some(current_etag.to_string()),
                 ..Default::default()

crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs

@@ -1733,6 +1733,7 @@ async fn save_config_if_none_fenced(
         data.clone(),
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_none_match: Some("*".to_string()),
                 ..Default::default()
@@ -1832,6 +1833,7 @@ async fn save_decommission_manifest_checkpoint_if_match(
 
         let mut opts = ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             no_lock: true,
             http_preconditions: Some(HTTPPreconditions {
                 if_match: Some(observed_etag),
@@ -1960,6 +1962,7 @@ async fn save_config_if_match_fenced(
         data.clone(),
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_match: Some(etag.to_string()),
                 ..Default::default()
@@ -3780,6 +3783,7 @@ where
                 data.clone(),
                 &ObjectOptions {
                     max_parity: true,
+                    write_completion: crate::object_api::WriteCompletion::TailDrained,
                     http_preconditions: Some(HTTPPreconditions {
                         if_none_match: Some("*".to_string()),
                         ..Default::default()
@@ -3869,6 +3873,7 @@ where
                 data.clone(),
                 &ObjectOptions {
                     max_parity: true,
+                    write_completion: crate::object_api::WriteCompletion::TailDrained,
                     http_preconditions: Some(HTTPPreconditions {
                         if_match: Some(etag),
                         ..Default::default()
@@ -3893,6 +3898,7 @@ where
                 data.clone(),
                 &ObjectOptions {
                     max_parity: true,
+                    write_completion: crate::object_api::WriteCompletion::TailDrained,
                     http_preconditions: Some(HTTPPreconditions {
                         if_none_match: Some("*".to_string()),
                         ..Default::default()

crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs

@@ -15,8 +15,6 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use super::runtime_boundary as runtime_sources;
 use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryOp;
@@ -72,9 +70,11 @@ static REMOTE_DELETE_BREAKER: LazyLock<Mutex<RemoteDeleteBreaker>> = LazyLock::n
 });
 
 #[cfg(test)]
-static REMOTE_TIER_DELETE_TEST_HOOK: std::sync::LazyLock<
+type RemoteTierDeleteTestHook = Box<dyn Fn(&str, &str, &str) -> std::io::Result<()> + Send + Sync>;
-    std::sync::Mutex<Option<Box<dyn Fn(&str, &str, &str) -> std::io::Result<()> + Send + Sync>>>,
+
-> = std::sync::LazyLock::new(|| std::sync::Mutex::new(None));
+#[cfg(test)]
+static REMOTE_TIER_DELETE_TEST_HOOK: std::sync::LazyLock<std::sync::Mutex<Option<RemoteTierDeleteTestHook>>> =
+    std::sync::LazyLock::new(|| std::sync::Mutex::new(None));
 
 #[derive(Debug)]
 struct RemoteDeleteBreaker {
@@ -107,7 +107,7 @@ impl RemoteDeleteBreaker {
     fn prune(&mut self, now: Instant) {
         while let Some(ts) = self.failures.front().copied() {
             if now.duration_since(ts) > self.window {
-                self.failures.pop_front();
+                let _ = self.failures.pop_front();
             } else {
                 break;
             }
@@ -137,10 +137,10 @@ fn is_signer_header_error(err: &std::io::Error) -> bool {
         return false;
     }
 
-    if let Some(source) = err.get_ref() {
+    if let Some(source) = err.get_ref()
-        if error_chain_contains_signer_header_marker(source) {
+        && error_chain_contains_signer_header_marker(source)
-            return true;
+    {
-        }
+        return true;
     }
 
     let message = err.to_string().to_ascii_lowercase();
@@ -205,7 +205,7 @@ impl ObjSweeper {
 
     #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
     pub fn with_version(&mut self, vid: Option<Uuid>) -> &Self {
-        self.version_id = vid.clone();
+        self.version_id = vid;
         self
     }
 
@@ -219,7 +219,7 @@ impl ObjSweeper {
     #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
     pub fn get_opts(&self) -> lifecycle::ObjectOpts {
         let mut opts = ObjectOpts {
-            version_id: self.version_id.clone(),
+            version_id: self.version_id,
             versioned: self.versioned,
             version_suspended: self.suspended,
             ..Default::default()
@@ -388,8 +388,8 @@ impl Jentry {
 impl ExpiryOp for Jentry {
     fn op_hash(&self) -> u64 {
         let mut hasher = Sha256::new();
-        hasher.update(format!("{}", self.tier_name).as_bytes());
+        hasher.update(self.tier_name.as_bytes());
-        hasher.update(format!("{}", self.obj_name).as_bytes());
+        hasher.update(self.obj_name.as_bytes());
         xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
     }
 
@@ -436,7 +436,7 @@ async fn delete_object_from_remote_tier_raw_with_manager(
     tier_name: &str,
     tier_config_mgr: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
 ) -> Result<(), std::io::Error> {
-    let lease = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, tier_name)
+    let lease = TierConfigMgr::acquire_operation_lease(tier_config_mgr, tier_name)
         .await
         .map_err(std::io::Error::other)?;
     delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false, true).await

crates/ecstore/src/bucket/lifecycle/transition_transaction.rs

@@ -612,6 +612,7 @@ pub(crate) async fn save_transition_transaction_record(
         data.clone(),
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_none_match: Some("*".to_string()),
                 ..Default::default()
@@ -658,6 +659,7 @@ pub(crate) async fn save_transition_transaction_record_if_current(
         data.clone(),
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_match: Some(etag),
                 ..Default::default()

crates/ecstore/src/bucket/on_demand_migration/backfill.rs

@@ -683,6 +683,7 @@ async fn write_checkpoint(
     };
     let opts = ObjectOptions {
         max_parity: true,
+        write_completion: crate::object_api::WriteCompletion::TailDrained,
         http_preconditions: Some(preconditions),
         ..Default::default()
     };
@@ -1190,6 +1191,7 @@ impl Job {
 
     async fn main_loop(&mut self) -> Result<(), Stop> {
         let mut cursor = self.checkpoint.continuation_token.clone();
+        let failed_at_resume = self.checkpoint.failed;
         loop {
             self.check_cancel()?;
             let page = self.list_page(cursor.as_deref()).await?;
@@ -1208,7 +1210,7 @@ impl Job {
             // progress. Keep it at the first failed page for crash recovery.
             self.drain_all().await?;
             cursor = page.next_continuation_token;
-            if self.checkpoint.failed == 0 {
+            if self.checkpoint.failed == failed_at_resume {
                 self.checkpoint.continuation_token = cursor.clone();
             }
             self.tick(true).await?;
@@ -2190,6 +2192,68 @@ mod tests {
         assert_eq!(runner.recover_once().await.taken_over, 0, "a finished job is not recovered");
     }
 
+    #[tokio::test]
+    async fn recovery_advances_past_historical_failures_but_pins_new_failures() {
+        let bucket = "backfill-takeover-failed";
+        let mut context = MockContext::new(8, 2);
+        {
+            let ctx = Arc::get_mut(&mut context).expect("unshared");
+            ctx.auto_complete = AtomicBool::new(false);
+            ctx.fail_keys.insert("k/00004".to_string());
+        }
+        let (_dirs, store, runner) = runner_with("node-b", bucket, Arc::clone(&context)).await;
+        let crashed_at = OffsetDateTime::now_utc() - Duration::from_secs(300);
+        let mut crashed = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-a", crashed_at);
+        crashed.continuation_token = Some("2".to_string());
+        crashed.failed = 1;
+        crashed.record_failure("local_write", Some("k/00002"), crashed_at);
+        write_checkpoint(&store, bucket, &crashed, None)
+            .await
+            .expect("seed failed page with an expired lease");
+
+        assert_eq!(runner.recover_once().await.taken_over, 1);
+        for (page_start, durable_token, failures) in [(2, "2", 1), (4, "4", 1), (6, "4", 2)] {
+            tokio::time::timeout(Duration::from_secs(10), async {
+                loop {
+                    if context.pending.lock().len() == 2 {
+                        break;
+                    }
+                    tokio::task::yield_now().await;
+                }
+            })
+            .await
+            .expect("resumed page enqueued before its reports complete");
+            assert_eq!(
+                context.pending.lock().iter().map(|(key, _)| key.clone()).collect::<Vec<_>>(),
+                vec![format!("k/{page_start:05}"), format!("k/{:05}", page_start + 1)]
+            );
+            let cp = read_checkpoint(&store, bucket)
+                .await
+                .expect("read persisted page boundary")
+                .expect("checkpoint")
+                .checkpoint;
+            assert_eq!(cp.job_id, crashed.job_id);
+            assert_eq!(cp.owner.as_ref().map(|owner| owner.node.as_str()), Some("node-b"));
+            assert_eq!(cp.continuation_token.as_deref(), Some(durable_token));
+            assert_eq!(cp.failed, failures);
+            context.complete_pending();
+        }
+        runner.wait_until_idle(bucket).await;
+        let cp = read_checkpoint(&store, bucket)
+            .await
+            .expect("read completed checkpoint")
+            .expect("checkpoint")
+            .checkpoint;
+        assert_eq!(cp.state, BackfillState::CompletedWithFailures);
+        assert_eq!((cp.pulled, cp.failed), (5, 2));
+        assert_eq!(cp.continuation_token.as_deref(), Some("4"));
+        assert_eq!(cp.failed_keys, vec![key_hash("k/00002"), key_hash("k/00004")]);
+        assert_eq!(
+            context.list_requests.lock().as_slice(),
+            &[Some("2".to_string()), Some("4".to_string()), Some("6".to_string())]
+        );
+    }
+
     #[tokio::test]
     async fn recovery_cancels_a_job_whose_config_changed_and_reclaims_own_node_jobs() {
         let bucket = "backfill-recovery-config";

crates/ecstore/src/bucket/on_demand_migration/breaker.rs

@@ -86,7 +86,12 @@ impl BreakerVerdict {
             Some(SourceError::Throttled | SourceError::Timeout | SourceError::Connect(_) | SourceError::ServerError(_)) => {
                 BreakerVerdict::Failure
             }
-            Some(SourceError::AccessDenied | SourceError::Unsupported(_) | SourceError::Other(_)) => BreakerVerdict::Neutral,
+            Some(
+                SourceError::AccessDenied
+                | SourceError::Unsupported(_)
+                | SourceError::InvalidPagination(_)
+                | SourceError::Other(_),
+            ) => BreakerVerdict::Neutral,
         }
     }
 }

crates/ecstore/src/bucket/on_demand_migration/list_through.rs

@@ -188,8 +188,8 @@ pub enum SourceListPlan {
     /// delimiter — the source's own roll-up boundary matches the request's.
     Page { prefix: String },
     /// `filter.prefix` reaches past a delimiter, so every key the source could
-    /// contribute rolls into this one common prefix. One bounded probe listing
+    /// contribute rolls into this one common prefix. Bounded probes follow
-    /// decides whether it exists; there is nothing to paginate.
+    /// empty progressing pages until a key proves existence or the source ends.
     Folded { probe_prefix: String, common_prefix: String },
 }
 
@@ -278,6 +278,29 @@ pub struct FetchRequest {
     pub token: Option<String>,
 }
 
+/// Invalid pagination metadata. Opaque cursor values are never included in errors.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
+pub enum ListPageError {
+    #[error("truncated listing has no continuation token")]
+    Missing,
+    #[error("truncated listing has an empty continuation token")]
+    Empty,
+    #[error("truncated listing repeats a continuation token")]
+    Repeated,
+}
+
+pub(crate) fn validate_list_page(is_truncated: bool, token: Option<&str>, next_token: Option<&str>) -> Result<(), ListPageError> {
+    if is_truncated {
+        match next_token {
+            None => return Err(ListPageError::Missing),
+            Some("") => return Err(ListPageError::Empty),
+            Some(next) if Some(next) == token => return Err(ListPageError::Repeated),
+            Some(_) => {}
+        }
+    }
+    Ok(())
+}
+
 #[derive(Debug, Default)]
 struct SideState {
     start: SideCursor,
@@ -363,6 +386,11 @@ impl ListThroughMerger {
     /// or `filter.prefix` excludes it.
     pub fn disable_source(&mut self) {
         self.source.disabled = true;
+        // A refill can fail after a valid first page. A local-only response
+        // must discard both that source payload and its ordering horizon.
+        self.source.entries.clear();
+        self.source.pages.clear();
+        self.source.more = false;
     }
 
     pub fn next_fetch(&self) -> Option<FetchRequest> {
@@ -377,7 +405,13 @@ impl ListThroughMerger {
     /// Records one fetched page. `entries` must be sorted by `name` and already
     /// filtered with [`Self::accepts`]; the caller keeps the matching payloads
     /// in the same order.
-    pub fn push_page(&mut self, side: MergeSide, entries: Vec<ListEntryKey>, is_truncated: bool, next_token: Option<String>) {
+    pub fn push_page(
+        &mut self,
+        side: MergeSide,
+        entries: Vec<ListEntryKey>,
+        is_truncated: bool,
+        next_token: Option<String>,
+    ) -> Result<(), ListPageError> {
         let state = match side {
             MergeSide::Local => &mut self.local,
             MergeSide::Source => &mut self.source,
@@ -386,15 +420,19 @@ impl ListThroughMerger {
             Some(last) => last.next_token.clone(),
             None => state.start.token.clone(),
         };
-        // A truncated page without a cursor cannot be continued; treating the
+        validate_list_page(is_truncated, token.as_deref(), next_token.as_deref())?;
-        // side as finished is the only alternative to looping on it forever.
+        // Also reject a cycle through an earlier page in this bounded fetch.
-        state.more = is_truncated && next_token.is_some();
+        if is_truncated && state.pages.iter().any(|page| page.token == next_token) {
+            return Err(ListPageError::Repeated);
+        }
+        state.more = is_truncated;
         state.pages.push(FetchedPage {
             token,
             count: entries.len(),
             next_token: is_truncated.then_some(next_token).flatten(),
         });
         state.entries.extend(entries);
+        Ok(())
     }
 
     pub fn finish(self) -> MergeOutcome {
@@ -598,9 +636,15 @@ mod tests {
                 let (entries, truncated, next) = reference_page(keys, prefix, delimiter, fetch.token.as_deref(), max_keys);
                 let kept: Vec<ListEntryKey> = entries.into_iter().filter(|entry| merger.accepts(&entry.name)).collect();
                 buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.iter().cloned());
-                merger.push_page(fetch.side, kept, truncated, next);
+                merger
+                    .push_page(fetch.side, kept, truncated, next)
+                    .expect("reference provider pages must advance");
             }
             let outcome = merger.finish();
+            assert_eq!(outcome.is_truncated, outcome.next_token.is_some());
+            if outcome.is_truncated {
+                assert_ne!(outcome.next_token, token, "every truncated merged page must make progress");
+            }
             page_sizes.push(outcome.picks.len());
             for pick in &outcome.picks {
                 let entry = buffers[usize::from(pick.side == MergeSide::Source)][pick.index].clone();
@@ -615,11 +659,25 @@ mod tests {
     }
 
     fn expected(local: &[String], source: &[String], prefix: &str, delimiter: Option<&str>) -> Vec<ListEntryKey> {
-        let mut all: Vec<String> = local.iter().chain(source.iter()).cloned().collect();
+        // This oracle builds the complete namespace independently of the
-        all.sort();
+        // provider's page/marker helper and the production merger.
-        all.dedup();
+        let mut namespace = std::collections::BTreeMap::new();
-        let (entries, _, _) = reference_page(&all, prefix, delimiter, None, usize::MAX);
+        for key in local.iter().chain(source) {
-        entries
+            let Some(suffix) = key.strip_prefix(prefix) else {
+                continue;
+            };
+            if let Some(delimiter) = delimiter.filter(|delimiter| !delimiter.is_empty())
+                && let Some((directory, _)) = suffix.split_once(delimiter)
+            {
+                namespace.insert(format!("{prefix}{directory}{delimiter}"), true);
+                continue;
+            }
+            namespace.insert(key.clone(), false);
+        }
+        namespace
+            .into_iter()
+            .map(|(name, is_prefix)| ListEntryKey { name, is_prefix })
+            .collect()
     }
 
     #[test]
@@ -661,7 +719,9 @@ mod tests {
                 token: None
             })
         );
-        merger.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None);
+        merger
+            .push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None)
+            .expect("local EOF is valid");
         assert_eq!(merger.next_fetch(), None);
         let outcome = merger.finish();
         assert_eq!(outcome.picks.len(), 1);
@@ -682,12 +742,14 @@ mod tests {
         };
         let mut merger = ListThroughMerger::new(1, Some(&resume));
         merger.disable_source();
-        merger.push_page(
+        merger
-            MergeSide::Local,
+            .push_page(
-            vec![ListEntryKey::object("b"), ListEntryKey::object("c")],
+                MergeSide::Local,
-            true,
+                vec![ListEntryKey::object("b"), ListEntryKey::object("c")],
-            Some("local-2".to_string()),
+                true,
-        );
+                Some("local-2".to_string()),
+            )
+            .expect("local cursor advances");
         let outcome = merger.finish();
         assert!(outcome.is_truncated);
         let token = outcome.next_token.expect("truncated page carries a token");
@@ -697,6 +759,212 @@ mod tests {
         assert_eq!(token.local.as_deref(), Some("local-1"), "a partly read page is re-listed");
     }
 
+    #[test]
+    fn truncated_pages_require_a_nonempty_advancing_cursor() {
+        for side in [MergeSide::Local, MergeSide::Source] {
+            for entries in [vec![], vec![ListEntryKey::object("a")]] {
+                for (next, expected) in [
+                    (None, Err(ListPageError::Missing)),
+                    (Some(""), Err(ListPageError::Empty)),
+                    (Some("stuck"), Err(ListPageError::Repeated)),
+                    (Some("advances"), Ok(())),
+                ] {
+                    let resume = ListThroughToken::new(
+                        SideCursor {
+                            token: Some("stuck".into()),
+                            done: false,
+                        },
+                        SideCursor {
+                            token: Some("stuck".into()),
+                            done: false,
+                        },
+                        None,
+                    );
+                    let mut merger = ListThroughMerger::new(2, Some(&resume));
+                    let result = merger.push_page(side, entries.clone(), true, next.map(str::to_string));
+                    assert_eq!(result, expected, "{side:?}, {entries:?}, {next:?}");
+                    let state = if side == MergeSide::Local {
+                        &merger.local
+                    } else {
+                        &merger.source
+                    };
+                    assert_eq!(state.pages.len(), usize::from(result.is_ok()), "invalid page must not be accepted");
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn repeated_empty_cursor_is_rejected_before_an_identical_page_can_escape() {
+        let resume = ListThroughToken::new(
+            SideCursor { token: None, done: true },
+            SideCursor {
+                token: Some("stuck".into()),
+                done: false,
+            },
+            None,
+        );
+        let mut merger = ListThroughMerger::new(2, Some(&resume));
+        assert_eq!(
+            merger.next_fetch(),
+            Some(FetchRequest {
+                side: MergeSide::Source,
+                token: Some("stuck".into())
+            })
+        );
+        assert_eq!(
+            merger.push_page(MergeSide::Source, vec![], true, Some("stuck".into())),
+            Err(ListPageError::Repeated)
+        );
+    }
+
+    #[test]
+    fn empty_pages_may_advance_within_the_fetch_budget_until_eof() {
+        let mut merger = ListThroughMerger::new(2, None);
+        merger.push_page(MergeSide::Local, vec![], false, None).expect("local EOF");
+        for next in ["opaque-z", "opaque-a"] {
+            assert_eq!(merger.next_fetch().expect("bounded source fetch").side, MergeSide::Source);
+            merger
+                .push_page(MergeSide::Source, vec![], true, Some(next.into()))
+                .expect("opaque cursor advances regardless of sort order");
+        }
+        assert!(merger.next_fetch().is_none(), "two source fetches exhaust the request budget");
+        let outcome = merger.finish();
+        assert!(outcome.picks.is_empty());
+        assert!(outcome.is_truncated);
+        let token = outcome.next_token.expect("empty progressing page has a cursor");
+        assert_eq!(token.source.as_deref(), Some("opaque-a"));
+        let mut merger = ListThroughMerger::new(2, Some(&token));
+        assert_eq!(merger.next_fetch().expect("source resumes").token.as_deref(), Some("opaque-a"));
+        merger
+            .push_page(MergeSide::Source, vec![ListEntryKey::object("result")], false, None)
+            .expect("source EOF");
+        let outcome = merger.finish();
+        assert_eq!(
+            outcome.picks,
+            vec![MergePick {
+                side: MergeSide::Source,
+                index: 0
+            }]
+        );
+        assert!(!outcome.is_truncated);
+        assert!(outcome.next_token.is_none());
+    }
+
+    #[test]
+    fn a_cursor_cycle_inside_the_fetch_budget_is_rejected() {
+        let resume = ListThroughToken::new(
+            SideCursor { token: None, done: true },
+            SideCursor {
+                token: Some("first".into()),
+                done: false,
+            },
+            None,
+        );
+        let mut merger = ListThroughMerger::new(2, Some(&resume));
+        merger
+            .push_page(MergeSide::Source, vec![], true, Some("second".into()))
+            .expect("first page advances");
+        assert_eq!(
+            merger.push_page(MergeSide::Source, vec![], true, Some("first".into())),
+            Err(ListPageError::Repeated)
+        );
+    }
+
+    #[test]
+    fn source_refill_failure_discards_buffered_source_entries_and_horizon() {
+        let mut merger = ListThroughMerger::new(2, None);
+        merger
+            .push_page(MergeSide::Local, vec![ListEntryKey::object("z")], false, None)
+            .expect("local EOF");
+        merger
+            .push_page(MergeSide::Source, vec![ListEntryKey::object("a")], true, Some("stuck".into()))
+            .expect("first source page advances");
+        assert_eq!(merger.next_fetch().expect("source refill is required").token.as_deref(), Some("stuck"));
+        assert_eq!(
+            merger.push_page(MergeSide::Source, vec![], true, Some("stuck".into())),
+            Err(ListPageError::Repeated)
+        );
+        merger.disable_source();
+        let outcome = merger.finish();
+        assert_eq!(
+            outcome.picks,
+            vec![MergePick {
+                side: MergeSide::Local,
+                index: 0
+            }]
+        );
+        assert!(!outcome.is_truncated);
+        assert!(outcome.next_token.is_none());
+    }
+
+    #[test]
+    fn list_through_static_namespace_boundary_matrix() {
+        let corpus = [
+            "a",
+            "a/",
+            "a/b",
+            "a/b/child",
+            "a0",
+            "b",
+            "b/leaf",
+            "quote\"&<",
+            "space key",
+            "z",
+            "é",
+            "中/文",
+        ];
+        for count in [0, 1, 3, 4, corpus.len()] {
+            let keys: Vec<String> = corpus[..count].iter().map(|key| (*key).to_string()).collect();
+            for placement in 0..3 {
+                let (local, source): (Vec<_>, Vec<_>) =
+                    keys.iter()
+                        .enumerate()
+                        .fold((vec![], vec![]), |(mut local, mut source), (index, key)| {
+                            if placement != 1 || index % 2 == 0 {
+                                local.push(key.clone());
+                            }
+                            if placement != 0 || index % 2 == 0 {
+                                source.push(key.clone());
+                            }
+                            (local, source)
+                        });
+                for prefix in ["", "a", "a/", "中/"] {
+                    for delimiter in [None, Some("/")] {
+                        for max_keys in [1, 3, 4] {
+                            let oracle = expected(&local, &source, prefix, delimiter);
+                            let (emitted, sizes) = walk(&local, &source, prefix, delimiter, max_keys);
+                            assert_eq!(
+                                emitted.iter().map(|(entry, _)| entry.clone()).collect::<Vec<_>>(),
+                                oracle,
+                                "count={count}, placement={placement}, prefix={prefix}, delimiter={delimiter:?}, max={max_keys}"
+                            );
+                            let expected_sizes: Vec<_> = if oracle.is_empty() {
+                                vec![0]
+                            } else {
+                                oracle.chunks(max_keys).map(<[ListEntryKey]>::len).collect()
+                            };
+                            assert_eq!(sizes, expected_sizes, "exact max and max+1 boundaries must agree");
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn list_through_large_overlap_walk_keeps_all_5300_keys() {
+        let source: Vec<_> = (0..5000).map(|index| format!("k{index:05}")).collect();
+        let local: Vec<_> = (4800..5300).map(|index| format!("k{index:05}")).collect();
+        let (emitted, sizes) = walk(&local, &source, "", None, 333);
+        assert_eq!(emitted.len(), 5300);
+        for (index, (entry, side)) in emitted.iter().enumerate() {
+            assert_eq!(entry.name, format!("k{index:05}"));
+            assert_eq!(*side, if index >= 4800 { MergeSide::Local } else { MergeSide::Source });
+        }
+        assert_eq!(sizes, [vec![333; 15], vec![305]].concat());
+    }
+
     #[test]
     fn token_round_trips_and_rejects_tampering() {
         let token = ListThroughToken::new(
@@ -802,7 +1070,10 @@ mod tests {
     }
 
     proptest! {
-        #![proptest_config(ProptestConfig::with_cases(256))]
+        #![proptest_config(ProptestConfig {
+            rng_seed: proptest::test_runner::RngSeed::Fixed(0xec5706),
+            ..ProptestConfig::with_cases(256)
+        })]
 
         /// Full pagination of a merged listing equals the sorted, deduplicated
         /// union of both sides, with every shared key served by local, and no

crates/ecstore/src/bucket/on_demand_migration/source_client.rs

@@ -25,6 +25,7 @@
 //! Client-supplied `If-*`, `Authorization`, `Host` and SSE-C headers are never
 //! forwarded: v1 rejects SSE-C source objects outright.
 
+use super::list_through::{ListPageError, validate_list_page};
 use crate::bucket::remote_s3_client::{
     PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_config,
 };
@@ -223,6 +224,8 @@ pub enum SourceError {
     ServerError(u16),
     #[error("unsupported source object: {0}")]
     Unsupported(String),
+    #[error("invalid source listing: {0}")]
+    InvalidPagination(#[from] ListPageError),
     #[error("source request failed: {0}")]
     Other(String),
 }
@@ -245,6 +248,7 @@ impl SourceError {
             SourceError::Connect(_) => "connect",
             SourceError::ServerError(_) => "server_error",
             SourceError::Unsupported(_) => "unsupported",
+            SourceError::InvalidPagination(_) => "invalid_pagination",
             SourceError::Other(_) => "other",
         }
     }
@@ -712,6 +716,7 @@ impl SourceClient {
                 ..*request
             })
             .await?;
+        validate_list_page(page.is_truncated, request.continuation_token, page.next_continuation_token.as_deref())?;
         page.objects = page
             .objects
             .into_iter()
@@ -799,11 +804,6 @@ impl SourceBackend for S3SourceBackend {
 
         let is_truncated = output.is_truncated.unwrap_or(false);
         let next_continuation_token = output.next_continuation_token;
-        if is_truncated && next_continuation_token.is_none() {
-            return Err(SourceError::Other(
-                "source reported a truncated listing without a continuation token".to_string(),
-            ));
-        }
         let objects = output
             .contents
             .unwrap_or_default()
@@ -1279,7 +1279,9 @@ mod tests {
 <CommonPrefixes><Prefix>data/photos/</Prefix></CommonPrefixes>
 <CommonPrefixes><Prefix>outside/</Prefix></CommonPrefixes>
 </ListBucketResult>"#;
-        let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), body)]).await;
+        let next_body = body.replace("data/opaque", "data/next");
+        let (client, requests) =
+            scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), &next_body)]).await;
         let first = client
             .list_page(&SourceListRequest {
                 prefix: Some("photos/"),
@@ -1341,7 +1343,104 @@ mod tests {
             .list_objects_v2(None, None, 10)
             .await
             .expect_err("truncated page without token is corrupt");
-        assert!(matches!(err, SourceError::Other(_)), "{err:?}");
+        assert!(matches!(err, SourceError::InvalidPagination(ListPageError::Missing)), "{err:?}");
+    }
+
+    #[tokio::test]
+    async fn list_page_validates_s3_cursor_progress_before_mapping_entries() {
+        for contents in ["", "<Contents><Key>data/a</Key><Size>1</Size></Contents>"] {
+            for (truncated, next, expected) in [
+                (true, None, Some(ListPageError::Missing)),
+                (true, Some(""), Some(ListPageError::Empty)),
+                (true, Some("stuck"), Some(ListPageError::Repeated)),
+                (true, Some("opaque-next"), None),
+                (false, None, None),
+                (false, Some("stuck"), None),
+            ] {
+                let next_xml = next
+                    .map(|next| format!("<NextContinuationToken>{next}</NextContinuationToken>"))
+                    .unwrap_or_default();
+                let body = format!(
+                    "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><IsTruncated>{truncated}</IsTruncated>{next_xml}{contents}</ListBucketResult>"
+                );
+                let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), &body)]).await;
+                let result = client
+                    .list_page(&SourceListRequest {
+                        continuation_token: Some("stuck"),
+                        max_keys: 2,
+                        ..Default::default()
+                    })
+                    .await;
+                match expected {
+                    Some(expected) => {
+                        let error = result.expect_err("malformed pagination must fail at the provider boundary");
+                        assert!(
+                            matches!(&error, SourceError::InvalidPagination(actual) if *actual == expected),
+                            "{error:?}"
+                        );
+                        assert_eq!(error.class_label(), "invalid_pagination");
+                        assert!(!error.is_retryable());
+                        assert!(!error.to_string().contains("stuck"), "errors must not echo opaque tokens");
+                    }
+                    None => {
+                        let page = result.expect("progressing empty/nonempty pages and EOF are valid");
+                        assert_eq!(page.is_truncated, truncated);
+                        assert_eq!(page.next_continuation_token.as_deref(), next);
+                        assert_eq!(page.objects.len(), usize::from(!contents.is_empty()));
+                        if let Some(object) = page.objects.first() {
+                            assert_eq!(object.key, "a");
+                        }
+                    }
+                }
+                let requests = recorded(&requests);
+                assert_eq!(requests.len(), 1, "invalid pagination must not be retried");
+                assert!(requests[0].uri.contains("continuation-token=stuck"));
+            }
+        }
+    }
+
+    struct ListOnlyBackend(SourcePage);
+
+    #[async_trait::async_trait]
+    impl SourceBackend for ListOnlyBackend {
+        async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
+            assert_eq!(request.continuation_token, Some("stuck"), "opaque cursors reach every provider unchanged");
+            Ok(self.0.clone())
+        }
+
+        async fn head(&self, _key: &str) -> Result<SourceHead, SourceError> {
+            panic!("unexpected HEAD in list test")
+        }
+        async fn get(&self, _key: &str, _range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
+            panic!("unexpected GET in list test")
+        }
+        async fn tagging(&self, _key: &str) -> Result<HashMap<String, String>, SourceError> {
+            panic!("unexpected tagging in list test")
+        }
+        async fn probe(&self) -> Result<(), SourceError> {
+            panic!("unexpected probe in list test")
+        }
+    }
+
+    #[tokio::test]
+    async fn list_page_validates_non_s3_provider_cursors_at_the_common_boundary() {
+        for (next, expected) in [
+            (None, ListPageError::Missing),
+            (Some(""), ListPageError::Empty),
+            (Some("stuck"), ListPageError::Repeated),
+        ] {
+            let mut client = prefix_client(Some("data/".into()));
+            client.backend = Box::new(ListOnlyBackend(SourcePage {
+                is_truncated: true,
+                next_continuation_token: next.map(str::to_string),
+                ..Default::default()
+            }));
+            let error = client
+                .list_objects_v2(None, Some("stuck"), 2)
+                .await
+                .expect_err("all providers must advance pagination");
+            assert!(matches!(error, SourceError::InvalidPagination(actual) if actual == expected));
+        }
     }
 
     const TAGGING_BODY: &str = r#"<?xml version="1.0" encoding="UTF-8"?>

crates/ecstore/src/bucket/on_demand_migration/stats.rs

@@ -177,7 +177,7 @@ impl From<&SourceError> for PullFailureReason {
             SourceError::Connect(_) => PullFailureReason::SourceConnect,
             SourceError::ServerError(_) => PullFailureReason::SourceServerError,
             SourceError::Unsupported(_) => PullFailureReason::SourceUnsupported,
-            SourceError::Other(_) => PullFailureReason::SourceOther,
+            SourceError::InvalidPagination(_) | SourceError::Other(_) => PullFailureReason::SourceOther,
         }
     }
 }

crates/ecstore/src/bucket/remote_s3_client.rs

@@ -652,9 +652,10 @@ async fn build_aws_s3_http_client_from_tls_path() -> Option<SharedHttpClient> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use aws_smithy_async::time::TimeSource;
     use aws_smithy_runtime_api::http::StatusCode as SmithyStatusCode;
     use std::sync::Mutex;
-    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
 
     fn spec(endpoint: &str, secure: bool) -> RemoteS3EndpointSpec {
         RemoteS3EndpointSpec {
@@ -824,6 +825,174 @@ mod tests {
         );
     }
 
+    #[derive(Clone, Debug)]
+    struct ClockSkewTimeSource(Arc<AtomicU64>);
+
+    impl TimeSource for ClockSkewTimeSource {
+        fn now(&self) -> SystemTime {
+            SystemTime::UNIX_EPOCH + Duration::from_secs(self.0.load(Ordering::SeqCst))
+        }
+    }
+
+    #[derive(Clone, Debug)]
+    struct ClockSkewConnector {
+        request_headers: RecordedHeaders,
+        error_code: &'static str,
+        skew_seconds: i64,
+        clock: ClockSkewTimeSource,
+    }
+
+    fn recorded_header<'a>(headers: &'a [(String, String)], name: &str) -> &'a str {
+        headers
+            .iter()
+            .find(|(key, _)| key.eq_ignore_ascii_case(name))
+            .map(|(_, value)| value.as_str())
+            .unwrap_or_else(|| panic!("signed request must contain {name}"))
+    }
+
+    fn signing_time(headers: &[(String, String)]) -> chrono::NaiveDateTime {
+        chrono::NaiveDateTime::parse_from_str(recorded_header(headers, "x-amz-date"), "%Y%m%dT%H%M%SZ")
+            .expect("SDK signing timestamp must use the SigV4 format")
+    }
+
+    impl SmithyHttpConnector for ClockSkewConnector {
+        fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
+            let mut headers = self.request_headers.lock().expect("clock skew request capture lock");
+            assert!(headers.len() < 3, "clock skew fixture must not exceed two GET attempts and one HEAD");
+            headers.push(
+                request
+                    .headers()
+                    .iter()
+                    .map(|(key, value)| (key.to_string(), value.to_string()))
+                    .collect(),
+            );
+            let server_time = chrono::DateTime::<chrono::Utc>::from(self.clock.now()).naive_utc()
+                + chrono::Duration::seconds(self.skew_seconds);
+            let (status, body) = if headers.len() == 1 {
+                (
+                    403,
+                    format!("<Error><Code>{}</Code><Message>Clock skew fixture</Message></Error>", self.error_code),
+                )
+            } else {
+                (200, String::new())
+            };
+            let response = http::Response::builder()
+                .status(status)
+                .header("date", server_time.format("%a, %d %b %Y %H:%M:%S GMT").to_string())
+                .header("content-type", "application/xml")
+                .header("content-length", body.len())
+                .body(SdkBody::from(body))
+                .expect("clock skew fixture response");
+            HttpConnectorFuture::ready(Ok(HttpResponse::try_from(response).expect("Smithy fixture response")))
+        }
+    }
+
+    async fn clock_skew_client(
+        error_code: &'static str,
+        skew_seconds: i64,
+        retry: RemoteS3RetryPolicy,
+    ) -> (S3Client, RecordedHeaders, ClockSkewTimeSource) {
+        let headers: RecordedHeaders = Arc::new(Mutex::new(Vec::new()));
+        let clock = ClockSkewTimeSource(Arc::new(AtomicU64::new(1_700_000_000)));
+        let connector = SharedHttpConnector::new(ClockSkewConnector {
+            request_headers: Arc::clone(&headers),
+            error_code,
+            skew_seconds,
+            clock: clock.clone(),
+        });
+        let mut spec = spec("s3.example.com", true);
+        spec.retry = retry;
+        let config = build_remote_s3_config(&spec)
+            .await
+            .expect("clock skew fixture uses the production outbound configuration")
+            .http_client(http_client_fn(move |_settings, _components| connector.clone()))
+            .time_source(clock.clone())
+            .build();
+        (S3Client::from_conf(config), headers, clock)
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn remote_s3_clock_skew_retries_resign_and_seed_next_operation() {
+        for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
+            for skew_seconds in [-600, 600] {
+                let (client, headers, clock) = clock_skew_client(error_code, skew_seconds, REPLICATION_TARGET_RETRY_POLICY).await;
+                let initial = chrono::DateTime::<chrono::Utc>::from(clock.now()).naive_utc();
+                client
+                    .get_object()
+                    .bucket("bucket")
+                    .key("object")
+                    .send()
+                    .await
+                    .expect("clock skew GET must retry successfully");
+                assert_eq!(
+                    headers.lock().expect("captured requests").len(),
+                    2,
+                    "{error_code}: GET needs exactly one retry"
+                );
+                clock.0.fetch_add(17, Ordering::SeqCst);
+                // SDK signing time is independent of Tokio's retry/scheduler clock.
+                tokio::time::advance(Duration::from_secs(61)).await;
+                client
+                    .head_bucket()
+                    .bucket("bucket")
+                    .send()
+                    .await
+                    .expect("subsequent HEAD must use the client's cached skew");
+                let headers = headers.lock().expect("captured signed requests");
+                assert_eq!(headers.len(), 3, "subsequent operation must succeed on its first attempt");
+                assert_eq!(signing_time(&headers[0]), initial, "the first attempt must use the injected clock");
+                assert_eq!(
+                    signing_time(&headers[1]),
+                    initial + chrono::Duration::seconds(skew_seconds),
+                    "{error_code}: retry must apply the measured offset exactly"
+                );
+                assert_eq!(
+                    signing_time(&headers[2]),
+                    initial + chrono::Duration::seconds(skew_seconds + 17),
+                    "{error_code}: the next operation must apply cached skew to the advanced signing clock"
+                );
+                let signature = |index: usize| {
+                    recorded_header(&headers[index], "authorization")
+                        .rsplit_once("Signature=")
+                        .expect("SigV4 authorization contains a signature")
+                        .1
+                };
+                assert_ne!(
+                    signature(0),
+                    signature(1),
+                    "{error_code}: retry must be signed again after adjusting its date"
+                );
+            }
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn remote_s3_clock_skew_respects_one_attempt_policy() {
+        use aws_smithy_types::error::metadata::ProvideErrorMetadata;
+
+        for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
+            for retry in [
+                RemoteS3RetryPolicy::Disabled,
+                RemoteS3RetryPolicy::Standard { max_attempts: 1 },
+            ] {
+                let (client, headers, _clock) = clock_skew_client(error_code, 600, retry).await;
+                let error = client
+                    .get_object()
+                    .bucket("bucket")
+                    .key("object")
+                    .send()
+                    .await
+                    .expect_err("clock skew must not override the caller's one-attempt budget");
+                assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some(error_code));
+                assert_eq!(
+                    headers.lock().expect("captured requests").len(),
+                    1,
+                    "{error_code}: {retry:?} must send exactly one request"
+                );
+            }
+        }
+    }
+
     #[test]
     fn path_style_auto_and_path_force_path_style() {
         assert!(PathStyle::Auto.force_path_style());

crates/ecstore/src/core/pools.rs

@@ -5493,6 +5493,7 @@ where
     fence.ensure_held()?;
     let mut opts = ObjectOptions {
         max_parity: true,
+        write_completion: crate::object_api::WriteCompletion::TailDrained,
         no_lock: true,
         http_preconditions: Some(pool_meta_cas_preconditions(token, object)?),
         ..Default::default()
@@ -14412,6 +14413,7 @@ impl ECStore {
                 encoded.clone(),
                 &ObjectOptions {
                     max_parity: true,
+                    write_completion: crate::object_api::WriteCompletion::TailDrained,
                     http_preconditions: Some(HTTPPreconditions {
                         if_none_match: Some("*".to_string()),
                         ..Default::default()
@@ -14566,6 +14568,7 @@ impl ECStore {
                 encoded,
                 &ObjectOptions {
                     max_parity: true,
+                    write_completion: crate::object_api::WriteCompletion::TailDrained,
                     http_preconditions: Some(http_preconditions),
                     ..Default::default()
                 },
@@ -14957,6 +14960,7 @@ impl ECStore {
                 encoded,
                 &ObjectOptions {
                     max_parity: true,
+                    write_completion: crate::object_api::WriteCompletion::TailDrained,
                     http_preconditions: Some(HTTPPreconditions {
                         if_match: Some(etag),
                         ..Default::default()

crates/ecstore/src/disk/disk_store.rs

@@ -317,6 +317,22 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
         dst_path: &str,
         external_guard: Option<Arc<dyn Send + Sync>>,
     ) -> Result<RenameDataResp> {
+        self.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, external_guard)
+            .await
+            .result
+    }
+}
+
+impl LocalDiskWrapper {
+    pub(in crate::disk) async fn rename_data_observed(
+        &self,
+        src_volume: &str,
+        src_path: &str,
+        fi: &FileInfo,
+        dst_volume: &str,
+        dst_path: &str,
+        external_guard: Option<Arc<dyn Send + Sync>>,
+    ) -> super::RenameDataObservation {
         let operation = self.clone();
         let src_volume = src_volume.to_owned();
         let src_path = src_path.to_owned();
@@ -333,22 +349,35 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
         } else {
             get_max_timeout_duration()
         };
-        run_owned_mutation(external_guard, move || async move {
+        let observed = run_owned_mutation(external_guard, move || async move {
-            operation
+            let mut preflight_rejection = None;
+            let result = operation
                 .track_disk_health_mutation(
                     "rename_data",
                     DiskMetricMutation::Write,
                     || async {
-                        operation
+                        // Preserve the former DiskAPI future's single boxing boundary.
-                            .disk
+                        let observed =
-                            .rename_data_borrowed(&src_volume, &src_path, &fi, &dst_volume, &dst_path)
+                            Box::pin(
-                            .await
+                                operation
+                                    .disk
+                                    .rename_data_observed(&src_volume, &src_path, &fi, &dst_volume, &dst_path),
+                            )
+                            .await;
+                        preflight_rejection = observed.preflight_rejection;
+                        observed.result
                     },
                     timeout_duration,
                 )
-                .await
+                .await;
+            // Health tracking must observe the real disk error, not an Ok tuple.
+            Ok(super::RenameDataObservation {
+                result,
+                preflight_rejection,
+            })
         })
-        .await
+        .await;
+        observed.unwrap_or_else(|error| super::RenameDataObservation::unknown(Err(error)))
     }
 }
 
@@ -2588,6 +2617,46 @@ mod tests {
         assert_eq!(wrapper.metrics_snapshot().api_calls.get("unknown"), Some(&1));
     }
 
+    #[tokio::test]
+    async fn rename_preflight_evidence_preserves_health_errors_and_owned_reply() {
+        for source_exists in [false, true] {
+            for guarded in [false, true] {
+                let dir = tempfile::tempdir().expect("temp dir should be created");
+                let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8"))
+                    .expect("endpoint should parse");
+                let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
+                if source_exists {
+                    disk.make_volume("source").await.expect("source volume should exist");
+                }
+                let wrapper = LocalDiskWrapper::new(disk, false);
+                let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+                let external_guard = guarded.then(|| Arc::new(DropProbe(Arc::clone(&drops))) as Arc<dyn Send + Sync>);
+                let mut file_info = FileInfo::new("object", 1, 0);
+                file_info.mod_time = Some(::time::OffsetDateTime::now_utc());
+                file_info.erasure.index = 1;
+                let observed = wrapper
+                    .rename_data_observed("source", "object", &file_info, "missing-destination", "object", external_guard)
+                    .await;
+                assert!(observed.rejected_before_publication(), "normal access rejection must carry proof");
+                assert!(matches!(observed.result, Err(DiskError::VolumeNotFound)));
+                let snapshot = wrapper.metrics_snapshot();
+                assert_eq!(snapshot.api_calls.get("rename_data"), Some(&1));
+                assert_eq!(snapshot.total_writes, 0, "health tracking must not observe the rejection as Ok");
+                assert_eq!(drops.load(Ordering::SeqCst), usize::from(guarded));
+
+                wrapper.health.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
+                let observed = wrapper
+                    .rename_data_observed("source", "object", &file_info, "missing-destination", "object", None)
+                    .await;
+                assert!(!observed.rejected_before_publication(), "wrapper errors carry no local preflight proof");
+                assert!(matches!(observed.result, Err(DiskError::FaultyDisk)));
+                let snapshot = wrapper.metrics_snapshot();
+                assert_eq!(snapshot.total_errors_availability, 1);
+                assert_eq!(snapshot.total_writes, 0);
+            }
+        }
+    }
+
     #[tokio::test]
     async fn local_disk_health_wrapper_counts_returned_availability_errors() {
         let dir = tempfile::tempdir().expect("temp dir should be created");

crates/ecstore/src/disk/mod.rs

@@ -75,6 +75,25 @@ use time::OffsetDateTime;
 use tokio::io::{AsyncRead, AsyncWrite};
 use uuid::Uuid;
 
+/// Local preflight evidence stays outside DiskAPI and the RPC response format.
+pub(crate) struct RenameDataObservation {
+    pub(crate) result: Result<RenameDataResp>,
+    preflight_rejection: Option<local::LocalRenamePreflightRejection>,
+}
+
+impl RenameDataObservation {
+    fn unknown(result: Result<RenameDataResp>) -> Self {
+        Self {
+            result,
+            preflight_rejection: None,
+        }
+    }
+
+    pub(crate) fn rejected_before_publication(&self) -> bool {
+        self.result.is_err() && self.preflight_rejection.is_some()
+    }
+}
+
 const QUOTA_MUTATION_FENCE_PREFIX: &str = "tmp/quota-mutation-fences/";
 pub(crate) const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token";
 
@@ -711,6 +730,36 @@ impl Disk {
             .await
     }
 
+    pub(crate) async fn rename_data_borrowed_with_fence_observed(
+        &self,
+        src_volume: &str,
+        src_path: &str,
+        fi: &FileInfo,
+        dst_volume: &str,
+        dst_path: &str,
+        scanner_publication_lease_token: Option<Uuid>,
+    ) -> RenameDataObservation {
+        match self {
+            Disk::Local(local_disk) => {
+                local_disk
+                    .rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, None)
+                    .await
+            }
+            Disk::Remote(remote_disk) => RenameDataObservation::unknown(
+                remote_disk
+                    .rename_data_borrowed_with_fence(
+                        src_volume,
+                        src_path,
+                        fi,
+                        dst_volume,
+                        dst_path,
+                        scanner_publication_lease_token,
+                    )
+                    .await,
+            ),
+        }
+    }
+
     pub(crate) async fn rename_data_borrowed_with_fence(
         &self,
         src_volume: &str,

crates/ecstore/src/object_api/types.rs

@@ -870,6 +870,18 @@ impl TierFreeVersionReceiptSink {
     }
 }
 
+/// Internal PUT completion boundary; this does not change fsync or write quorum.
+#[doc(hidden)]
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
+pub enum WriteCompletion {
+    /// Return at write quorum when the commit owner can retain its guards.
+    #[default]
+    Quorum,
+    /// Drain the rename fan-out before returning. Minority failures still heal
+    /// after a successful quorum commit; this does not require every disk to succeed.
+    TailDrained,
+}
+
 #[derive(Default, Clone)]
 pub struct ObjectOptions {
     // Use the maximum parity (N/2), used when saving server configuration files
@@ -896,6 +908,10 @@ pub struct ObjectOptions {
     /// Persisted bucket incarnation observed before authorization.
     pub expected_bucket_incarnation_id: Option<Uuid>,
     pub no_lock: bool,
+    /// Control-plane writers that immediately read or CAS the same namespace
+    /// key use TailDrained without changing namespace lock ownership.
+    #[doc(hidden)]
+    pub write_completion: WriteCompletion,
     /// True when an upper layer already holds the object read lock before
     /// forwarding a no_lock read to the set layer.
     pub metadata_cache_safe: bool,

crates/ecstore/src/services/tier/tier_admin.rs

@@ -15,8 +15,6 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
 
@@ -145,7 +143,7 @@ mod tests {
 
         assert_eq!(creds.access_key, "access");
         assert_eq!(creds.secret_key, "secret");
-        assert_eq!(creds.creds_json.as_slice(), &service_account[..]);
+        assert_eq!(creds.creds_json.as_slice(), service_account);
 
         let wire = serde_json::to_value(&creds).expect("madmin tier credentials should encode");
         assert_eq!(wire["access"], "access");
@@ -162,7 +160,7 @@ mod tests {
         .expect("the former RustFS field names and byte-array encoding should remain readable");
         assert_eq!(legacy.access_key, "legacy-access");
         assert_eq!(legacy.secret_key, "legacy-secret");
-        assert_eq!(legacy.creds_json.as_slice(), &service_account[..]);
+        assert_eq!(legacy.creds_json.as_slice(), service_account);
     }
 
     #[test]

crates/ecstore/src/services/tier/tier_mutation_intent.rs

@@ -460,6 +460,7 @@ where
         data,
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_none_match: Some("*".to_string()),
                 ..Default::default()
@@ -556,6 +557,7 @@ where
         data,
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_match: Some(current_etag.to_string()),
                 ..Default::default()

crates/ecstore/src/services/tier/tier_probe_intent.rs

@@ -494,6 +494,7 @@ where
         data,
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_none_match: Some("*".to_string()),
                 ..Default::default()
@@ -549,6 +550,7 @@ where
         data,
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_match: Some(current.record_etag.clone()),
                 ..Default::default()

crates/ecstore/src/services/tier/warm_backend_aliyun.rs

@@ -15,8 +15,6 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use std::collections::HashMap;
 

crates/ecstore/src/services/tier/warm_backend_azure.rs

@@ -15,8 +15,6 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use std::collections::HashMap;
 

crates/ecstore/src/services/tier/warm_backend_gcs.rs

@@ -15,8 +15,6 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use std::collections::{HashMap, HashSet};
 use std::future::Future;
@@ -146,11 +144,11 @@ pub struct WarmBackendGCS {
 
 impl WarmBackendGCS {
     pub async fn new(conf: &TierGCS, tier: &str) -> Result<Self, std::io::Error> {
-        if conf.creds == "" {
+        if conf.creds.is_empty() {
             return Err(std::io::Error::other("both access and secret keys are required"));
         }
 
-        if conf.bucket == "" {
+        if conf.bucket.is_empty() {
             return Err(std::io::Error::other("no bucket name was provided"));
         }
 
@@ -195,11 +193,11 @@ impl WarmBackendGCS {
     }
 
     pub fn get_dest(&self, object: &str) -> String {
-        let mut dest_obj = object.to_string();
+        if self.prefix.is_empty() {
-        if self.prefix != "" {
+            object.to_string()
-            dest_obj = format!("{}/{}", &self.prefix, object);
+        } else {
+            format!("{}/{}", self.prefix, object)
         }
-        return dest_obj;
     }
 }
 
@@ -223,7 +221,7 @@ impl WarmBackend for WarmBackendGCS {
         let bucket = gcs_bucket_resource_name(&self.bucket);
         let Ok(res) = Box::pin(
             self.client
-                .write_object(&bucket, &self.get_dest(object), Bytes::from(d))
+                .write_object(&bucket, self.get_dest(object), Bytes::from(d))
                 .send_buffered(),
         )
         .await
@@ -240,7 +238,7 @@ impl WarmBackend for WarmBackendGCS {
 
     async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
         let bucket = gcs_bucket_resource_name(&self.bucket);
-        let mut req = self.client.read_object(&bucket, &self.get_dest(object));
+        let mut req = self.client.read_object(&bucket, self.get_dest(object));
         let mut max_response_bytes = None;
         if let Some(generation) = parse_generation(rv)? {
             req = req.set_generation(generation);

crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs

@@ -15,8 +15,6 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use std::collections::HashMap;
 

crates/ecstore/src/services/tier/warm_backend_minio.rs

@@ -15,8 +15,6 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use std::collections::HashMap;
 

crates/ecstore/src/services/tier/warm_backend_r2.rs

@@ -15,8 +15,6 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use std::collections::HashMap;
 

crates/ecstore/src/services/tier/warm_backend_rustfs.rs

@@ -15,8 +15,6 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use std::collections::HashMap;
 

crates/ecstore/src/services/tier/warm_backend_tencent.rs

@@ -15,8 +15,6 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use std::collections::HashMap;
 

crates/ecstore/src/set_disk/ops/object.rs

@@ -299,11 +299,11 @@ use crate::error::is_err_invalid_upload_id;
 use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
 use crate::object_api::{
     NamespaceLockFence, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode,
-    SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY,
+    SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, WriteCompletion,
 };
 use crate::services::notification_sys::RemoteVersionStateFleetProofToken;
 use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata};
-use crate::set_disk::core::io_primitives::{RenameTailCleanup, finish_rename_tail_heal};
+use crate::set_disk::core::io_primitives::{RenameRollbackReceipt, RenameTailCleanup, finish_rename_tail_heal};
 #[cfg(test)]
 use crate::storage_api_contracts::namespace::NamespaceLocking;
 #[cfg(test)]
@@ -3548,6 +3548,7 @@ impl SetDisks {
                 (None, None, None)
             };
         let mut tmp_cleanup_owned = false;
+        let rollback_receipt = RenameRollbackReceipt::default();
         let operation = async {
             let erasure = Arc::new(erasure_from_file_info(&fi, false)?);
 
@@ -4256,6 +4257,7 @@ impl SetDisks {
             let commit_bucket = bucket.to_owned();
             let commit_object = object.to_owned();
             let commit_tmp_dir = tmp_dir.clone();
+            let commit_rollback_receipt = rollback_receipt.clone();
             let commit_object_lock_guard = object_lock_guard.take();
             let commit_decommission_object_lock_guard = decommission_object_lock_guard.take();
             let commit_publication_guard = publication_commit_guard.take();
@@ -4266,13 +4268,17 @@ impl SetDisks {
             // complete rename fan-out drains. Keep this path synchronous so
             // its terminal state is known before the coordinator releases
             // remote leases.
-            let commit_allows_early_ack = !(opts.data_movement && opts.has_decommission_capacity_reservation())
+            let commit_owns_namespace_guard = commit_object_lock_guard.is_some()
-                && (commit_object_lock_guard.is_some()
+                || commit_decommission_object_lock_guard.is_some()
-                    || commit_decommission_object_lock_guard.is_some()
+                || commit_publication_guard.is_some();
-                    || commit_publication_guard.is_some())
+            let commit_allows_early_ack = opts.write_completion == WriteCompletion::Quorum
+                && !(opts.data_movement && opts.has_decommission_capacity_reservation())
+                && commit_owns_namespace_guard
                 && commit_scanner_publication_scope.is_none();
+            // Full-tail callers also transfer owned guards to the coordinator:
+            // cancelling their ACK waiter must not cancel an in-flight rename.
             let detach_commit_owner = commit_scanner_publication_scope.is_some()
-                || commit_allows_early_ack
+                || commit_owns_namespace_guard
                 || commit_bucket_lifecycle_guard.is_some()
                 || quota_mutation_fence;
             let commit_write_path_label = write_path.metric_label();
@@ -4453,6 +4459,7 @@ impl SetDisks {
                         commit_scanner_publication_lease_tokens.as_ref(),
                     )
                     .with_publication_scope(commit_scanner_publication_scope.clone())
+                    .with_rollback_receipt(commit_rollback_receipt.clone())
                     .with_namespace_commit_guard(
                         (!is_meta_bucketname(&commit_bucket)).then(|| commit_set.ctx.begin_namespace_commit()),
                     ),
@@ -4588,6 +4595,11 @@ impl SetDisks {
                 let rename_commit = match rename_result {
                     Ok(commit) => commit,
                     Err(err) => {
+                        if commit_rollback_receipt.is_incomplete() {
+                            // Incomplete undo retains the staging source and
+                            // rollback backup for recovery; cleanup is unsafe.
+                            return Err(err.into());
+                        }
                         if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await {
                             warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data");
                         } else if issue3031_diag_enabled() {
@@ -4620,9 +4632,8 @@ impl SetDisks {
                     request.object_version_id = committed_version_id
                         .or_else(|| commit_version_suspended.then(Uuid::nil))
                         .map(|version_id| version_id.to_string());
-                    tokio::spawn(async move {
+                    let heal_set = commit_set.clone();
-                        let _ = rustfs_heal_contracts::heal_channel::send_heal_request(request).await;
+                    tokio::spawn(async move { heal_set.submit_rename_tail_heal(request).await });
-                    });
                 }
 
                 let rename_stage_elapsed = rename_stage_start.elapsed();
@@ -4888,7 +4899,7 @@ impl SetDisks {
                     );
                 }
             });
-        } else {
+        } else if !rollback_receipt.is_incomplete() {
             // Failure path (quorum loss / rollback): keep the cleanup inline so
             // a failed PUT never returns while its tmp shards are still on disk
             // (state-residue hardening tracked by backlog#864 / backlog#898).
@@ -17497,27 +17508,69 @@ mod put_object_tmp_cleanup_tests {
     }
 
     #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
     async fn put_object_failure_cleans_tmp_workspace_inline() {
-        let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
+        temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
+            for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] {
+                let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
+                let bucket = "tmp-clean-missing-bucket";
+                let object = "orphan-object";
+                let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace);
+                let writer = Arc::clone(&set_disks);
+                let put = tokio::spawn(async move {
+                    let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]);
+                    writer
+                        .put_object(
+                            bucket,
+                            object,
+                            &mut reader,
+                            &ObjectOptions {
+                                write_completion,
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                });
+                tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
+                    .await
+                    .expect("missing-bucket PUT must stage before rename");
+                let staged = non_trash_tmp_entries(&temp_dirs).await;
+                assert_eq!(staged.len(), 4, "every disk must have a staged workspace before rejection");
+                for workspace in staged {
+                    let mut entries = tokio::fs::read_dir(&workspace)
+                        .await
+                        .expect("staged workspace should be readable");
+                    let mut shards = 0;
+                    while let Some(entry) = entries.next_entry().await.expect("staged data directory should be readable") {
+                        if entry.file_type().await.expect("staged entry type").is_dir() {
+                            let part = tokio::fs::metadata(entry.path().join("part.1"))
+                                .await
+                                .expect("staging must contain an actual erasure shard");
+                            assert!(part.len() > 0, "the shard must be written before the missing-bucket failure");
+                            shards += 1;
+                        }
+                    }
+                    assert_eq!(shards, 1);
+                }
+                assert!(temp_dirs.iter().all(|dir| !dir.path().join(bucket).exists()));
+                barrier.release();
+                let err = tokio::time::timeout(Duration::from_secs(30), put)
+                    .await
+                    .expect("missing-bucket PUT must finish")
+                    .expect("PUT task should join")
+                    .expect_err("put_object into a missing bucket volume must fail");
+                assert!(matches!(err, StorageError::VolumeNotFound), "original disk error expected: {err}");
 
-        // The bucket volume is never created, so the shards are written into
+                // No polling: known pre-publication rejection must clean staging
-        // the tmp workspace and the commit fails at rename_data with a quorum
+                // inline, before PUT returns (backlog#864 / backlog#898).
-        // error — exercising the failure-path cleanup.
+                let leftovers = non_trash_tmp_entries(&temp_dirs).await;
-        let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]);
+                assert!(
-        let err = set_disks
+                    leftovers.is_empty(),
-            .put_object("tmp-clean-missing-bucket", "orphan-object", &mut reader, &ObjectOptions::default())
+                    "failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}"
-            .await
+                );
-            .expect_err("put_object into a missing bucket volume must fail");
+            }
-
+        })
-        // No polling: the failure path must clean the tmp workspace inline,
+        .await;
-        // before put_object returns (backlog#864 / backlog#898 hardening).
-        let leftovers = non_trash_tmp_entries(&temp_dirs).await;
-        assert!(
-            leftovers.is_empty(),
-            "failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}"
-        );
-
-        drop(temp_dirs);
     }
 
     #[tokio::test]
@@ -18160,6 +18213,354 @@ mod put_object_tmp_cleanup_tests {
         .await;
     }
 
+    async fn make_completion_test_bucket(disks: &[DiskStore], bucket: &str) {
+        for disk in disks {
+            disk.make_volume(bucket)
+                .await
+                .expect("completion test bucket should be created");
+        }
+    }
+
+    /// Observe the actual metadata quorum while the remaining rename is parked.
+    /// A completed task count alone can race tasks that have not started yet.
+    async fn wait_for_paused_tail_metadata_quorum(disks: &[DiskStore], bucket: &str, object: &str) {
+        tokio::time::timeout(Duration::from_secs(30), async {
+            loop {
+                let mut committed = 0;
+                for disk in disks {
+                    match disk.read_version("", bucket, object, "", &ReadOptions::default()).await {
+                        Ok(_) => committed += 1,
+                        Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => {}
+                        Err(err) => panic!("unexpected metadata error while observing {bucket}/{object}: {err}"),
+                    }
+                }
+                if committed == 3 {
+                    break;
+                }
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("three disks must publish metadata while the fourth rename remains paused");
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn tail_drained_put_waits_for_tail_and_allows_immediate_cas() {
+        temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
+            for size in [4096, 1024 * 1024] {
+                let (_dirs, disks, set) = hermetic_set_disks(4).await;
+                let bucket = "put-full-tail-cas";
+                let object = "full-tail-cas-object";
+                make_completion_test_bucket(&disks, bucket).await;
+                let tasks = rename_fanout_barrier::observe_tasks(object);
+                let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
+                let writer = Arc::clone(&set);
+                let put = tokio::spawn(async move {
+                    let mut reader = PutObjReader::from_vec(vec![b'1'; size]);
+                    writer
+                        .put_object(
+                            bucket,
+                            object,
+                            &mut reader,
+                            &ObjectOptions {
+                                write_completion: WriteCompletion::TailDrained,
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                });
+                tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
+                    .await
+                    .expect("full-tail PUT must reach the rename barrier");
+                wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
+                assert!(!put.is_finished(), "full-tail PUT must remain pending after metadata quorum");
+                let mut lock_probe = Box::pin(set.acquire_write_lock_diag("full_tail_probe", bucket, object));
+                assert!(
+                    futures::poll!(lock_probe.as_mut()).is_pending(),
+                    "the owned namespace guard must remain held"
+                );
+                barrier.release();
+                let written = tokio::time::timeout(Duration::from_secs(30), put)
+                    .await
+                    .expect("full-tail PUT should finish after release")
+                    .expect("full-tail PUT task should join")
+                    .expect("full-tail PUT must commit");
+                assert_eq!(tasks.running(), 0, "full-tail response must follow every rename task");
+                drop(
+                    tokio::time::timeout(Duration::from_secs(5), lock_probe)
+                        .await
+                        .expect("same-key lock should be available on return")
+                        .expect("same-key lock probe should succeed"),
+                );
+                for disk in &disks {
+                    disk.read_version("", bucket, object, "", &ReadOptions::default())
+                        .await
+                        .expect("successful full-tail PUT must publish on every healthy disk");
+                }
+                drop(barrier);
+                let mut replacement = PutObjReader::from_vec(b"cas successor".to_vec());
+                set.put_object(
+                    bucket,
+                    object,
+                    &mut replacement,
+                    &ObjectOptions {
+                        write_completion: WriteCompletion::TailDrained,
+                        http_preconditions: Some(HTTPPreconditions {
+                            if_match: written.etag,
+                            ..Default::default()
+                        }),
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("immediate same-key CAS must acquire the namespace guard");
+                let mut read = set
+                    .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+                    .await
+                    .expect("CAS successor must be immediately readable");
+                let mut body = Vec::new();
+                read.stream.read_to_end(&mut body).await.expect("successor body must drain");
+                assert_eq!(body, b"cas successor");
+            }
+        })
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn tail_drained_put_preserves_quorum_success_and_heals_failed_tail() {
+        let (_dirs, disks, set) = hermetic_set_disks(4).await;
+        let bucket = "put-full-tail-heal";
+        let object = "full-tail-heal-object";
+        make_completion_test_bucket(&disks, bucket).await;
+        let mut heals = set.capture_test_rename_tail_heals();
+        let tasks = rename_fanout_barrier::observe_tasks(object);
+        let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
+        let _fault = rename_fault_injection::fail_rename_on(object, &[0]);
+        let writer = Arc::clone(&set);
+        let put = tokio::spawn(async move {
+            let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
+            writer
+                .put_object(
+                    bucket,
+                    object,
+                    &mut reader,
+                    &ObjectOptions {
+                        write_completion: WriteCompletion::TailDrained,
+                        ..Default::default()
+                    },
+                )
+                .await
+        });
+        tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
+            .await
+            .expect("failed tail must first reach the rename barrier");
+        wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
+        assert!(!put.is_finished(), "committed quorum must still wait for the failing tail");
+        barrier.release();
+        tokio::time::timeout(Duration::from_secs(30), put)
+            .await
+            .expect("failed tail should drain")
+            .expect("PUT task should join")
+            .expect("a minority tail error must not negate committed quorum");
+        assert_eq!(tasks.running(), 0);
+        let heal = tokio::time::timeout(Duration::from_secs(30), heals.recv())
+            .await
+            .expect("failed tail must schedule heal")
+            .expect("heal capture must remain connected");
+        assert_eq!(heal.bucket, bucket);
+        assert_eq!(heal.object_prefix.as_deref(), Some(object));
+        let info = set
+            .get_object_info(bucket, object, &ObjectOptions::default())
+            .await
+            .expect("committed object must remain readable despite the failed tail");
+        assert_eq!(info.size, TEST_OBJECT_SIZE as i64);
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn tail_drained_put_rejects_quorum_minus_one() {
+        let (_dirs, disks, set) = hermetic_set_disks(4).await;
+        let bucket = "put-full-tail-no-quorum";
+        let object = "full-tail-no-quorum-object";
+        make_completion_test_bucket(&disks, bucket).await;
+        let _fault = rename_fault_injection::fail_rename_on(object, &[0, 1]);
+        let tasks = rename_fanout_barrier::observe_tasks(object);
+        let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
+        let err = set
+            .put_object(
+                bucket,
+                object,
+                &mut reader,
+                &ObjectOptions {
+                    write_completion: WriteCompletion::TailDrained,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect_err("draining two successful disks cannot satisfy write quorum three");
+        assert!(
+            matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)),
+            "original quorum error expected: {err}"
+        );
+        assert_eq!(tasks.running(), 0, "failed fan-out and rollback must complete before return");
+        assert!(
+            set.get_object_info(bucket, object, &ObjectOptions::default()).await.is_err(),
+            "failed fresh write must not become visible"
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn put_incomplete_rollback_preserves_staging_and_old_version_backup() {
+        use crate::set_disk::core::io_primitives::rollback_fault_injection;
+
+        temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
+            for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] {
+                for fault in [
+                    rollback_fault_injection::Fault::Io,
+                    rollback_fault_injection::Fault::VolumeNotFoundAfterRename,
+                ] {
+                    let (dirs, disks, set) = hermetic_set_disks(4).await;
+                    let bucket = "put-incomplete-undo";
+                    let object = "incomplete-undo-object";
+                    make_completion_test_bucket(&disks, bucket).await;
+                    let mut old_reader = PutObjReader::from_vec(vec![b'0'; TEST_OBJECT_SIZE]);
+                    set.put_object(
+                        bucket,
+                        object,
+                        &mut old_reader,
+                        &ObjectOptions {
+                            write_completion: WriteCompletion::TailDrained,
+                            ..Default::default()
+                        },
+                    )
+                    .await
+                    .expect("old generation should be completely committed");
+                    wait_for_tmp_workspace_to_drain(&dirs, "old PUT must leave no unrelated staging").await;
+                    let old = disks[0]
+                        .read_version("", bucket, object, "", &ReadOptions::default())
+                        .await
+                        .expect("old metadata must be readable");
+                    let old_data_dir = old.data_dir.expect("non-inline old version needs a data directory");
+                    let tasks = rename_fanout_barrier::observe_tasks(object);
+                    let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
+                    let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]);
+                    let _undo_fault = rollback_fault_injection::arm(object, 0, fault);
+                    let writer = Arc::clone(&set);
+                    let put = tokio::spawn(async move {
+                        let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
+                        writer
+                            .put_object(
+                                bucket,
+                                object,
+                                &mut reader,
+                                &ObjectOptions {
+                                    write_completion,
+                                    ..Default::default()
+                                },
+                            )
+                            .await
+                    });
+                    tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
+                        .await
+                        .expect("overwrite must enter the actual rename fan-out before failure injection");
+                    barrier.release();
+                    let err = tokio::time::timeout(Duration::from_secs(30), put)
+                        .await
+                        .expect("incomplete undo must return without hanging")
+                        .expect("PUT task should join")
+                        .expect_err("two renamed disks cannot satisfy write quorum three");
+                    assert!(
+                        matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)),
+                        "original quorum error expected: {err}"
+                    );
+                    assert_eq!(tasks.running(), 0, "every rename and undo task must be reaped before return");
+                    let leftovers = non_trash_tmp_entries(&dirs).await;
+                    assert!(!leftovers.is_empty(), "incomplete undo must retain the new staging source for recovery");
+                    let backups = dirs
+                        .iter()
+                        .filter(|dir| {
+                            dir.path()
+                                .join(bucket)
+                                .join(object)
+                                .join(old_data_dir.to_string())
+                                .join(crate::disk::STORAGE_FORMAT_FILE_BACKUP)
+                                .exists()
+                        })
+                        .count();
+                    assert_eq!(backups, 1, "exactly the failed undo disk must retain its old-version backup");
+                    // The remaining three disks still serve the old generation;
+                    // the failed minority must never become an acknowledged write.
+                    let mut read = set
+                        .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+                        .await
+                        .expect("old generation must remain readable after incomplete rollback");
+                    let mut body = Vec::new();
+                    read.stream
+                        .read_to_end(&mut body)
+                        .await
+                        .expect("old generation should stream");
+                    assert_eq!(body, vec![b'0'; TEST_OBJECT_SIZE]);
+                }
+            }
+        })
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn tail_drained_put_owned_commit_survives_waiter_cancellation() {
+        let (dirs, disks, set) = hermetic_set_disks(4).await;
+        let bucket = RUSTFS_META_BUCKET;
+        let object = "full-tail-cancelled-receipt";
+        // Internal config writes do not own a bucket lifecycle guard. The object
+        // guard alone must keep the full-tail coordinator alive after cancellation.
+        let tasks = rename_fanout_barrier::observe_tasks(object);
+        let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
+        let writer = Arc::clone(&set);
+        let put = tokio::spawn(async move {
+            let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
+            writer
+                .put_object(
+                    bucket,
+                    object,
+                    &mut reader,
+                    &ObjectOptions {
+                        write_completion: WriteCompletion::TailDrained,
+                        ..Default::default()
+                    },
+                )
+                .await
+        });
+        tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
+            .await
+            .expect("cancelled receipt must first reach the rename barrier");
+        wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
+        put.abort();
+        assert!(put.await.expect_err("ACK waiter should cancel").is_cancelled());
+        let mut lock_probe = Box::pin(set.acquire_write_lock_diag("cancelled_full_tail_probe", bucket, object));
+        assert!(
+            futures::poll!(lock_probe.as_mut()).is_pending(),
+            "owned coordinator must retain the namespace guard after waiter cancellation"
+        );
+        barrier.release();
+        drop(
+            tokio::time::timeout(Duration::from_secs(30), lock_probe)
+                .await
+                .expect("cancelled coordinator must eventually release its guard")
+                .expect("post-commit lock probe should succeed"),
+        );
+        assert_eq!(tasks.running(), 0, "cancelled coordinator must reap every rename task");
+        for disk in &disks {
+            disk.read_version("", bucket, object, "", &ReadOptions::default())
+                .await
+                .expect("caller cancellation must not interrupt committed receipt materialization");
+        }
+        wait_for_tmp_workspace_to_drain(&dirs, "cancelled full-tail commit should release staging ownership").await;
+    }
+
     #[tokio::test]
     #[serial_test::serial(capacity_dirty_scope)]
     async fn no_lock_put_waits_for_rename_tail_under_outer_guard() {
@@ -18187,6 +18588,7 @@ mod put_object_tmp_cleanup_tests {
                         &mut reader,
                         &ObjectOptions {
                             no_lock: true,
+                            write_completion: WriteCompletion::TailDrained,
                             ..Default::default()
                         },
                     )
@@ -18212,7 +18614,18 @@ mod put_object_tmp_cleanup_tests {
             put.await
                 .expect("no-lock PUT task should join")
                 .expect("no-lock PUT should commit after the rename tail releases");
+            let mut lock_probe = Box::pin(set_disks.acquire_write_lock_diag("borrowed_full_tail_probe", bucket, object));
+            assert!(
+                futures::poll!(lock_probe.as_mut()).is_pending(),
+                "full-tail PUT must not release the caller's outer guard"
+            );
             drop(outer_guard);
+            drop(
+                tokio::time::timeout(Duration::from_secs(5), lock_probe)
+                    .await
+                    .expect("outer owner releasing its guard should unblock the probe")
+                    .expect("post-outer-guard probe should succeed"),
+            );
         })
         .await;
     }

crates/ecstore/src/set_disk/transition_matrix_tests.rs

@@ -18,6 +18,7 @@ use super::{
 };
 use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time};
 use crate::ecstore_validation_blackbox::make_local_set_disks;
+use crate::object_api::WriteCompletion;
 use crate::services::tier::test_util::register_mock_tier;
 use crate::storage_api_contracts::bucket::BucketOperations;
 use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
@@ -72,7 +73,7 @@ async fn transition_and_restore_reclaim_prior_metadata_generations() {
             object,
             &mut reader,
             &ObjectOptions {
-                no_lock: true,
+                write_completion: WriteCompletion::TailDrained,
                 ..Default::default()
             },
         )
@@ -185,7 +186,7 @@ async fn prepared_snapshot_transition_duplicate_and_late_get_use_committed_remot
             object,
             &mut reader,
             &ObjectOptions {
-                no_lock: true,
+                write_completion: WriteCompletion::TailDrained,
                 ..Default::default()
             },
         )

crates/ecstore/src/store/init.rs

@@ -3328,41 +3328,33 @@ mod tests {
     #[cfg(feature = "test-util")]
     const DECOMMISSION_TEST_FAULT_STAGE_TIERED: &str = "decommission_tiered_object";
 
-    fn inject_decommission_copy_fault(faults: &AtomicUsize, attempt: usize, succeeded: bool) -> bool {
+    fn decommission_retry_fault_hook(
-        // Entry retries reset attempt, not the global post-commit fault budget.
+        bucket: &str,
-        // A real failure may consume an attempt, so preserve the final chance.
+        object: &str,
-        succeeded
+        faults: Arc<AtomicUsize>,
-            && faults
+    ) -> crate::core::pools::DecommissionTestFaultDecision {
+        let target_bucket = bucket.to_string();
+        let target_object = object.to_string();
+        Arc::new(move |stage, bucket, object, attempt, succeeded| {
+            if !succeeded
+                || attempt >= crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS
+                || stage != DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
+                || bucket != target_bucket
+                || object != target_object
+            {
+                return false;
+            }
+
+            // Entry retries reset the local attempt; real copy errors can skip
+            // successful attempts. Only injected faults spend this global budget.
+            // A real failure may consume an attempt, so preserve the final chance.
+            faults
                 .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| {
-                    (faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1)
+                    (faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1))
-                        && attempt < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS)
                         .then_some(faults.saturating_add(1))
                 })
                 .is_ok()
-    }
+        })
-
-    #[test]
-    fn decommission_copy_fault_budget_survives_entry_restarts_and_preserves_last_attempt() {
-        let cases: &[&[(usize, bool, bool)]] = &[
-            &[(1, true, true), (2, true, true), (3, true, false)],
-            &[(1, true, true), (2, false, false), (1, true, true), (2, true, false)],
-            &[(1, true, true), (2, false, false), (3, true, false)],
-            &[(1, true, true), (1, true, true), (1, true, false)],
-            &[(3, true, false), (4, true, false)],
-        ];
-        for case in cases {
-            let faults = AtomicUsize::new(0);
-            let mut expected_faults = 0;
-            for &(attempt, succeeded, expected) in *case {
-                assert_eq!(
-                    inject_decommission_copy_fault(&faults, attempt, succeeded),
-                    expected,
-                    "fault plan {case:?} at attempt {attempt}"
-                );
-                expected_faults += usize::from(expected);
-                assert_eq!(faults.load(Ordering::SeqCst), expected_faults);
-            }
-        }
     }
 
     async fn seed_decommission_source(
@@ -5507,6 +5499,43 @@ mod tests {
         shutdown.cancel();
     }
 
+    #[test]
+    fn decommission_retry_fault_budget_counts_successes_across_attempt_changes() {
+        let cases: &[&[(usize, bool, bool)]] = &[
+            &[(1, true, true), (2, true, true), (3, true, false)],
+            &[(1, true, true), (1, true, true), (2, true, false)],
+            &[(1, true, true), (3, true, false), (3, true, false)],
+            &[(1, true, true), (2, false, false), (1, true, true), (2, true, false)],
+            &[(1, true, true), (2, false, false), (3, true, false)],
+            &[(3, true, false), (4, true, false)],
+        ];
+        for case in cases {
+            let faults = Arc::new(AtomicUsize::new(0));
+            let hook = decommission_retry_fault_hook("bucket", "object", Arc::clone(&faults));
+
+            for (stage, bucket, object, succeeded) in [
+                ("other-stage", "bucket", "object", true),
+                (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "other-bucket", "object", true),
+                (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "other-object", true),
+                (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", false),
+            ] {
+                assert!(!hook(stage, bucket, object, 1, succeeded));
+            }
+            assert_eq!(faults.load(Ordering::SeqCst), 0, "unrelated or failed copies must not consume faults");
+
+            let mut expected_faults = 0;
+            for &(attempt, succeeded, expected) in *case {
+                assert_eq!(
+                    hook(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", attempt, succeeded),
+                    expected,
+                    "fault plan {case:?} at attempt {attempt}"
+                );
+                expected_faults += usize::from(expected);
+                assert_eq!(faults.load(Ordering::SeqCst), expected_faults);
+            }
+        }
+    }
+
     #[test]
     #[serial_test::serial(storage_class_env)]
     fn decommission_entry_retries_source_changed_without_canceling_other_bucket() {
@@ -5601,16 +5630,8 @@ mod tests {
                     ));
 
                     let ordinary_faults = Arc::new(AtomicUsize::new(0));
-                    let ordinary_faults_for_hook = Arc::clone(&ordinary_faults);
+                    let fault_hook = decommission_retry_fault_hook(&other_bucket, other_object, Arc::clone(&ordinary_faults));
-                    let fault_bucket = other_bucket.clone();
+                    let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(fault_hook);
-                    let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new(
-                        move |stage, bucket, object, attempt, succeeded| {
-                            let candidate = stage == DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
-                                && bucket == fault_bucket.as_str()
-                                && object == other_object;
-                            candidate && inject_decommission_copy_fault(&ordinary_faults_for_hook, attempt, succeeded)
-                        },
-                    ));
 
                     let rx = CancellationToken::new();
                     let source_changed_exhaustions = Arc::new(AtomicUsize::new(0));
@@ -8428,10 +8449,15 @@ mod tests {
         );
         assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok());
 
-        com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone())
+        let full_tail = ObjectOptions {
+            max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
+            ..Default::default()
+        };
+        com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone(), &full_tail)
             .await
             .expect("second page receipt should restore");
-        com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec())
+        com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec(), &full_tail)
             .await
             .expect("second page receipt should corrupt deterministically");
         let corrupt = store
@@ -10724,8 +10750,17 @@ mod tests {
         const JOURNAL_COUNT: usize = 40;
 
         let temp_dir = tempfile::tempdir().expect("create rollback retry store dir");
-        let (ctx, store, _shutdown) =
+        // Manual retries must own progress between fault removal and the next attempt.
-            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-rollback-retry", &[4])).await;
+        let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
+        instance_ctx.suppress_tier_delete_journal_recovery_for_test();
+        let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
+            temp_dir.path(),
+            "dispatch-rollback-retry",
+            &[(1, 4)],
+            CancellationToken::new(),
+            Some(Arc::new(instance_ctx)),
+        ))
+        .await;
         crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
         let bucket = "dispatch-rollback-retry-bucket";
         store
@@ -10799,6 +10834,7 @@ mod tests {
 
         assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
         assert_eq!(backend.remove_count().await, 0, "rollback retries must never call the remote tier");
+        shutdown.cancel();
     }
 
     #[cfg(feature = "test-util")]

crates/scanner/src/remote_scanner.rs

@@ -52,6 +52,9 @@ static REMOTE_SCANNER_CYCLE_REFRESH: LazyLock<AsyncMutex<()>> = LazyLock::new(||
 
 mod stream;
 
+#[cfg(test)]
+pub(crate) use stream::checkpoint_fixture_partial_return;
+
 pub use stream::{RemoteScannerAdmission, RemoteScannerRequest, serve_remote_scanner_request};
 pub(crate) use stream::{RemoteScannerOutcome, RemoteScannerScanSpec, scan_remote_bucket};
 use stream::{RemoteScannerReplayCache, RemoteScannerRequestWire, RemoteScannerValidatedCycle};

crates/scanner/src/remote_scanner/stream.rs

@@ -1017,6 +1017,48 @@ fn finish_remote_scanner_stream(
 #[cfg(test)]
 const TEST_NEXT_CYCLE: u64 = 11;
 
+#[cfg(test)]
+pub(crate) async fn checkpoint_fixture_partial_return(progress: (u64, u64), entries_visited: u64) {
+    let request_id = Uuid::new_v4();
+    let writer_auth = FrameAuthenticator::for_test(request_id);
+    let reader_auth = FrameAuthenticator::for_test(request_id);
+    let mut bytes = Vec::new();
+    write_frame(
+        &mut bytes,
+        &writer_auth,
+        &mut 0,
+        &RemoteScannerFrame::terminal(
+            RemoteScannerProgress {
+                objects_scanned: progress.0,
+                directories_started: progress.1,
+                entries_visited,
+            },
+            RemoteScannerFrameResult::Partial,
+        ),
+    )
+    .await
+    .expect("checkpoint partial frame must encode");
+    let frame = read_frame(&mut std::io::Cursor::new(bytes.as_slice()), &reader_auth, &mut 0)
+        .await
+        .expect("checkpoint progress frame must authenticate");
+    assert_eq!(frame.progress.entries_visited, entries_visited);
+    let parent = CancellationToken::new();
+    let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default());
+    let result = consume_remote_scanner_stream(
+        std::io::Cursor::new(bytes),
+        parent,
+        budget.clone(),
+        "bucket",
+        DataUsageCacheSource::new(0, 0),
+        DataUsageScanPlanDigest([17; 32]),
+        reader_auth,
+    )
+    .await
+    .expect("checkpoint partial frame must decode");
+    assert!(matches!(result, RemoteScannerOutcome::Partial));
+    assert_eq!(budget.progress(), progress);
+}
+
 #[cfg(test)]
 async fn consume_remote_scanner_stream<R>(
     reader: R,

crates/scanner/src/scanner_folder/tests.rs

@@ -24,6 +24,8 @@ use std::io::Write;
 use std::os::unix::fs::{PermissionsExt, symlink};
 use std::sync::Mutex;
 
+mod checkpoint_fixture;
+
 /// Reset the process-global alert cooldown map; test-only.
 fn reset_alert_cooldowns() {
     *SCANNER_ALERT_EMISSION_COOLDOWN

crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs

@@ -0,0 +1,410 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::*;
+use crate::scanner_budget::ScannerCycleBudgetConfig;
+use crate::scanner_io::{ScannerDiskScanOutcome, ScannerIODisk};
+use crate::storage_api::scanner_io::ObjectIO;
+use crate::{DataUsageCacheSource, DataUsageScanPlanDigest};
+use std::io::Cursor;
+use tokio::io::AsyncReadExt;
+
+const CACHE_NAME: &str = "bucket/checkpoint-fixture.bin";
+const STATIC_OBJECTS: u64 = 24;
+const MAX_CACHE_BYTES: u64 = 1024 * 1024;
+const SOURCE: DataUsageCacheSource = DataUsageCacheSource::new(0, 0);
+const PLAN: DataUsageScanPlanDigest = DataUsageScanPlanDigest([17; 32]);
+
+/// Real cache persistence codec and CAS calls, backed by two bounded local files.
+#[derive(Debug)]
+struct FixtureStore {
+    root: tempfile::TempDir,
+    reject_save: AtomicBool,
+}
+
+impl FixtureStore {
+    fn new() -> Arc<Self> {
+        Arc::new(Self {
+            root: tempfile::tempdir().expect("checkpoint fixture storage directory"),
+            reject_save: AtomicBool::new(false),
+        })
+    }
+
+    fn path(&self, object: &str) -> std::path::PathBuf {
+        assert!(object.ends_with(CACHE_NAME) || object.ends_with(&format!("{CACHE_NAME}.bkp")));
+        self.root
+            .path()
+            .join(if object.ends_with(".bkp") { "backup" } else { "main" })
+    }
+
+    async fn strict_load(&self) -> DataUsageCache {
+        let bytes = tokio::fs::read(self.root.path().join("main"))
+            .await
+            .expect("saved checkpoint fixture must exist");
+        decode_fixture(&bytes).expect("saved checkpoint fixture must contain a valid bucket root")
+    }
+}
+
+#[async_trait::async_trait]
+impl ObjectIO for FixtureStore {
+    type Error = crate::EcstoreError;
+    type RangeSpec = crate::storage_api::scanner_io::HTTPRangeSpec;
+    type HeaderMap = http::HeaderMap;
+    type ObjectOptions = crate::ScannerObjectOptions;
+    type ObjectInfo = crate::ScannerObjectInfo;
+    type GetObjectReader = crate::ScannerGetObjectReader;
+    type PutObjectReader = crate::ScannerPutObjReader;
+
+    async fn get_object_reader(
+        &self,
+        _bucket: &str,
+        object: &str,
+        _range: Option<Self::RangeSpec>,
+        _headers: Self::HeaderMap,
+        _options: &Self::ObjectOptions,
+    ) -> crate::EcstoreResult<Self::GetObjectReader> {
+        let bytes = tokio::fs::read(self.path(object)).await.map_err(|error| {
+            if error.kind() == std::io::ErrorKind::NotFound {
+                crate::EcstoreError::FileNotFound
+            } else {
+                crate::EcstoreError::from(error)
+            }
+        })?;
+        assert!(u64::try_from(bytes.len()).expect("cache length") <= MAX_CACHE_BYTES);
+        Ok(crate::ScannerGetObjectReader {
+            stream: Box::new(Cursor::new(bytes)),
+            object_info: crate::ScannerObjectInfo {
+                etag: Some("fixture".into()),
+                ..Default::default()
+            },
+            buffered_body: None,
+            body_source: Default::default(),
+        })
+    }
+
+    async fn put_object(
+        &self,
+        _bucket: &str,
+        object: &str,
+        data: &mut Self::PutObjectReader,
+        options: &Self::ObjectOptions,
+    ) -> crate::EcstoreResult<Self::ObjectInfo> {
+        if self.reject_save.load(Ordering::SeqCst) {
+            return Err(crate::EcstoreError::PreconditionFailed);
+        }
+        let path = self.path(object);
+        let exists = tokio::fs::try_exists(&path).await?;
+        let preconditions = options.http_preconditions.as_ref().expect("checkpoint writes must use CAS");
+        if (exists && preconditions.if_none_match_value() == Some("*"))
+            || (!exists && preconditions.if_match_value().is_some())
+            || (exists && preconditions.if_match_value() != Some("fixture"))
+        {
+            return Err(crate::EcstoreError::PreconditionFailed);
+        }
+        let mut bytes = Vec::new();
+        (&mut data.stream).take(MAX_CACHE_BYTES + 1).read_to_end(&mut bytes).await?;
+        assert!(u64::try_from(bytes.len()).expect("cache length") <= MAX_CACHE_BYTES);
+        tokio::fs::write(path, bytes).await?;
+        Ok(crate::ScannerObjectInfo {
+            etag: Some("fixture".into()),
+            ..Default::default()
+        })
+    }
+}
+
+#[async_trait::async_trait]
+impl crate::ScannerConfigObjectDelete for FixtureStore {
+    async fn delete_config_object(
+        &self,
+        _bucket: &str,
+        _object: &str,
+        _options: crate::ScannerObjectOptions,
+    ) -> crate::EcstoreResult<crate::ScannerObjectInfo> {
+        Err(crate::EcstoreError::NotImplemented)
+    }
+
+    async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
+        Some(crate::ScannerDataUsagePublicationAdmission::unfenced())
+    }
+}
+
+fn decode_fixture(bytes: &[u8]) -> Result<DataUsageCache, &'static str> {
+    if bytes.is_empty() || bytes.len() > usize::try_from(MAX_CACHE_BYTES).expect("fixture bound") {
+        return Err("missing or oversized checkpoint fixture");
+    }
+    let cache = DataUsageCache::unmarshal(bytes).map_err(|_| "corrupt checkpoint fixture")?;
+    if cache.info.name != "bucket" || cache.checked_flatten("bucket").is_none() {
+        return Err("checkpoint fixture has no valid bucket root");
+    }
+    Ok(cache)
+}
+
+fn retained(cache: &DataUsageCache) -> u64 {
+    assert!(
+        !cache.root().is_some_and(|root| root.compacted),
+        "a compacted bucket root cannot prove static-prefix coverage"
+    );
+    cache
+        .checked_flatten("bucket/static")
+        .map_or(0, |entry| u64::try_from(entry.objects).expect("fixture object count fits u64"))
+}
+
+#[derive(Debug, PartialEq, Eq)]
+enum CoverageDiagnosis {
+    Progress,
+    NoNewWork,
+    LostAtPrepare,
+    LostAtReload,
+    WalkWithoutRetention,
+}
+
+fn diagnose(previous: u64, prepared: u64, walked: u64, scanned: u64, reloaded: u64) -> CoverageDiagnosis {
+    if reloaded < scanned {
+        CoverageDiagnosis::LostAtReload
+    } else if prepared < previous {
+        CoverageDiagnosis::LostAtPrepare
+    } else if walked > 0 && reloaded <= previous {
+        CoverageDiagnosis::WalkWithoutRetention
+    } else if reloaded > previous {
+        CoverageDiagnosis::Progress
+    } else {
+        CoverageDiagnosis::NoNewWork
+    }
+}
+
+#[test]
+fn checkpoint_fixture_diagnosis_rejects_walk_without_retention() {
+    assert_eq!(diagnose(4, 4, 9, 8, 8), CoverageDiagnosis::Progress);
+    assert_eq!(diagnose(4, 4, 9, 4, 4), CoverageDiagnosis::WalkWithoutRetention);
+    assert_eq!(diagnose(4, 0, 9, 4, 4), CoverageDiagnosis::LostAtPrepare);
+    assert_eq!(diagnose(4, 4, 9, 8, 4), CoverageDiagnosis::LostAtReload);
+    assert_eq!(diagnose(4, 4, 0, 4, 4), CoverageDiagnosis::NoNewWork);
+}
+
+#[test]
+fn checkpoint_fixture_missing_and_corrupt_inputs_fail() {
+    for bytes in [
+        vec![],
+        vec![0xc1],
+        DataUsageCache::default().marshal_msg().expect("empty cache encoding"),
+        vec![0; usize::try_from(MAX_CACHE_BYTES + 1).expect("oversized fixture")],
+    ] {
+        assert!(decode_fixture(&bytes).is_err(), "invalid fixture must not become an empty complete root");
+    }
+}
+
+#[test]
+fn checkpoint_fixture_compaction_preserves_aggregate_not_child_enumeration() {
+    let mut cache = DataUsageCache::default();
+    cache.info.name = "bucket".to_string();
+    cache.replace("bucket", "", DataUsageEntry::default());
+    cache.replace("bucket/static", "bucket", DataUsageEntry::default());
+    for index in 0..4 {
+        cache.replace(
+            &format!("bucket/static/{index}"),
+            "bucket/static",
+            DataUsageEntry {
+                objects: 1,
+                ..Default::default()
+            },
+        );
+    }
+    cache.reduce_children_of(&hash_path("bucket/static"), 1, true);
+    let decoded = decode_fixture(&cache.marshal_msg().expect("encode compacted cache")).expect("decode compacted fixture");
+    let entry = decoded
+        .find("bucket/static")
+        .expect("compaction must retain the static subtree root");
+    assert!(entry.compacted);
+    assert!(entry.children.is_empty());
+    assert_eq!(
+        retained(&decoded),
+        4,
+        "compaction retains aggregate coverage even when leaf keys are absent"
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn checkpoint_fixture_save_reload_resume() {
+    run_checkpoint_fixture(false).await;
+}
+
+#[tokio::test]
+#[serial]
+async fn checkpoint_fixture_hot_digest_diagnostic() {
+    run_checkpoint_fixture(true).await;
+}
+
+async fn run_checkpoint_fixture(change_digest: bool) {
+    let (scanner, root) = build_test_scanner().await;
+    let _guard = TestGuard {
+        temp_dir: Some(root.clone()),
+    };
+    for index in 0..STATIC_OBJECTS {
+        write_test_object_metadata(&root, "bucket", &format!("static/{index:04}")).await;
+    }
+    let store = FixtureStore::new();
+    let mut previous = 0;
+    let mut visited = 0;
+    for round in 0..3_u8 {
+        write_test_object_metadata(&root, "bucket", "hot/current").await;
+        let mut cache = DataUsageCache::default();
+        let revisions = cache
+            .load_with_revisions(store.clone(), CACHE_NAME)
+            .await
+            .expect("load checkpoint revisions");
+        if round > 0 {
+            assert_eq!(retained(&store.strict_load().await), previous);
+        }
+        let plan = crate::scanner_io::checkpoint_fixture_bucket_digest(PLAN, change_digest.then_some(u64::from(round)));
+        crate::scanner_io::current_cache_root_or_prepare_with_generation(
+            &mut cache,
+            "bucket",
+            SOURCE,
+            11,
+            7,
+            plan,
+            crate::scanner_io::DataUsageCacheReuseOptions {
+                require_source: true,
+                tier_registry_generation: None,
+            },
+        );
+        let prepared = retained(&cache);
+        let parent = CancellationToken::new();
+        let budget = ScannerCycleBudget::new_with_progress_tracking(
+            &parent,
+            ScannerCycleBudgetConfig {
+                max_objects: Some(4),
+                ..Default::default()
+            },
+        );
+        let outcome = scanner
+            .local_disk
+            .clone()
+            .nsscanner_disk(
+                budget.token(),
+                budget.clone(),
+                vec![scanner.local_disk.clone()],
+                cache,
+                None,
+                HealScanMode::Normal,
+            )
+            .await
+            .expect("budgeted local disk scan returns partial cache");
+        let ScannerDiskScanOutcome::Partial(cache) = outcome else {
+            panic!("budgeted fixture must remain partial")
+        };
+        assert!(!cache.info.snapshot_complete, "partial must never publish a complete root");
+        assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects));
+        let scanned = retained(&cache);
+        cache
+            .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
+            .await
+            .expect("persist partial checkpoint");
+        let mut loaded = DataUsageCache::default();
+        loaded
+            .load(store.clone(), CACHE_NAME)
+            .await
+            .expect("reload persisted partial checkpoint");
+        let reloaded = retained(&loaded);
+        assert_eq!(reloaded, retained(&store.strict_load().await));
+        assert_eq!(scanned, reloaded, "save/load must retain static subtree coverage");
+        assert!(!loaded.info.snapshot_complete);
+        visited += budget.entries_visited();
+        let diagnosis = diagnose(previous, prepared, budget.entries_visited(), scanned, reloaded);
+        eprintln!(
+            "checkpoint_fixture round={round} hot_digest={change_digest} visited_total={visited} before={previous} prepared={prepared} scanned={scanned} reloaded={reloaded} diagnosis={diagnosis:?}"
+        );
+        if !change_digest || std::env::var_os("RUSTFS_CHECKPOINT_REQUIRE_PROGRESS").is_some() {
+            assert_eq!(
+                diagnosis,
+                CoverageDiagnosis::Progress,
+                "visited growth must produce durable static coverage"
+            );
+        }
+        crate::remote_scanner::checkpoint_fixture_partial_return(budget.progress(), budget.entries_visited()).await;
+        previous = reloaded;
+    }
+    assert!(visited > 0, "fixture must exercise the directory walk");
+    assert!(previous > 0, "fixture must retain and enumerate static subtree entries");
+
+    let mut loaded = DataUsageCache::default();
+    let revisions = loaded
+        .load_with_revisions(store.clone(), CACHE_NAME)
+        .await
+        .expect("load final checkpoint");
+    let before = tokio::fs::read(store.root.path().join("main"))
+        .await
+        .expect("read durable checkpoint bytes");
+    let epoch_error = loaded
+        .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 1)
+        .await
+        .expect_err("stale publication epoch must reject persistence");
+    assert!(epoch_error.to_string().contains(crate::SCANNER_PUBLICATION_EPOCH_CHANGED));
+    store.reject_save.store(true, Ordering::SeqCst);
+    loaded.info.next_cycle += 1;
+    loaded
+        .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
+        .await
+        .expect_err("injected save failure must not report durable progress");
+    assert_eq!(
+        tokio::fs::read(store.root.path().join("main"))
+            .await
+            .expect("read unchanged checkpoint bytes"),
+        before
+    );
+
+    let parent = CancellationToken::new();
+    parent.cancel();
+    let budget = ScannerCycleBudget::new(&parent, Default::default());
+    let result = scanner
+        .local_disk
+        .clone()
+        .nsscanner_disk(
+            budget.token(),
+            budget.clone(),
+            vec![scanner.local_disk.clone()],
+            loaded.clone(),
+            None,
+            HealScanMode::Normal,
+        )
+        .await;
+    assert!(result.is_err(), "pre-scan cancellation must not produce a complete root");
+    assert_eq!(budget.reason(), None, "parent cancellation is not object budget exhaustion");
+
+    let parent = CancellationToken::new();
+    let budget = ScannerCycleBudget::new(&parent, Default::default());
+    let result = scanner
+        .local_disk
+        .clone()
+        .nsscanner_disk(
+            budget.token(),
+            budget,
+            vec![scanner.local_disk.clone()],
+            loaded,
+            None,
+            HealScanMode::Normal,
+        )
+        .await
+        .expect("unbounded scan must complete after durable partial progress");
+    let ScannerDiskScanOutcome::Complete(cache) = result else {
+        panic!("unbounded fixture must produce a complete disk cache");
+    };
+    assert!(cache.info.snapshot_complete);
+    assert!(cache.info.scan_checkpoint.is_none());
+    assert_eq!(
+        cache.checked_flatten("bucket").expect("complete bucket root").objects,
+        usize::try_from(STATIC_OBJECTS + 1).expect("fixture object count fits usize")
+    );
+}

crates/scanner/src/scanner_io.rs

@@ -209,6 +209,14 @@ fn scanner_bucket_cache_digest(
     DataUsageScanPlanDigest(hasher.finalize().into())
 }
 
+#[cfg(test)]
+pub(crate) fn checkpoint_fixture_bucket_digest(
+    scan_plan_digest: DataUsageScanPlanDigest,
+    dirty_generation: Option<u64>,
+) -> DataUsageScanPlanDigest {
+    scanner_bucket_cache_digest(scan_plan_digest, dirty_generation)
+}
+
 fn finalize_nsscanner_result(results: &[DataUsageCache], first_err: Option<Error>) -> Result<()> {
     if results.iter().any(|result| result.info.last_update.is_some()) {
         return Ok(());

crates/scanner/src/scanner_io/tests.rs

@@ -1148,6 +1148,27 @@ fn scanner_cycle_status_requires_a_clean_complete_snapshot() {
     }
 }
 
+#[test]
+fn checkpoint_fixture_superseded_is_distinct_from_partial_and_cancel() {
+    for (budget, cancelled, bucket, expected) in [
+        (false, false, ScannerBucketScanStatus::Complete, ScannerCycleStatus::Superseded),
+        (true, false, ScannerBucketScanStatus::Partial, ScannerCycleStatus::Incomplete),
+        (false, true, ScannerBucketScanStatus::Partial, ScannerCycleStatus::Incomplete),
+    ] {
+        assert_eq!(
+            classify_nsscanner_cycle(
+                true,
+                budget,
+                cancelled,
+                bucket,
+                DirtyUsageSnapshotStatus::Changed,
+                ScannerCycleActivityStatus::Unchanged
+            ),
+            expected,
+        );
+    }
+}
+
 #[test]
 fn unverified_activity_defers_partial_and_floor_cycles() {
     let expected = ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);

crates/zip/Cargo.toml

@@ -49,5 +49,14 @@ rustfs-rio.workspace = true
 tokio = { workspace = true, features = ["io-util", "macros", "rt"] }
 thiserror = { workspace = true }
 
+[dev-dependencies]
+astral-tokio-tar = { workspace = true }
+futures = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
+sha2 = { workspace = true }
+tar-codec = { workspace = true }
+tar-framing = { workspace = true }
+
 [lints]
 workspace = true

crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/README.md

@@ -0,0 +1,24 @@
+# minio-go Snowball fixtures
+
+These request bodies are generated by
+`github.com/minio/minio-go/v7.Client.PutObjectsSnowball` at the version pinned
+in `generate/go.mod`. They cover the raw TAR and S2-compressed forms accepted by
+RustFS Snowball extraction.
+
+The decoded TAR intentionally ends immediately after the final padded member
+body because minio-go flushes, rather than closes, its TAR writer. The
+compatibility test permits that shape only when the authenticated request body
+is complete at the exact member boundary; it does not make incomplete TAR
+terminators generally valid.
+
+Regenerate them from this directory with Go 1.25:
+
+```console
+cd generate
+go mod download
+go run . -out ..
+```
+
+`manifest.json` records the input objects and SHA-256 digest of each captured
+request body. Review changes to the manifest and binary fixtures together when
+updating minio-go.

crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.mod

@@ -0,0 +1,26 @@
+module rustfs.local/snowball-fixture
+
+go 1.25.0
+
+require github.com/minio/minio-go/v7 v7.3.0
+
+require (
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/klauspost/compress v1.19.2 // indirect
+	github.com/klauspost/cpuid/v2 v2.4.0 // indirect
+	github.com/klauspost/crc32 v1.3.0 // indirect
+	github.com/minio/crc64nvme v1.1.1 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/philhofer/fwd v1.2.0 // indirect
+	github.com/rs/xid v1.6.0 // indirect
+	github.com/tinylib/msgp v1.6.4 // indirect
+	github.com/zeebo/xxh3 v1.1.0 // indirect
+	go.yaml.in/yaml/v3 v3.0.5 // indirect
+	golang.org/x/crypto v0.55.0 // indirect
+	golang.org/x/net v0.58.0 // indirect
+	golang.org/x/sys v0.47.0 // indirect
+	golang.org/x/text v0.41.0 // indirect
+	gopkg.in/ini.v1 v1.67.3 // indirect
+)

crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.sum

@@ -0,0 +1,59 @@
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
+github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
+github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
+github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
+github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
+github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
+github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.3.0 h1:HM4pFCSQq/TK+j0/zmorSh5ddh81iDgRgU0BG0Vz/YU=
+github.com/minio/minio-go/v7 v7.3.0/go.mod h1:KUPWdecEO1LWyUz+sTGXAuf2jZHrPh5fCsRH86QbPfk=
+github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
+github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
+github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
+github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
+github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
+github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
+github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
+go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
+go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
+golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
+golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
+golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
+golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
+golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw=
+gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/main.go

@@ -0,0 +1,193 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+const minioGoVersion = "v7.3.0"
+
+type fixtureManifest struct {
+	Generator   string           `json:"generator"`
+	MinioGo     string           `json:"minio_go"`
+	GeneratedAt string           `json:"generated_at"`
+	Objects     []fixtureObject  `json:"objects"`
+	Archives    []fixtureArchive `json:"archives"`
+}
+
+type fixtureObject struct {
+	Key       string              `json:"key"`
+	Body      string              `json:"body"`
+	ModTime   string              `json:"mod_time"`
+	VersionID string              `json:"version_id,omitempty"`
+	Headers   map[string][]string `json:"headers,omitempty"`
+}
+
+type fixtureArchive struct {
+	File       string `json:"file"`
+	Compressed bool   `json:"compressed"`
+	Length     int    `json:"length"`
+	SHA256     string `json:"sha256"`
+}
+
+func objects() []fixtureObject {
+	return []fixtureObject{
+		{
+			Key:       "alpha.txt",
+			Body:      "alpha-body",
+			ModTime:   "2024-01-02T03:04:05Z",
+			VersionID: "018cc251-f400-7c22-9e8d-8b1800000001",
+			Headers: map[string][]string{
+				"Content-Type":     {"text/plain"},
+				"X-Amz-Meta-Owner": {"snowball-fixture"},
+				"X-Amz-Tagging":    {"project=rustfs&source=minio-go"},
+			},
+		},
+		{
+			Key:     "nested/世界.txt",
+			Body:    "bravo-body",
+			ModTime: "2024-01-02T03:05:05Z",
+			Headers: map[string][]string{
+				"Content-Language": {"zh-CN"},
+				"X-Amz-Meta-Note":  {"unicode-path"},
+			},
+		},
+	}
+}
+
+func captureSnowball(compressed bool, specs []fixtureObject) ([]byte, error) {
+	body := make(chan []byte, 1)
+	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		payload, err := io.ReadAll(request.Body)
+		if err != nil {
+			http.Error(writer, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		body <- payload
+		writer.Header().Set("ETag", `"snowball-fixture"`)
+		writer.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{
+		// The S3 authentication layer removes AWS streaming-signature framing
+		// before Snowball extraction sees the request body. Anonymous signing
+		// captures those decoded archive bytes directly.
+		Creds:  credentials.NewStatic("", "", "", credentials.SignatureAnonymous),
+		Secure: false,
+		Region: "us-east-1",
+	})
+	if err != nil {
+		return nil, fmt.Errorf("construct minio client: %w", err)
+	}
+
+	input := make(chan minio.SnowballObject, len(specs))
+	for _, spec := range specs {
+		modTime, err := time.Parse(time.RFC3339, spec.ModTime)
+		if err != nil {
+			return nil, fmt.Errorf("parse mod time for %q: %w", spec.Key, err)
+		}
+		headers := make(http.Header, len(spec.Headers))
+		for name, values := range spec.Headers {
+			headers[name] = append([]string(nil), values...)
+		}
+		input <- minio.SnowballObject{
+			Key:       spec.Key,
+			Size:      int64(len(spec.Body)),
+			ModTime:   modTime,
+			Content:   bytes.NewReader([]byte(spec.Body)),
+			VersionID: spec.VersionID,
+			Headers:   headers,
+		}
+	}
+	close(input)
+
+	err = client.PutObjectsSnowball(context.Background(), "fixture-bucket", minio.SnowballOptions{
+		Opts: minio.PutObjectOptions{
+			ContentType: "application/octet-stream",
+		},
+		InMemory: true,
+		Compress: compressed,
+	}, input)
+	if err != nil {
+		return nil, fmt.Errorf("generate snowball request: %w", err)
+	}
+	return <-body, nil
+}
+
+func main() {
+	outDir := flag.String("out", "..", "fixture output directory")
+	flag.Parse()
+
+	specs := objects()
+	archives := make([]fixtureArchive, 0, 2)
+	for _, fixture := range []struct {
+		name       string
+		compressed bool
+	}{
+		{name: "snowball.tar"},
+		{name: "snowball.tar.s2", compressed: true},
+	} {
+		payload, err := captureSnowball(fixture.compressed, specs)
+		if err != nil {
+			panic(err)
+		}
+		path := filepath.Join(*outDir, fixture.name)
+		if err := os.WriteFile(path, payload, 0o644); err != nil {
+			panic(fmt.Errorf("write %s: %w", path, err))
+		}
+		digest := sha256.Sum256(payload)
+		archives = append(archives, fixtureArchive{
+			File:       fixture.name,
+			Compressed: fixture.compressed,
+			Length:     len(payload),
+			SHA256:     hex.EncodeToString(digest[:]),
+		})
+	}
+
+	manifest := fixtureManifest{
+		Generator:   "github.com/minio/minio-go/v7.Client.PutObjectsSnowball",
+		MinioGo:     minioGoVersion,
+		GeneratedAt: "2026-09-05T00:00:00Z",
+		Objects:     specs,
+		Archives:    archives,
+	}
+	payload, err := json.MarshalIndent(manifest, "", "  ")
+	if err != nil {
+		panic(err)
+	}
+	payload = append(payload, '\n')
+	path := filepath.Join(*outDir, "manifest.json")
+	if err := os.WriteFile(path, payload, 0o644); err != nil {
+		panic(fmt.Errorf("write %s: %w", path, err))
+	}
+}

crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/manifest.json

@@ -0,0 +1,51 @@
+{
+  "generator": "github.com/minio/minio-go/v7.Client.PutObjectsSnowball",
+  "minio_go": "v7.3.0",
+  "generated_at": "2026-09-05T00:00:00Z",
+  "objects": [
+    {
+      "key": "alpha.txt",
+      "body": "alpha-body",
+      "mod_time": "2024-01-02T03:04:05Z",
+      "version_id": "018cc251-f400-7c22-9e8d-8b1800000001",
+      "headers": {
+        "Content-Type": [
+          "text/plain"
+        ],
+        "X-Amz-Meta-Owner": [
+          "snowball-fixture"
+        ],
+        "X-Amz-Tagging": [
+          "project=rustfs\u0026source=minio-go"
+        ]
+      }
+    },
+    {
+      "key": "nested/世界.txt",
+      "body": "bravo-body",
+      "mod_time": "2024-01-02T03:05:05Z",
+      "headers": {
+        "Content-Language": [
+          "zh-CN"
+        ],
+        "X-Amz-Meta-Note": [
+          "unicode-path"
+        ]
+      }
+    }
+  ],
+  "archives": [
+    {
+      "file": "snowball.tar",
+      "compressed": false,
+      "length": 4096,
+      "sha256": "f00f2789dcb65b567f722f49cfdac9705e7bdac6c0badae75194327c32193d2e"
+    },
+    {
+      "file": "snowball.tar.s2",
+      "compressed": true,
+      "length": 528,
+      "sha256": "f8a9d9aa9b9ccdfae24ded1bff3741aacb935f1457a252efc9266674ff13c992"
+    }
+  ]
+}

crates/zip/tests/snowball_tar_codec_compat.rs

@@ -0,0 +1,548 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::BTreeMap;
+use std::fmt::Write as _;
+use std::io::Cursor;
+
+use futures::StreamExt;
+use rustfs_zip::CompressionFormat;
+use serde::Deserialize;
+use sha2::{Digest, Sha256};
+use tar_codec::{Archive as _, DecodePolicy, Member, MemberPayload as _, PaxDecodePolicy, PaxVendorExtensionPolicy, TarArchive};
+use tar_framing::{
+    FrameError, FrameErrorInner, PaxKeyword, PaxRecord, PaxValue, StreamPolicy, UstarKind,
+    logical::{MemberExtensions, PaxState, TarReader},
+};
+use tokio::io::AsyncReadExt;
+
+const FIXTURE_ROOT: &str = "fixtures/snowball/minio-go-v7.3.0";
+const RAW_FIXTURE: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/snowball.tar");
+const S2_FIXTURE: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/snowball.tar.s2");
+const MANIFEST: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/manifest.json");
+
+#[derive(Debug, Deserialize)]
+struct FixtureManifest {
+    generator: String,
+    minio_go: String,
+    generated_at: String,
+    objects: Vec<FixtureObject>,
+    archives: Vec<FixtureArchive>,
+}
+
+#[derive(Debug, Deserialize)]
+struct FixtureObject {
+    key: String,
+    body: String,
+    mod_time: String,
+    #[serde(default)]
+    version_id: String,
+    #[serde(default)]
+    headers: BTreeMap<String, Vec<String>>,
+}
+
+#[derive(Debug, Deserialize)]
+struct FixtureArchive {
+    file: String,
+    compressed: bool,
+    length: usize,
+    sha256: String,
+}
+
+#[derive(Debug, Eq, PartialEq)]
+struct ParsedMember {
+    path: String,
+    size: u64,
+    mtime: Option<u64>,
+    body: Vec<u8>,
+    minio_pax: BTreeMap<String, Option<Vec<u8>>>,
+}
+
+fn sha256_hex(bytes: &[u8]) -> String {
+    let mut encoded = String::with_capacity(64);
+    for byte in Sha256::digest(bytes) {
+        write!(&mut encoded, "{byte:02x}").expect("writing to a String should not fail");
+    }
+    encoded
+}
+
+async fn decode_s2(bytes: &[u8]) -> Vec<u8> {
+    let mut decoder = CompressionFormat::S2
+        .get_decoder(Cursor::new(bytes.to_vec()))
+        .expect("S2 fixture decoder should be available");
+    let mut decoded = Vec::new();
+    decoder.read_to_end(&mut decoded).await.expect("S2 fixture should decode");
+    decoded
+}
+
+async fn parse_with_tokio_tar(bytes: &[u8]) -> Vec<ParsedMember> {
+    let mut archive = tokio_tar::Archive::new(Cursor::new(bytes.to_vec()));
+    let mut entries = archive.entries().expect("tokio-tar should create an entry stream");
+    let mut parsed = Vec::new();
+
+    while let Some(entry) = entries.next().await {
+        let mut entry = entry.expect("tokio-tar should parse the fixture member");
+        let kind = entry.header().entry_type();
+        if kind == tokio_tar::EntryType::XGlobalHeader {
+            continue;
+        }
+
+        let path_bytes = entry.path_bytes().expect("tokio-tar should resolve the fixture path");
+        let path = std::str::from_utf8(path_bytes.as_ref())
+            .expect("fixture paths should be UTF-8")
+            .to_owned();
+        let size = entry.effective_size();
+        let mtime = entry.header().mtime().ok();
+        let mut minio_pax = BTreeMap::new();
+        if let Some(extensions) = entry
+            .pax_extensions()
+            .await
+            .expect("tokio-tar should parse local PAX records")
+        {
+            for extension in extensions {
+                let extension = extension.expect("fixture PAX record should be valid");
+                let key = extension.key().expect("fixture PAX keys should be UTF-8");
+                if key.starts_with("minio.") {
+                    minio_pax.insert(key.to_owned(), Some(extension.value_bytes().to_vec()));
+                }
+            }
+        }
+        let mut body = Vec::new();
+        entry
+            .read_to_end(&mut body)
+            .await
+            .expect("tokio-tar should read the fixture body");
+        parsed.push(ParsedMember {
+            path,
+            size,
+            mtime,
+            body,
+            minio_pax,
+        });
+    }
+    parsed
+}
+
+fn effective_minio_pax(state: &PaxState<'_>, known_keywords: &mut Vec<PaxKeyword>) -> BTreeMap<String, Option<Vec<u8>>> {
+    for extension in state.extensions() {
+        for record in extension.records() {
+            let keyword = record.keyword();
+            if matches!(&keyword, PaxKeyword::Vendor { vendor, .. } if vendor.as_ref() == "minio")
+                && !known_keywords.contains(&keyword)
+            {
+                known_keywords.push(keyword);
+            }
+        }
+    }
+
+    known_keywords
+        .iter()
+        .filter_map(|keyword| {
+            let record = state.effective_record(keyword)?;
+            let PaxRecord::Vendor { vendor, name, value } = record else {
+                return None;
+            };
+            let key = format!("{vendor}.{name}");
+            let value = match value {
+                PaxValue::Value(value) => Some(value.to_vec()),
+                PaxValue::Deleted => None,
+            };
+            Some((key, value))
+        })
+        .collect()
+}
+
+fn effective_mtime(header_mtime: Option<u64>, extensions: &MemberExtensions<'_>) -> Option<u64> {
+    let MemberExtensions::Pax(state) = extensions else {
+        return header_mtime;
+    };
+    match state.effective_record(&PaxKeyword::Mtime) {
+        Some(PaxRecord::Mtime(PaxValue::Value(value))) => Some(*value),
+        Some(PaxRecord::Mtime(PaxValue::Deleted)) => None,
+        _ => header_mtime,
+    }
+}
+
+fn padded_member_end(position: u64, size: u64) -> u64 {
+    let padded_size = size.checked_add(511).expect("fixture member size should not overflow") / 512 * 512;
+    position
+        .checked_add(512)
+        .and_then(|position| position.checked_add(padded_size))
+        .expect("fixture member end should not overflow")
+}
+
+fn is_authenticated_footerless_end(error: &FrameError, last_member_end: Option<u64>, request_body_complete: bool) -> bool {
+    // The production gate must source `request_body_complete` from RustFS's
+    // length, checksum, and trailing-header validation state.
+    request_body_complete && matches!(&error.inner, FrameErrorInner::MissingEndMarker) && last_member_end == Some(error.position)
+}
+
+fn candidate_snowball_decode_policy() -> DecodePolicy {
+    DecodePolicy::default()
+        .allow_gnu(true)
+        .allow_all_nul_numeric_fields(true)
+        .max_gnu_extension_size(1_048_576)
+        .pax_policy(
+            PaxDecodePolicy::default()
+                .max_extension_size(1_048_576)
+                .max_global_extensions_size(67_108_864)
+                .allow_global_pax_extensions(false)
+                .allow_non_utf8_pax_vendor_values(false)
+                .allow_duplicate_pax_records(false)
+                .allow_global_pax_member_metadata(false)
+                .vendor_extension_policy(PaxVendorExtensionPolicy::ignore(["minio"])),
+        )
+}
+
+async fn parse_with_tar_framing(bytes: &[u8]) -> (Vec<ParsedMember>, Option<FrameError>, Option<u64>) {
+    let policy = StreamPolicy::default()
+        .max_pax_extension_size(1024 * 1024)
+        .max_global_pax_extensions_size(4 * 1024 * 1024)
+        .max_gnu_extension_size(128 * 1024);
+    let mut reader = TarReader::new(Cursor::new(bytes.to_vec())).with_policy(policy);
+    let mut parsed = Vec::new();
+    let mut known_minio_keywords = Vec::new();
+    let mut last_member_end = None;
+
+    loop {
+        let mut frame = match reader.next_frame().await {
+            Ok(Some(frame)) => frame,
+            Ok(None) => return (parsed, None, last_member_end),
+            Err(error) => return (parsed, Some(error), last_member_end),
+        };
+        assert_eq!(frame.header.kind, UstarKind::Regular);
+        let path = String::from_utf8(
+            frame
+                .effective_path()
+                .expect("tar-framing should resolve the fixture path")
+                .into_owned(),
+        )
+        .expect("fixture paths should be UTF-8");
+        let size = frame.header.effective_size;
+        let mtime = effective_mtime(frame.header.mtime, &frame.extensions);
+        let minio_pax = match &frame.extensions {
+            MemberExtensions::Pax(state) => effective_minio_pax(state, &mut known_minio_keywords),
+            MemberExtensions::Gnu { .. } => BTreeMap::new(),
+        };
+        let mut body = Vec::new();
+        let mut chunk = Vec::new();
+        while frame
+            .payload
+            .next_chunk(&mut chunk, 64 * 1024)
+            .await
+            .expect("tar-framing should read the fixture body")
+        {
+            body.extend_from_slice(&chunk);
+        }
+        last_member_end = Some(padded_member_end(frame.header.position, size));
+        parsed.push(ParsedMember {
+            path,
+            size,
+            mtime,
+            body,
+            minio_pax,
+        });
+    }
+}
+
+#[test]
+fn checked_in_fixtures_match_the_minio_go_manifest() {
+    let manifest: FixtureManifest = serde_json::from_slice(MANIFEST).expect("fixture manifest should be valid JSON");
+    assert_eq!(manifest.generator, "github.com/minio/minio-go/v7.Client.PutObjectsSnowball");
+    assert_eq!(manifest.minio_go, "v7.3.0");
+    assert_eq!(manifest.generated_at, "2026-09-05T00:00:00Z");
+    assert_eq!(manifest.objects.len(), 2);
+    assert_eq!(manifest.objects[0].key, "alpha.txt");
+    assert_eq!(manifest.objects[0].body, "alpha-body");
+    assert_eq!(manifest.objects[0].mod_time, "2024-01-02T03:04:05Z");
+    assert_eq!(manifest.objects[0].version_id, "018cc251-f400-7c22-9e8d-8b1800000001");
+    assert_eq!(
+        manifest.objects[0].headers.get("X-Amz-Meta-Owner"),
+        Some(&vec!["snowball-fixture".to_owned()])
+    );
+
+    for archive in &manifest.archives {
+        let bytes = match archive.file.as_str() {
+            "snowball.tar" => RAW_FIXTURE,
+            "snowball.tar.s2" => S2_FIXTURE,
+            file => panic!("unexpected archive in {FIXTURE_ROOT}/manifest.json: {file}"),
+        };
+        assert_eq!(bytes.len(), archive.length);
+        assert_eq!(sha256_hex(bytes), archive.sha256);
+        assert_eq!(archive.compressed, archive.file.ends_with(".s2"));
+    }
+}
+
+#[tokio::test]
+async fn minio_go_raw_and_s2_fixtures_have_identical_footerless_tar_data() {
+    assert_eq!(decode_s2(S2_FIXTURE).await, RAW_FIXTURE);
+    assert_eq!(RAW_FIXTURE.len() % 512, 0);
+    assert!(RAW_FIXTURE.len() >= 1024);
+    assert!(
+        !RAW_FIXTURE[RAW_FIXTURE.len() - 1024..].iter().all(|byte| *byte == 0),
+        "minio-go Flush output should not contain the standard two-block terminator"
+    );
+}
+
+#[tokio::test]
+async fn tar_framing_matches_tokio_tar_before_rejecting_the_missing_terminator() {
+    let expected = parse_with_tokio_tar(RAW_FIXTURE).await;
+    let (actual, error, last_member_end) = parse_with_tar_framing(RAW_FIXTURE).await;
+    let error = error.expect("footerless minio-go fixture should fail strict termination");
+
+    assert_eq!(actual, expected);
+    assert_eq!(
+        actual,
+        [
+            ParsedMember {
+                path: "alpha.txt".to_owned(),
+                size: 10,
+                mtime: Some(1_704_164_645),
+                body: b"alpha-body".to_vec(),
+                minio_pax: BTreeMap::from([
+                    ("minio.metadata.Content-Type".to_owned(), Some(b"text/plain".to_vec()),),
+                    ("minio.metadata.X-Amz-Meta-Owner".to_owned(), Some(b"snowball-fixture".to_vec()),),
+                    (
+                        "minio.metadata.X-Amz-Tagging".to_owned(),
+                        Some(b"project=rustfs&source=minio-go".to_vec()),
+                    ),
+                    ("minio.versionId".to_owned(), Some(b"018cc251-f400-7c22-9e8d-8b1800000001".to_vec()),),
+                ]),
+            },
+            ParsedMember {
+                path: "nested/世界.txt".to_owned(),
+                size: 10,
+                mtime: Some(1_704_164_705),
+                body: b"bravo-body".to_vec(),
+                minio_pax: BTreeMap::from([
+                    ("minio.metadata.Content-Language".to_owned(), Some(b"zh-CN".to_vec()),),
+                    ("minio.metadata.X-Amz-Meta-Note".to_owned(), Some(b"unicode-path".to_vec()),),
+                ]),
+            },
+        ]
+    );
+    assert!(matches!(&error.inner, FrameErrorInner::MissingEndMarker));
+    assert_eq!(
+        error.position,
+        u64::try_from(RAW_FIXTURE.len()).expect("fixture length should fit in u64")
+    );
+    assert_eq!(last_member_end, Some(error.position));
+}
+
+#[tokio::test]
+async fn footerless_compatibility_requires_authenticated_eof_at_the_member_boundary() {
+    let (_, error, last_member_end) = parse_with_tar_framing(RAW_FIXTURE).await;
+    let error = error.expect("the real fixture should be footerless");
+    assert!(is_authenticated_footerless_end(&error, last_member_end, true));
+    assert!(!is_authenticated_footerless_end(&error, last_member_end, false));
+
+    let mut one_zero_block = RAW_FIXTURE.to_vec();
+    one_zero_block.extend([0; 512]);
+    let (_, error, last_member_end) = parse_with_tar_framing(&one_zero_block).await;
+    let error = error.expect("one zero block is not a valid TAR terminator");
+    assert!(matches!(&error.inner, FrameErrorInner::MissingEndMarker));
+    assert_eq!(
+        last_member_end,
+        Some(u64::try_from(RAW_FIXTURE.len()).expect("fixture length should fit in u64"))
+    );
+    assert_eq!(
+        error.position,
+        u64::try_from(one_zero_block.len()).expect("fixture length should fit in u64")
+    );
+    assert!(!is_authenticated_footerless_end(&error, last_member_end, true));
+}
+
+#[tokio::test]
+async fn tar_codec_policy_accepts_only_the_explicit_minio_vendor_namespace() {
+    let default_error = match TarArchive::new(Cursor::new(RAW_FIXTURE.to_vec())).members().next().await {
+        Err(error) => error,
+        Ok(_) => panic!("the default policy should reject minio vendor records"),
+    };
+    assert!(default_error.to_string().contains("pax vendor extension minio."));
+
+    let mut members = TarArchive::new(Cursor::new(RAW_FIXTURE.to_vec()))
+        .with_policy(candidate_snowball_decode_policy())
+        .members();
+    let mut bodies = Vec::new();
+    loop {
+        let member = match members.next().await {
+            Ok(Some(member)) => member,
+            Ok(None) => panic!("footerless minio-go fixture should not report a valid archive end"),
+            Err(error) => {
+                assert!(error.to_string().contains("missing two-block end-of-archive marker"));
+                break;
+            }
+        };
+        let Member::File { mut payload, .. } = member else {
+            panic!("fixture should contain only regular files");
+        };
+        let mut body = Vec::new();
+        let mut chunk = Vec::new();
+        while payload
+            .next_chunk(&mut chunk, 64 * 1024)
+            .await
+            .expect("tar-codec should read the fixture body")
+        {
+            body.extend_from_slice(&chunk);
+        }
+        bodies.push(body);
+    }
+    assert_eq!(bodies, [b"alpha-body".to_vec(), b"bravo-body".to_vec()]);
+    assert!(
+        members
+            .next()
+            .await
+            .expect("the member cursor should be fused after an error")
+            .is_none()
+    );
+}
+
+fn pax_record(key: &str, value: &str) -> Vec<u8> {
+    let payload = format!("{key}={value}\n");
+    let mut len = payload.len() + 3;
+    loop {
+        let record = format!("{len} {payload}");
+        if record.len() == len {
+            return record.into_bytes();
+        }
+        len = record.len();
+    }
+}
+
+async fn append_pax_header(
+    builder: &mut tokio_tar::Builder<Cursor<Vec<u8>>>,
+    entry_type: tokio_tar::EntryType,
+    records: &[(&str, &str)],
+) {
+    let mut payload = Vec::new();
+    for (key, value) in records {
+        payload.extend(pax_record(key, value));
+    }
+    let mut header = tokio_tar::Header::new_ustar();
+    header.set_entry_type(entry_type);
+    header.set_size(u64::try_from(payload.len()).expect("PAX test payload should fit in u64"));
+    header.set_mode(0o644);
+    header.set_cksum();
+    builder
+        .append_data(&mut header, "PaxHeaders.X/snowball", Cursor::new(payload))
+        .await
+        .expect("PAX test header should be written");
+}
+
+async fn append_regular(builder: &mut tokio_tar::Builder<Cursor<Vec<u8>>>, path: &str) {
+    let body = path.as_bytes();
+    let mut header = tokio_tar::Header::new_ustar();
+    header.set_entry_type(tokio_tar::EntryType::Regular);
+    header.set_size(u64::try_from(body.len()).expect("test member body should fit in u64"));
+    header.set_mode(0o644);
+    header.set_mtime(1_704_164_645);
+    header.set_cksum();
+    builder
+        .append_data(&mut header, path, Cursor::new(body))
+        .await
+        .expect("ordinary test member should be written");
+}
+
+async fn archive_with_local_pax(records: &[(&str, &str)]) -> Vec<u8> {
+    let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
+    append_pax_header(&mut builder, tokio_tar::EntryType::XHeader, records).await;
+    append_regular(&mut builder, "member.txt").await;
+    builder.into_inner().await.expect("policy archive should finish").into_inner()
+}
+
+#[tokio::test]
+async fn candidate_policy_rejects_unknown_vendor_and_duplicate_pax_records() {
+    let unknown_vendor = archive_with_local_pax(&[("acme.metadata.owner", "mallory")]).await;
+    let error = match TarArchive::new(Cursor::new(unknown_vendor))
+        .with_policy(candidate_snowball_decode_policy())
+        .members()
+        .next()
+        .await
+    {
+        Err(error) => error,
+        Ok(_) => panic!("the candidate Snowball policy should reject unknown vendors"),
+    };
+    assert!(
+        error
+            .to_string()
+            .contains("pax vendor extension acme.metadata.owner is not allowed")
+    );
+
+    let duplicate = archive_with_local_pax(&[
+        ("minio.metadata.x-amz-meta-owner", "first"),
+        ("minio.metadata.x-amz-meta-owner", "second"),
+    ])
+    .await;
+    let error = match TarArchive::new(Cursor::new(duplicate))
+        .with_policy(candidate_snowball_decode_policy())
+        .members()
+        .next()
+        .await
+    {
+        Err(error) => error,
+        Ok(_) => panic!("the candidate Snowball policy should reject duplicate PAX records"),
+    };
+    assert!(
+        error
+            .to_string()
+            .contains("pax extended header contains duplicate record minio.metadata.x-amz-meta-owner")
+    );
+}
+
+#[tokio::test]
+async fn global_minio_pax_inheritance_is_an_explicit_migration_difference() {
+    let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
+    append_pax_header(
+        &mut builder,
+        tokio_tar::EntryType::XGlobalHeader,
+        &[("minio.metadata.x-amz-meta-owner", "global")],
+    )
+    .await;
+    append_pax_header(
+        &mut builder,
+        tokio_tar::EntryType::XHeader,
+        &[("minio.metadata.x-amz-meta-owner", "local")],
+    )
+    .await;
+    append_regular(&mut builder, "local.txt").await;
+    append_regular(&mut builder, "inherited.txt").await;
+    let archive = builder
+        .into_inner()
+        .await
+        .expect("precedence archive should finish")
+        .into_inner();
+
+    let legacy = parse_with_tokio_tar(&archive).await;
+    let (framing, error, _) = parse_with_tar_framing(&archive).await;
+    assert!(error.is_none());
+    assert_eq!(legacy.len(), 2);
+    assert_eq!(framing.len(), 2);
+
+    let owner_key = "minio.metadata.x-amz-meta-owner";
+    assert_eq!(legacy[0].minio_pax.get(owner_key), Some(&Some(b"local".to_vec())));
+    assert!(!legacy[1].minio_pax.contains_key(owner_key));
+    assert_eq!(framing[0].minio_pax.get(owner_key), Some(&Some(b"local".to_vec())));
+    assert_eq!(framing[1].minio_pax.get(owner_key), Some(&Some(b"global".to_vec())));
+
+    let error = match TarArchive::new(Cursor::new(archive))
+        .with_policy(candidate_snowball_decode_policy())
+        .members()
+        .next()
+        .await
+    {
+        Err(error) => error,
+        Ok(_) => panic!("the candidate Snowball policy should reject global PAX state"),
+    };
+    assert!(error.to_string().contains("global pax extended headers are not allowed"));
+}

deny.toml

@@ -37,8 +37,8 @@ unknown-git = "deny"
 allow-registry = ["https://github.com/rust-lang/crates.io-index"]
 allow-git = [
     # Temporary tokio-tar fork pinned to the reviewed parser limits,
-    # cancellation safety, and error-fusing change while
+    # cancellation safety, and error-fusing change while Snowball is
-    # astral-sh/tokio-tar#118 awaits an upstream release.
+    # prototyped against tar-codec and Swift retains its current reader.
     # owner: cxymds  review: 2026-10
     "https://github.com/cxymds/tokio-tar.git",
     # Official s3s repository. Temporarily pinned to the merged generic REST

docs/architecture/compat-cleanup-register.md

@@ -13,7 +13,7 @@
 
 - `backlog-1337` legacy restore orphan recovery: releases that predate the restore worker-lock marker can leave a valid operation-id and `ongoing-request="true"` after cancellation or process failure, with no durable liveness proof. New servers allow an exact, non-nil legacy generation to be superseded only when its consistently parsed request date is at least 24 hours old. Remove the clock-based legacy fallback after the minimum supported direct-upgrade release writes the v1 worker-lock marker on every restore and operators have resolved every retained pre-v1 ongoing generation.
 - `backlog-2133-tier-delete-chunk-parent` bounded tier-delete dispatch compatibility: prefixes at or below the legacy manifest limit keep the byte-compatible v1 single-manifest protocol, while larger prefixes place a chunk-parent sentinel at the original deterministic root path and use operation-scoped child manifests. Older binaries reject the sentinel and child paths, preserving the v6 sole-owner downgrade fence instead of starting a competing local delete. Remove the v1 reader and fail-closed mixed-version sentinel only after every supported rollback release validates the parent/child protocol and migration tooling confirms that no retained v1 dispatch manifest remains.
-- `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on per-entry and cumulative GNU long-name, GNU long-link, and PAX extension limits; physical-entry, GNU sparse-map, and sparse-continuation limits; cancellation-safe sparse parsing; and fused entry streams after parser errors. The released tokio-tar API does not provide this complete boundary. Keep the reviewed fork pin until astral-sh/tokio-tar#118 is merged and one published tokio-tar release contains every listed capability with the Snowball regression fixtures passing against that release.
+- `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on precedence-resolved MinIO PAX metadata; per-entry and cumulative extension limits; a physical-entry limit; cancellation-safe parsing and ownership of large streamed members; fused streams after errors; and compatibility with minio-go streams that omit the two-block terminator. Swift bulk extraction also uses the same fork. Keep the reviewed pin while the Snowball path is prototyped against tar-codec/tar-framing. Remove it only after a released API exposes the effective allowed vendor records, RustFS provides a cancellation-safe handoff for borrowed member payloads, footerless input is accepted solely when authenticated request framing proves EOF immediately after a complete member, the existing resource-limit, cancellation, error-fuse, and real minio-go fixtures pass against the replacement, and Swift no longer depends on the fork.
 - `backlog-2102` rc.2/rc.3 empty scanner usage floor recovery: old DeleteBucket cleanup could synthesize an empty incomplete v2 usage primary/backup before leadership added an epoch, while newer scanners require a durable authoritative baseline identity. New scanners recognize only that exact serialized empty-fence shape, preserve its epoch through a CAS-protected recovery marker, and rebuild namespace coverage without treating zero usage as authoritative. Remove this recovery path and marker after rc.2 and rc.3 are no longer supported direct-upgrade sources.
 - `backlog-2122` rc.1-rc.3 non-empty scanner usage floor recovery: leadership fencing in those releases can stamp scanner_epoch onto a real bucket-usage snapshot before any scanner cycle completed, leaving a non-empty floor with no scanner_cycle and no authoritative baseline identity. New scanners recognize only this consistent incomplete fenced shape, preserve the epoch through the CAS-protected recovery marker, and rebuild namespace coverage without treating the old usage data as authoritative. Remove this recovery path after rc.1, rc.2, and rc.3 are no longer supported direct-upgrade sources.
 - `s3gate-metadata-xml` persisted bucket XML migration: mixed-version site-replication peers, retained `.metadata.bin` objects, and backup archives can all carry XML written by the s3s codec, so the gateway migration must keep the legacy codec available until every stored form has crossed a verified rewrite boundary. Remove the legacy s3s parser and serializer only after the minimum supported direct-upgrade release reads and writes every persisted XML configuration family through the gateway codec, every supported mixed-version site-replication topology has completed its writer upgrade, and migration tooling has verified or rewritten every retained bucket metadata object and restorable backup archive.

docs/testing/README.md

@@ -21,6 +21,8 @@ Pick the lowest layer that can prove the change; add a higher-layer test only wh
 
 Every script named above is indexed with status and wiring in [`scripts/README.md`](../../scripts/README.md). Fixed GHSA advisories map to named regression tests in [security-regressions.md](security-regressions.md).
 
+The [scanner checkpoint fixture](scanner-checkpoint-fixture.md) diagnoses retained subtree coverage across budget interruption, persistence, reload, and plan invalidation.
+
 ## Naming conventions
 
 ### Reserved test-name substrings (migration gate)

docs/testing/ecstore-validation-suite-design.md

@@ -54,6 +54,22 @@ Fail-closed invariants every row enforces:
 
 Fault injection is explicit and deterministic: local disk mocks for unit tests, process-level disk manipulation (`crates/e2e_test/src/chaos.rs`) for e2e tests. Property tests replay a fixed seed for payload, range, and missing-shard selection.
 
+### PUT completion fixtures
+
+`ObjectOptions::default()` uses `WriteCompletion::Quorum`: a namespace-lock-owning PUT may acknowledge write quorum while its rename tail retains the lock. A fixture that immediately inspects every disk or primes a metadata generation must set `write_completion: WriteCompletion::TailDrained` and keep normal locking. TailDrained waits for the existing rename fan-out; it does not require every disk to succeed or change fsync policy. Codec-only `no_lock` fixtures do not cover namespace locking.
+
+The object tests reuse `rename_fanout_barrier::arm(object, disk_slot, phase)` and `observe_tasks(object)`. Wait for the barrier with a deadline, observe actual metadata quorum with `wait_for_paused_tail_metadata_quorum`, then release or cancel. The metadata check distinguishes a real quorum from disk tasks that have not started. Assert zero remaining rename tasks after the owned coordinator releases its lock; cancellation tests also wait for staging cleanup.
+
+| Fixture | Completion boundary |
+|---|---|
+| `early_ack_tail_drain_retains_namespace_lock_until_background_rename_finishes` | Default PUT returns before the parked tail; a second writer remains blocked. |
+| `tail_drained_put_*` | Explicit full-tail PUT retains its guard, preserves quorum success with a failed minority, rejects quorum-minus-one, and survives ACK waiter cancellation. |
+| `transition_and_restore_reclaim_prior_metadata_generations` | Both source fixtures use TailDrained before cache priming, with normal namespace locks. |
+| `object_transaction_fencing_persists_epoch_on_multipart_commit` | Multipart completion already always drains rename before inspecting all per-disk transaction UUIDs. |
+| `decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page`, `dispatch_completion_cas_is_bounded_and_reaches_the_tail` | Durable receipt, journal, and manifest writers choose TailDrained; the pagination fixture also drains deliberate receipt replacement writes. |
+
+Select these checks with `cargo nextest list -p rustfs-ecstore --features test-util -E 'test(tail_drained_put) | test(early_ack_tail_drain) | test(no_lock_put_waits_for_rename_tail) | test(object_transaction_fencing_persists_epoch_on_multipart_commit) | test(transition_and_restore_reclaim) | test(decommission_durable_ilm_receipt_pagination) | test(dispatch_completion_cas)'`, then run the same expression under the default and CI profiles without retries. Remaining crash, reopen, rollback, and lock-loss schedules use the existing domain tests; this completion fixture is not a replacement for those checks.
+
 ### Coverage gate
 
 `full` and `destructive` run `cargo llvm-cov -p rustfs-ecstore --lib` and fail when line coverage of the gate scope is below `--unit-coverage-min`. The default minimum and the 100% target for EC read, write, decode, heal, metadata-quorum, and rollback paths are the `UNIT_COVERAGE_*` constants at the top of the runner. `cargo-llvm-cov` must be installed unless `--skip-coverage` is passed explicitly. The default scope `ec-critical` is:

docs/testing/scanner-checkpoint-fixture.md

@@ -0,0 +1,22 @@
+# Scanner Checkpoint Fixture
+
+The `checkpoint_fixture` tests exercise a bounded namespace of 24 static objects and one repeatedly updated hot object. Each of three rounds runs the production local disk scanner with an object budget, saves the returned partial cache through the production persistence codec and revision checks to a two-file test backend, and reloads it before preparing the next round. The fixture prints static-subtree coverage at each boundary and cumulative visited entries. This is a diagnostic of retained coverage, not a throughput benchmark.
+
+Run the fixture and confirm the test filter selects a nonzero number of tests:
+
+```sh
+cargo test -p rustfs-scanner --lib checkpoint_fixture -- --list
+RUST_MIN_STACK=4194304 cargo test -p rustfs-scanner --lib checkpoint_fixture -- --nocapture
+```
+
+The unchanged-plan case requires durable static coverage to increase each round. The hot-plan diagnostic changes the bucket plan digest between rounds and reports where coverage is lost without asserting that a particular defect must remain present. To require progress in this diagnostic as well:
+
+```sh
+RUST_MIN_STACK=4194304 RUSTFS_CHECKPOINT_REQUIRE_PROGRESS=1 cargo test -p rustfs-scanner --lib checkpoint_fixture_hot_digest_diagnostic -- --nocapture
+```
+
+A nonzero exit from the strict command means that walked work did not become additional retained static coverage. `LostAtPrepare` identifies invalidation before traversal; `LostAtReload` identifies loss between the returned cache and persisted data; `WalkWithoutRetention` identifies visited growth without durable coverage growth. Missing, corrupt, empty-root, and oversized checkpoint inputs are rejected by the strict fixture reader. Save failure and publication-epoch rejection must preserve the preceding file bytes. Parent cancellation is checked separately from object-budget exhaustion. Superseded classification is tested separately from either incomplete outcome.
+
+For every saved partial cache, the fixture also passes its progress through the production authenticated remote terminal-frame writer and stream consumer. A remote partial result must remain partial even when its progress reports visited objects. This covers the return-frame contract; it does not execute the remote RPC server, distributed locks, EC quorum persistence, mixed-version peers, process crashes, or fsync durability. The file backend models revision preconditions and persistence errors, not a concurrent object store.
+
+The synthetic namespace contains no customer data. Temporary files are removed with their owning fixture. Production scan semantics and persistent formats are unchanged, so rollback consists of removing these tests and this guide. A passing fixture alone does not establish that the field report in [issue #7108](https://github.com/rustfs/rustfs/issues/7108) has been independently reproduced or fixed. A field diagnosis must separately identify the source capture, cycle and leader identity, and decoded bucket/set caches.

rustfs/src/app/bucket_list_through.rs

@@ -252,8 +252,16 @@ pub(crate) async fn merged_list_objects_v2(
             .filter(|entry| merger.accepts(&entry.key().name))
             .collect();
         let keys: Vec<ListEntryKey> = kept.iter().map(SideEntry::key).collect();
+        if let Err(error) = merger.push_page(fetch.side, keys, is_truncated, next_token) {
+            match fetch.side {
+                MergeSide::Source => {
+                    degrade_or_fail(&mut merger, &mut degraded, policy.source_error, "invalid_pagination")?;
+                    continue;
+                }
+                MergeSide::Local => return Err(S3Error::with_message(S3ErrorCode::InternalError, error.to_string())),
+            }
+        }
         buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.into_iter().map(Some));
-        merger.push_page(fetch.side, keys, is_truncated, next_token);
     }
 
     let outcome = merger.finish();
@@ -340,10 +348,12 @@ async fn fetch_source_page(
             continuation_token: token,
             max_keys: params.max_keys,
         },
-        // Everything under `filter.prefix` rolls into one common prefix, so a
+        // Everything under `filter.prefix` rolls into one common prefix. An
-        // single bounded listing settles whether it exists.
+        // empty truncated probe must still follow its cursor before declaring
+        // that prefix absent.
         SourceListPlan::Folded { probe_prefix, .. } => SourceListRequest {
             prefix: Some(probe_prefix.as_str()),
+            continuation_token: token,
             max_keys: 1,
             ..Default::default()
         },
@@ -368,8 +378,8 @@ async fn fetch_source_page(
                 } else {
                     Vec::new()
                 },
-                false,
+                !exists && page.is_truncated,
-                None,
+                if exists { None } else { page.next_continuation_token },
             ))
         }
         _ => {
@@ -417,6 +427,17 @@ async fn local_delete_markers(store: &Arc<ECStore>, bucket: &str, keys: &[String
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::app::bucket_usecase::DefaultBucketUsecase;
+    use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ecstore};
+    use crate::app::storage_api::bucket_usecase::bucket::on_demand_migration::{
+        FilterConfig, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig, SourceCredentials, TlsConfig,
+    };
+    use crate::app::storage_api::bucket_usecase::s3::{ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response};
+    use crate::app::storage_api::test::StoragePutObjReader;
+    use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
+    use crate::app::storage_api::test::contract::object::ObjectIO as _;
+    use std::time::Duration;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
     fn token(local: Option<&str>, local_done: bool) -> ListThroughToken {
         ListThroughToken {
@@ -529,4 +550,332 @@ mod tests {
         assert!(degraded);
         assert_eq!(merger.next_fetch().map(|fetch| fetch.side), Some(MergeSide::Local));
     }
+
+    /// Serves exactly the scripted S3 pages and joins every connection before
+    /// returning. A source retry or unexpected operation fails the test.
+    async fn scripted_list_source(pages: Vec<String>) -> (String, tokio_util::task::AbortOnDropHandle<Vec<String>>) {
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
+            .await
+            .expect("bind listing source");
+        let address = listener.local_addr().expect("listing source address");
+        let server = tokio::spawn(async move {
+            let mut requests = Vec::new();
+            for body in pages {
+                let (mut stream, _) = listener.accept().await.expect("accept source listing");
+                let mut request = Vec::new();
+                let mut chunk = [0; 4096];
+                while !request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    let count = stream.read(&mut chunk).await.expect("read signed listing request");
+                    assert!(count > 0, "source request must include complete headers");
+                    request.extend_from_slice(&chunk[..count]);
+                    assert!(request.len() <= 32 * 1024, "listing request headers must be bounded");
+                }
+                let first_line = String::from_utf8_lossy(&request)
+                    .lines()
+                    .next()
+                    .expect("request line")
+                    .to_string();
+                // The SDK joins the bucket endpoint with the LIST operation's `/` path.
+                assert!(
+                    first_line.starts_with("GET /source-bucket/?"),
+                    "expected a path-style bucket-root LIST request, got {first_line:?}"
+                );
+                assert!(first_line.contains("list-type=2"), "expected a ListObjectsV2 query, got {first_line:?}");
+                requests.push(first_line);
+                let response = format!(
+                    "HTTP/1.1 200 OK\r\ncontent-type: application/xml\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
+                    body.len()
+                );
+                stream.write_all(response.as_bytes()).await.expect("write source page");
+                stream.shutdown().await.expect("finish source response");
+            }
+            requests
+        });
+        (format!("http://{address}"), tokio_util::task::AbortOnDropHandle::new(server))
+    }
+
+    fn source_xml(next: Option<&str>, truncated: bool, key: Option<&str>) -> String {
+        let next = next
+            .map(|token| format!("<NextContinuationToken>{token}</NextContinuationToken>"))
+            .unwrap_or_default();
+        let contents = key
+            .map(|key| format!("<Contents><Key>{key}</Key><Size>1</Size></Contents>"))
+            .unwrap_or_default();
+        format!(
+            "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><IsTruncated>{truncated}</IsTruncated>{next}{contents}</ListBucketResult>"
+        )
+    }
+
+    struct ListThroughTestState {
+        bucket: String,
+        module_enabled: bool,
+    }
+
+    impl Drop for ListThroughTestState {
+        fn drop(&mut self) {
+            let sys = OnDemandMigrationSys::get();
+            sys.remove(&self.bucket);
+            sys.set_module_enabled(self.module_enabled);
+        }
+    }
+
+    async fn source_policy_request(
+        pages: Vec<String>,
+        policy: SourceErrorPolicy,
+        resume_source: Option<&str>,
+        filter_prefix: Option<&str>,
+    ) -> (S3Result<S3Response<ListObjectsV2Output>>, Vec<String>) {
+        let store = shared_gating_ecstore().await;
+        crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
+        let bucket = format!("odm-list-{}", uuid::Uuid::new_v4().simple());
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create list-through bucket");
+        store
+            .put_object(
+                &bucket,
+                "z-local",
+                &mut StoragePutObjReader::from_vec(vec![1]),
+                &StorageObjectOptions::default(),
+            )
+            .await
+            .expect("seed real local listing");
+        let (endpoint, server) = scripted_list_source(pages).await;
+        let sys = OnDemandMigrationSys::get();
+        let _state_guard = ListThroughTestState {
+            bucket: bucket.clone(),
+            module_enabled: sys.is_module_enabled(),
+        };
+        sys.set_module_enabled(true);
+        let config = OnDemandMigrationConfig {
+            version: 1,
+            enabled: true,
+            source: SourceConfig {
+                provider: Provider::Minio,
+                endpoint: Some(endpoint),
+                region: "us-east-1".into(),
+                bucket: "source-bucket".into(),
+                path_style: PathStyle::Path,
+                credentials: Some(SourceCredentials {
+                    access_key: "test-access".into(),
+                    secret_key: "test-secret".into(),
+                    session_token: None,
+                }),
+                tls: TlsConfig::default(),
+            },
+            filter: FilterConfig {
+                prefix: filter_prefix.map(str::to_string),
+                ..Default::default()
+            },
+            policy: PolicyConfig {
+                list_through: true,
+                source_error: policy,
+                ..Default::default()
+            },
+        };
+        sys.apply(&bucket, Some(&config)).await;
+        assert!(
+            sys.state(&bucket).expect("ODM state installed").client().is_ok(),
+            "fake source client must build"
+        );
+        let continuation_token = resume_source.map(|source| {
+            let token = ListThroughToken {
+                t: "odm-list".into(),
+                v: 1,
+                local: None,
+                local_done: false,
+                source: Some(source.into()),
+                source_done: false,
+                last_key: None,
+            };
+            base64_simd::STANDARD.encode_to_string(token.encode().as_bytes())
+        });
+        let input = ListObjectsV2Input {
+            bucket,
+            max_keys: Some(2),
+            continuation_token,
+            delimiter: filter_prefix.map(|_| "/".to_string()),
+            encoding_type: None,
+            expected_bucket_owner: None,
+            fetch_owner: None,
+            optional_object_attributes: None,
+            prefix: None,
+            request_payer: None,
+            start_after: None,
+        };
+        let request = S3Request {
+            input,
+            method: http::Method::GET,
+            uri: http::Uri::from_static("/?list-type=2"),
+            headers: HeaderMap::new(),
+            extensions: http::Extensions::new(),
+            credentials: None,
+            region: None,
+            service: None,
+            trailing_headers: None,
+        };
+        let result = tokio::time::timeout(
+            Duration::from_secs(10),
+            DefaultBucketUsecase::from_global().execute_list_objects_v2(request),
+        )
+        .await
+        .expect("listing must complete within its bounded source budget");
+        let requests = tokio::time::timeout(Duration::from_secs(5), server)
+            .await
+            .expect("source connections must finish")
+            .expect("source server must not panic");
+        (result, requests)
+    }
+
+    #[test]
+    #[serial_test::serial]
+    fn list_through_invalid_source_pagination_obeys_policy_on_the_handler_path() {
+        run_large_stack_test("list-through-source-policy", || async {
+            temp_env::async_with_vars(
+                [
+                    ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
+                    ("HTTP_PROXY", None),
+                    ("HTTPS_PROXY", None),
+                    ("ALL_PROXY", None),
+                    ("http_proxy", None),
+                    ("https_proxy", None),
+                    ("all_proxy", None),
+                    ("NO_PROXY", Some("*")),
+                    ("no_proxy", Some("*")),
+                ],
+                async {
+                    for policy in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] {
+                        for next in [None, Some(""), Some("stuck")] {
+                            for key in [None, Some("a-source")] {
+                                let (result, requests) =
+                                    source_policy_request(vec![source_xml(next, true, key)], policy, Some("stuck"), None).await;
+                                assert_eq!(requests.len(), 1, "a malformed source page must not be retried");
+                                assert!(requests[0].contains("continuation-token=stuck"));
+                                assert_source_policy_result(result, policy);
+                            }
+                        }
+                        let (result, requests) = source_policy_request(
+                            vec![
+                                source_xml(Some("stuck"), true, Some("a-source")),
+                                source_xml(Some("stuck"), true, None),
+                            ],
+                            policy,
+                            None,
+                            None,
+                        )
+                        .await;
+                        assert_eq!(requests.len(), 2, "the failure must occur during a real refill");
+                        assert!(!requests[0].contains("continuation-token="));
+                        assert!(requests[1].contains("continuation-token=stuck"));
+                        assert_source_policy_result(result, policy);
+                    }
+                },
+            )
+            .await;
+        });
+    }
+
+    #[test]
+    #[serial_test::serial]
+    fn list_through_empty_advancing_source_pages_reach_eof_on_the_handler_path() {
+        run_large_stack_test("list-through-empty-source-pages", || async {
+            temp_env::async_with_vars(
+                [
+                    ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
+                    ("HTTP_PROXY", None),
+                    ("HTTPS_PROXY", None),
+                    ("ALL_PROXY", None),
+                    ("http_proxy", None),
+                    ("https_proxy", None),
+                    ("all_proxy", None),
+                    ("NO_PROXY", Some("*")),
+                    ("no_proxy", Some("*")),
+                ],
+                async {
+                    for filter_prefix in [None, Some("photos/2024/")] {
+                        let source_key = if filter_prefix.is_some() {
+                            "photos/2024/a-source"
+                        } else {
+                            "a-source"
+                        };
+                        let (result, requests) = source_policy_request(
+                            vec![
+                                source_xml(Some("opaque-next"), true, None),
+                                source_xml(None, false, Some(source_key)),
+                            ],
+                            SourceErrorPolicy::Propagate,
+                            None,
+                            filter_prefix,
+                        )
+                        .await;
+                        assert_eq!(requests.len(), 2, "an empty truncated source page must reach its successor");
+                        assert!(requests[1].contains("continuation-token=opaque-next"));
+                        let response = result.expect("empty progressing source page is valid");
+                        assert!(!response.headers.contains_key("x-rustfs-on-demand-migration-list"));
+                        let output = response.output;
+                        let objects: Vec<_> = output
+                            .contents
+                            .unwrap_or_default()
+                            .into_iter()
+                            .map(|object| object.key.expect("listed object key"))
+                            .collect();
+                        if filter_prefix.is_some() {
+                            assert_eq!(objects, vec!["z-local"]);
+                            assert_eq!(
+                                output
+                                    .common_prefixes
+                                    .unwrap_or_default()
+                                    .into_iter()
+                                    .map(|prefix| prefix.prefix.expect("rolled-up prefix"))
+                                    .collect::<Vec<_>>(),
+                                vec!["photos/"]
+                            );
+                        } else {
+                            assert_eq!(objects, vec!["a-source", "z-local"]);
+                            assert!(output.common_prefixes.unwrap_or_default().is_empty());
+                        }
+                        assert_eq!(output.key_count, Some(2));
+                        assert_eq!(output.is_truncated, Some(false));
+                        assert!(output.next_continuation_token.is_none());
+                    }
+                },
+            )
+            .await;
+        });
+    }
+
+    fn assert_source_policy_result(result: S3Result<S3Response<ListObjectsV2Output>>, policy: SourceErrorPolicy) {
+        match policy {
+            SourceErrorPolicy::Propagate => {
+                let error = result.expect_err("propagate must expose malformed pagination");
+                assert_eq!(error.status_code(), Some(http::StatusCode::FAILED_DEPENDENCY));
+                assert_eq!(error.code(), &S3ErrorCode::Custom("SourceUnavailable".into()));
+                assert_eq!(error.message(), Some("invalid_pagination"));
+            }
+            SourceErrorPolicy::NotFound => {
+                let response = result.expect("not_found must preserve the local listing");
+                assert_eq!(
+                    response
+                        .headers
+                        .get("x-rustfs-on-demand-migration-list")
+                        .expect("local_only header"),
+                    "local_only"
+                );
+                let output = response.output;
+                assert_eq!(
+                    output
+                        .contents
+                        .unwrap_or_default()
+                        .into_iter()
+                        .map(|object| object.key.expect("local key"))
+                        .collect::<Vec<_>>(),
+                    vec!["z-local"]
+                );
+                assert_eq!(output.is_truncated, Some(false));
+                assert_eq!(output.key_count, Some(1));
+                assert!(output.next_continuation_token.is_none());
+            }
+        }
+    }
 }

rustfs/src/app/storage_api.rs

@@ -29,11 +29,13 @@ pub(crate) fn EndpointServerPools(
 pub(crate) mod s3 {
     #[cfg(test)]
     pub(crate) use s3s::dto::{
-        BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ReplicationConfiguration,
+        BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ListObjectsV2Input,
-        ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, ServerSideEncryptionByDefault,
+        ListObjectsV2Output, ReplicationConfiguration, ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus,
-        ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration,
+        ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration,
     };
     pub(crate) use s3s::{S3Error, S3ErrorCode, S3Result};
+    #[cfg(test)]
+    pub(crate) use s3s::{S3Request, S3Response};
 }
 
 pub(crate) mod admin {

scripts/ecstore-module-lint-register.txt

@@ -20,8 +20,6 @@
 crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs|clippy::all
 crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs|unused_must_use
 crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs|unused_variables
-crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|clippy::all
-crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|unused_must_use
 crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|unused_variables
 crates/s3-client/src/api_error_response.rs|clippy::all
 crates/s3-client/src/api_error_response.rs|unused_must_use
@@ -74,36 +72,18 @@ crates/ecstore/src/services/event_notification.rs|unused_variables
 crates/ecstore/src/services/tier/tier.rs|clippy::all
 crates/ecstore/src/services/tier/tier.rs|unused_must_use
 crates/ecstore/src/services/tier/tier.rs|unused_variables
-crates/ecstore/src/services/tier/tier_admin.rs|clippy::all
-crates/ecstore/src/services/tier/tier_admin.rs|unused_must_use
 crates/ecstore/src/services/tier/tier_admin.rs|unused_variables
 crates/ecstore/src/services/tier/warm_backend.rs|clippy::all
 crates/ecstore/src/services/tier/warm_backend.rs|unused_must_use
 crates/ecstore/src/services/tier/warm_backend.rs|unused_variables
-crates/ecstore/src/services/tier/warm_backend_aliyun.rs|clippy::all
-crates/ecstore/src/services/tier/warm_backend_aliyun.rs|unused_must_use
 crates/ecstore/src/services/tier/warm_backend_aliyun.rs|unused_variables
-crates/ecstore/src/services/tier/warm_backend_azure.rs|clippy::all
-crates/ecstore/src/services/tier/warm_backend_azure.rs|unused_must_use
 crates/ecstore/src/services/tier/warm_backend_azure.rs|unused_variables
-crates/ecstore/src/services/tier/warm_backend_gcs.rs|clippy::all
-crates/ecstore/src/services/tier/warm_backend_gcs.rs|unused_must_use
 crates/ecstore/src/services/tier/warm_backend_gcs.rs|unused_variables
-crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs|clippy::all
-crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs|unused_must_use
 crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs|unused_variables
-crates/ecstore/src/services/tier/warm_backend_minio.rs|clippy::all
-crates/ecstore/src/services/tier/warm_backend_minio.rs|unused_must_use
 crates/ecstore/src/services/tier/warm_backend_minio.rs|unused_variables
-crates/ecstore/src/services/tier/warm_backend_r2.rs|clippy::all
-crates/ecstore/src/services/tier/warm_backend_r2.rs|unused_must_use
 crates/ecstore/src/services/tier/warm_backend_r2.rs|unused_variables
-crates/ecstore/src/services/tier/warm_backend_rustfs.rs|clippy::all
-crates/ecstore/src/services/tier/warm_backend_rustfs.rs|unused_must_use
 crates/ecstore/src/services/tier/warm_backend_rustfs.rs|unused_variables
 crates/ecstore/src/services/tier/warm_backend_s3.rs|clippy::all
 crates/ecstore/src/services/tier/warm_backend_s3.rs|unused_must_use
 crates/ecstore/src/services/tier/warm_backend_s3.rs|unused_variables
-crates/ecstore/src/services/tier/warm_backend_tencent.rs|clippy::all
-crates/ecstore/src/services/tier/warm_backend_tencent.rs|unused_must_use
 crates/ecstore/src/services/tier/warm_backend_tencent.rs|unused_variables