test: report tier reference proof setup failures without hanging

Co-authored-by: zhi22915 <qiuzgang@gmail.com>

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=4696a43b167ac608b3b8677027c9fe9fdac3396d37c8cca11dce531c720ac6d2
-sha256-darwin=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07
+sha256-linux=563bff8f1171d6dbe166ff8440310dbe98430e466aa3ecd8dc39e3c872b320f7
+sha256-darwin=563bff8f1171d6dbe166ff8440310dbe98430e466aa3ecd8dc39e3c872b320f7

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=f0c78fdb93471575d9a64c5c46eae6c806bdd0bc10a6e33d7fb574aabd8db5a3
-sha256-linux=03ed7016cab672de9320e31375a0358eceacb4408b0e79cf063614fa7c878b87
+sha256-darwin=874c881d7b45f12378a5817c7f42c95c4981960a2ec9ce12dcf4af239ae1f9d5
+sha256-linux=9515861be899ceb10e2e0ef93c34208bb7a7a8a7f8067a02db4cfba23270ebd6

.config/e2e-nightly-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=364f2329a7b72eb9f1608dbe1a3af37af4095354014f3cbe23ca448492d89961
-sha256-linux=60983f1ebe7068cf660d473c5f76c76a650410ccc99d71934ddca7fd67607987
+sha256-darwin=83a7dcaffd5a789517ae9f02a224f66a9713937885cff96fca2ad7e216f197ae
+sha256-linux=626c10f8c964507ff987b6c86069e9019dc6d2ae7fb02db9be5df5aa8cc5145b

.config/make/lint-fmt.mak

@@ -89,6 +89,7 @@ offline-enrollment-e2e-check: core-deps ## Build and exercise the dedicated offl
 test-wiring-check: ## Check tests stay registered and selected by their intended runners
 	@echo "🧪 Checking test wiring..."
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --check-workflow
 
 .PHONY: log-analyzer-rules-check
 log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source

.config/make/tests.mak

@@ -32,6 +32,16 @@ script-tests: ## Run shell script tests
 	./scripts/test_hotpath_warp_ab_gate.sh
 	./scripts/test_hotpath_warp_abba.sh
 	./scripts/test_scanner_validation_harness.sh
+	./scripts/test_scanner_heal_checkpoint_crash_evidence.sh
+	./scripts/test_scanner_heal_authority_evidence.sh
+	./scripts/test_scanner_heal_scoped_ack_evidence.sh
+	./scripts/test_scanner_heal_legacy_rollback_evidence.sh
+	./scripts/test_scanner_heal_g14_multiset_evidence.sh
+	./scripts/test_scanner_heal_scheduler_pressure_evidence.sh
+	./scripts/test_scanner_heal_status_outcome_evidence.sh
+	./scripts/test_scanner_heal_maintenance_evidence.sh
+	./scripts/test_scanner_heal_w13_mrf_evidence.sh
+	./scripts/test_scanner_heal_w16_recovery_evidence.sh
 	./scripts/test_exact_1mib_handoff_abba.sh
 	./scripts/test_pinned_paired_abba_bench.sh
 	./scripts/test_manual_transition_runbooks.sh
@@ -39,6 +49,7 @@ script-tests: ## Run shell script tests
 	./scripts/test_python_bin.sh
 	./scripts/check_embedded_secrets.sh --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py

.config/nextest.toml

@@ -197,6 +197,12 @@ test-group = 'e2e-cluster-nightly'
 filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
 test-group = 'e2e-vault'
 
+# This four-disk, 65-member rollback probe already drives up to 32 concurrent
+# durable deletions. Reserve this nextest run's capacity for its progress oracle.
+[[profile.default.overrides]]
+filter = 'package(rustfs-ecstore) & test(=store::init::tests::dispatch_manifest_rollback_bounded_concurrency_reaches_tail_behind_slow_member)'
+threads-required = "num-test-threads"
+
 # ---------------------------------------------------------------------------
 # ci profile — the strict CI gate (ci.yml `cargo nextest run --profile ci`)
 # ---------------------------------------------------------------------------

.config/scanner-heal-required-tests.json

@@ -78,6 +78,43 @@
       "erasure": {"data_blocks": 8, "parity_blocks": 4},
       "erasure_set_drive_count": 12,
       "scope": "Target process killed during partial background rebuild on a single 3x4 EC8+4 set; real unclean-shutdown marker, exact unversioned S3 bodies and replacement-drive shards; not power loss, multi-set, or multi-pool."
+    },
+    "background-target-restart-ec8-4-multi-set": {
+      "gate": "G14",
+      "task": "W21",
+      "lane": "e2e-nightly",
+      "suite": "e2e_test",
+      "name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart",
+      "oracle": "background-target-restart-ec8-4-multi-set.json",
+      "evidence": "process-restart",
+      "unclean_shutdown_marker": false,
+      "min_objects": 9,
+      "max_objects": 65,
+      "topology": {"nodes": 3, "drives_per_node": 8},
+      "erasure": {"data_blocks": 8, "parity_blocks": 4},
+      "erasure_set_drive_count": 12,
+      "sets": 2,
+      "pools": 1,
+      "scope": "Target process restart during partial background rebuild on a 3x8 EC8+4 layout with two erasure sets in one pool; exact unversioned S3 bodies and replacement-drive shards; not power loss or multi-pool."
+    },
+    "background-target-crash-ec8-4-multi-pool": {
+      "gate": "G14",
+      "task": "W21",
+      "lane": "e2e-nightly",
+      "suite": "e2e_test",
+      "name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_pool_after_background_target_crash",
+      "oracle": "background-target-crash-ec8-4-multi-pool.json",
+      "evidence": "process-crash-restart",
+      "unclean_shutdown_marker": true,
+      "min_objects": 9,
+      "max_objects": 65,
+      "topology": {"nodes": 3, "drives_per_node": 12},
+      "erasure": {"data_blocks": 8, "parity_blocks": 4},
+      "erasure_set_drive_count": 12,
+      "sets": 3,
+      "pools": 3,
+      "outage_target_manifest_required": false,
+      "scope": "Target process crash during partial background rebuild on three single-node EC8+4 pools; exact baseline S3 bodies and replacement-drive shards, with outage object verified through S3 but not forced onto the replaced target drive."
     }
   },
   "release_lanes": {
@@ -134,7 +171,11 @@
       "lane": "authority-coverage",
       "status": "pending",
       "description": "Complete root and quota authority coverage",
-      "requires": ["root authority evidence", "quota authority evidence"]
+      "requires": ["root authority evidence", "quota authority evidence"],
+      "evidence_fields": [
+        "root_authority_evidence",
+        "quota_authority_evidence"
+      ]
     },
     {
       "gate": "G02",
@@ -142,7 +183,11 @@
       "lane": "checkpoint-and-crash",
       "status": "pending",
       "description": "Bounded checkpoint progress and independent version inventory",
-      "requires": ["bounded checkpoint oracle", "independent version inventory"]
+      "requires": ["bounded checkpoint oracle", "independent version inventory"],
+      "evidence_fields": [
+        "bounded_checkpoint_oracle",
+        "independent_version_inventory"
+      ]
     },
     {
       "gate": "G03",
@@ -164,7 +209,11 @@
       "lane": "checkpoint-and-crash",
       "status": "pending",
       "description": "Crash at every cache, root, floor and intent boundary",
-      "requires": ["cache boundary crash evidence", "root/floor/intent crash evidence"]
+      "requires": ["cache boundary crash evidence", "root/floor/intent crash evidence"],
+      "evidence_fields": [
+        "cache_boundary_crash_evidence",
+        "root_floor_intent_crash_evidence"
+      ]
     },
     {
       "gate": "G05",
@@ -172,7 +221,11 @@
       "lane": "status-and-outcome",
       "status": "pending",
       "description": "Per-object outcomes and bounded terminal retention",
-      "requires": ["per-object outcome oracle", "terminal retention bounds"]
+      "requires": ["per-object outcome oracle", "terminal retention bounds"],
+      "evidence_fields": [
+        "per_object_outcome_oracle",
+        "terminal_retention_bounds"
+      ]
     },
     {
       "gate": "G06",
@@ -180,7 +233,12 @@
       "lane": "status-and-outcome",
       "status": "pending",
       "description": "Concurrent status, legacy clients and truncation",
-      "requires": ["concurrent status evidence", "legacy client compatibility", "truncation behavior"]
+      "requires": ["concurrent status evidence", "legacy client compatibility", "truncation behavior"],
+      "evidence_fields": [
+        "concurrent_status_evidence",
+        "legacy_client_compatibility",
+        "truncation_behavior"
+      ]
     },
     {
       "gate": "G07",
@@ -226,7 +284,11 @@
       "lane": "scheduler-pressure",
       "status": "pending",
       "description": "Bounded scheduling and pressure recovery",
-      "requires": ["scheduler bound evidence", "pressure recovery evidence"]
+      "requires": ["scheduler bound evidence", "pressure recovery evidence"],
+      "evidence_fields": [
+        "scheduler_bound_evidence",
+        "pressure_recovery_evidence"
+      ]
     },
     {
       "gate": "G11",
@@ -251,7 +313,11 @@
       "lane": "authority-coverage",
       "status": "pending",
       "description": "Both quota paths during reset and settlement",
-      "requires": ["reset quota-path evidence", "settlement quota-path evidence"]
+      "requires": ["reset quota-path evidence", "settlement quota-path evidence"],
+      "evidence_fields": [
+        "reset_quota_path_evidence",
+        "settlement_quota_path_evidence"
+      ]
     },
     {
       "gate": "G13",
@@ -259,7 +325,12 @@
       "lane": "maintenance-producers",
       "status": "pending",
       "description": "Quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail",
-      "requires": ["quorum-minus-one matrix", "unknown-disk/remount matrix", "Object Lock dry-run grace evidence"]
+      "requires": ["quorum-minus-one matrix", "unknown-disk/remount matrix", "Object Lock dry-run grace evidence"],
+      "evidence_fields": [
+        "quorum_minus_one_matrix",
+        "unknown_disk_remount_matrix",
+        "object_lock_dry_run_grace_evidence"
+      ]
     },
     {
       "gate": "G14",
@@ -288,7 +359,12 @@
       "lane": "scheduler-pressure",
       "status": "pending",
       "description": "Measured cold-walk share and foreground latency/throughput",
-      "requires": ["cold-walk share measurement", "foreground latency/throughput measurement"]
+      "requires": ["cold-walk share measurement", "foreground latency/throughput measurement"],
+      "evidence_fields": [
+        "cold_walk_share_measurement",
+        "foreground_latency_throughput_measurement",
+        "profile_evidence"
+      ]
     },
     {
       "gate": "P2",
@@ -308,7 +384,12 @@
       "lane": "scheduler-pressure",
       "status": "pending",
       "description": "Measured two-hour pressure/heal capacity and recovery window",
-      "requires": ["two-hour pressure measurement", "heal capacity measurement", "recovery-window measurement"]
+      "requires": ["two-hour pressure measurement", "heal capacity measurement", "recovery-window measurement"],
+      "evidence_fields": [
+        "two_hour_pressure_measurement",
+        "heal_capacity_measurement",
+        "recovery_window_measurement"
+      ]
     },
     {
       "gate": "P4",
@@ -318,6 +399,9 @@
       "description": "Measured MRF scale and replay cost with retained responsibility",
       "requires": ["MRF scale measurement", "MRF replay-cost measurement", "retained responsibility evidence", "cleanup/GC soak evidence"],
       "evidence_fields": [
+        "mrf_scale_measurement",
+        "mrf_replay_cost_measurement",
+        "retained_responsibility_evidence",
         "mrf_cleanup_gc_soak_evidence"
       ]
     },
@@ -327,7 +411,12 @@
       "lane": "checkpoint-and-crash",
       "status": "pending",
       "description": "Fixed-budget real process restart through enumeration and classification",
-      "requires": ["fixed-budget restart evidence", "enumeration evidence", "classification evidence"]
+      "requires": ["fixed-budget restart evidence", "enumeration evidence", "classification evidence"],
+      "evidence_fields": [
+        "fixed_budget_restart_evidence",
+        "enumeration_evidence",
+        "classification_evidence"
+      ]
     },
     {
       "gate": "R-D",
@@ -335,7 +424,13 @@
       "lane": "status-and-outcome",
       "status": "pending",
       "description": "Manager-to-event-to-ledger exact disposition, including grace",
-      "requires": ["manager disposition evidence", "event disposition evidence", "ledger disposition evidence", "grace handling"]
+      "requires": ["manager disposition evidence", "event disposition evidence", "ledger disposition evidence", "grace handling"],
+      "evidence_fields": [
+        "manager_disposition_evidence",
+        "event_disposition_evidence",
+        "ledger_disposition_evidence",
+        "grace_handling"
+      ]
     },
     {
       "gate": "R-L",
@@ -343,7 +438,12 @@
       "lane": "mixed-version-rollback",
       "status": "pending",
       "description": "Legacy source conflicts, migration gaps and crash-safe source retirement",
-      "requires": ["legacy source-conflict evidence", "migration-gap evidence", "crash-safe source retirement evidence"]
+      "requires": ["legacy source-conflict evidence", "migration-gap evidence", "crash-safe source retirement evidence"],
+      "evidence_fields": [
+        "legacy_source_conflict_evidence",
+        "migration_gap_evidence",
+        "crash_safe_source_retirement_evidence"
+      ]
     }
   ]
 }

.docker/compose/docker-compose.cluster.local-build.profiling-amd64.yml

@@ -43,6 +43,7 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       # `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
       # should show richer nested spans during request-path verification.
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
@@ -89,6 +90,7 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       # `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
       # should show richer nested spans during request-path verification.
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
@@ -135,6 +137,7 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       # `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
       # should show richer nested spans during request-path verification.
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
@@ -181,6 +184,7 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       # `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
       # should show richer nested spans during request-path verification.
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}

.docker/compose/docker-compose.cluster.local-build.yml

@@ -26,6 +26,7 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
       - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
       - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
@@ -72,6 +73,7 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
       - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
       - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
@@ -118,6 +120,7 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
       - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
       - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
@@ -164,6 +167,7 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
       - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
       - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}

.docker/compose/docker-compose.observability.yaml

@@ -201,6 +201,7 @@ services:
       - RUSTFS_ADDRESS=:9000
       - RUSTFS_CONSOLE_ENABLE=true
       - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:
@@ -220,6 +221,7 @@ services:
       - RUSTFS_ADDRESS=:9000
       - RUSTFS_CONSOLE_ENABLE=true
       - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:
@@ -239,6 +241,7 @@ services:
       - RUSTFS_ADDRESS=:9000
       - RUSTFS_CONSOLE_ENABLE=true
       - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:
@@ -258,6 +261,7 @@ services:
       - RUSTFS_ADDRESS=:9000
       - RUSTFS_CONSOLE_ENABLE=true
       - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:

.docker/observability/README.md

@@ -47,6 +47,10 @@ Three pre-built Grafana dashboards are included for monitoring RustFS GET perfor
 | **GET Resource Impact** | `grafana-get-resource-impact.json` | Monitors resource usage: concurrent requests, IO queue utilization, disk permit wait, RSS trend |
 | **Object Data Cache** | `grafana-object-data-cache.json` | Monitors the GET body cache (`rustfs_object_data_cache_*`): hit ratio, lookup/plan/fill outcomes, fill duration quantiles, hit vs fill throughput, entries/weighted bytes, inflight fills, memory-pressure skips, invalidations, and size-class breakdowns |
 
+### Storage Metrics
+
+Storage panels require `prometheus-rules/rustfs-storage.yml` and the cluster resource attribute. See the [storage metrics guide](../../docs/operations/storage-metrics.md) for ownership, observer selection, freshness, and rolling upgrades.
+
 ### Prometheus Alert Rules
 
 The file `prometheus-rules/rustfs-get-optimization-alerts.yaml` contains pre-configured alerting rules:

.docker/observability/docker-compose-example-for-rustfs.yml

@@ -32,6 +32,7 @@ services:
       - RUSTFS_SECRET_KEY=rustfsadmin
       - RUSTFS_OBS_LOGGER_LEVEL=info
       - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_PROFILING_ENDPOINT=http://pyroscope:4040
     volumes:
       - rustfs-data:/data/rustfs

.docker/observability/grafana/dashboards/rustfs.json

@@ -223,14 +223,15 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_buckets_total{job=~\"$job\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_buckets_total\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
         }
       ],
       "title": "Total Buckets",
-      "type": "stat"
+      "type": "stat",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -289,14 +290,15 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_objects_total{job=~\"$job\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_objects_total\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
         }
       ],
       "title": "Total Objects",
-      "type": "stat"
+      "type": "stat",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -427,7 +429,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_capacity_used_bytes{job=~\"$job\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_used_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "legendFormat": "Used",
           "range": true,
           "refId": "A"
@@ -438,7 +440,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_raw_total_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "hide": false,
           "legendFormat": "Total",
           "range": true,
@@ -450,7 +452,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_capacity_used_bytes{job=~\"$job\"}) / sum(rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_used_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"} / ignoring(source_metric) rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_raw_total_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "hide": false,
           "instant": false,
           "legendFormat": "Percent",
@@ -459,7 +461,8 @@
         }
       ],
       "title": "Capacity",
-      "type": "stat"
+      "type": "stat",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -525,7 +528,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_capacity_stale_drives{job=~\"$job\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_stale_drives\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "legendFormat": "Stale Drives",
           "range": true,
           "refId": "A"
@@ -536,14 +539,15 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_capacity_missing_drives{job=~\"$job\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_missing_drives\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "legendFormat": "Missing Drives",
           "range": true,
           "refId": "B"
         }
       ],
       "title": "Capacity Observation",
-      "type": "stat"
+      "type": "stat",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -1989,8 +1993,8 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (drive) (rustfs_system_drive_used_bytes{job=~\"$job\", drive=~\"$drive\"})",
-          "legendFormat": "{{drive}} (bytes)",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}",
+          "legendFormat": "{{server}} | {{drive}} (bytes)",
           "range": true,
           "refId": "A"
         },
@@ -2000,16 +2004,17 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (drive) (rustfs_system_drive_used_bytes{job=~\"$job\", drive=~\"$drive\"}) / sum by (drive)(rustfs_system_drive_total_bytes{job=~\"$job\", drive=~\"$drive\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"} / ignoring(source_metric) rustfs:storage:current{source_metric=\"rustfs_system_drive_total_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}",
           "hide": false,
           "instant": false,
-          "legendFormat": "{{drive}} (percent)",
+          "legendFormat": "{{server}} | {{drive}} (percent)",
           "range": true,
           "refId": "B"
         }
       ],
       "title": "System Drive Usage",
-      "type": "timeseries"
+      "type": "timeseries",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -2097,14 +2102,15 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "max by (drive) (rustfs_system_drive_capacity_observation_age_seconds{job=~\"$job\", drive=~\"$drive\"})",
-          "legendFormat": "{{drive}}",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_capacity_observation_age_seconds\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}",
+          "legendFormat": "{{server}} | {{drive}}",
           "range": true,
           "refId": "A"
         }
       ],
       "title": "Drive Capacity Observation Age",
-      "type": "timeseries"
+      "type": "timeseries",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -2190,8 +2196,8 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "max by (drive) (rustfs_system_drive_capacity_observation_state{job=~\"$job\", drive=~\"$drive\", state=\"stale\"})",
-          "legendFormat": "{{drive}} stale",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_capacity_observation_state\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",state=\"stale\"}",
+          "legendFormat": "{{server}} | {{drive}} stale",
           "range": true,
           "refId": "A"
         },
@@ -2201,14 +2207,15 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "max by (drive) (rustfs_system_drive_capacity_observation_state{job=~\"$job\", drive=~\"$drive\", state=\"missing\"})",
-          "legendFormat": "{{drive}} missing",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_capacity_observation_state\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",state=\"missing\"}",
+          "legendFormat": "{{server}} | {{drive}} missing",
           "range": true,
           "refId": "B"
         }
       ],
       "title": "Drive Capacity Observation State",
-      "type": "timeseries"
+      "type": "timeseries",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -4551,7 +4558,7 @@
                   "index": 0,
                   "text": "INACTIVE"
                 },
-                "to": 1e-9
+                "to": 1e-09
               },
               "type": "range"
             }
@@ -6563,7 +6570,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs_cluster_health_drives_online_count{job=~\"$job\"}",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_health_drives_online_count\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "legendFormat": "online - {{job}}",
           "range": true,
           "refId": "A"
@@ -6574,7 +6581,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs_cluster_health_drives_offline_count{job=~\"$job\"}",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_health_drives_offline_count\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "legendFormat": "offline - {{job}}",
           "range": true,
           "refId": "B"
@@ -6585,14 +6592,15 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs_cluster_health_drives_count{job=~\"$job\"}",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_health_drives_count\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "legendFormat": "total - {{job}}",
           "range": true,
           "refId": "C"
         }
       ],
       "title": "Cluster Drive Health Counts",
-      "type": "timeseries"
+      "type": "timeseries",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "collapsed": false,
@@ -8554,14 +8562,15 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "{__name__=~\"rustfs_system_drive_.*\",job=~\"$job\",drive=~\"$drive\"}",
-          "legendFormat": "{{__name__}} | {{drive}}",
+          "expr": "rustfs:storage:current{source_metric=~\"rustfs_system_drive_.*\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}",
+          "legendFormat": "{{server}} | {{source_metric}} | {{drive}}",
           "range": true,
           "refId": "A"
         }
       ],
       "title": "System Drive (All)",
-      "type": "timeseries"
+      "type": "timeseries",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -8942,14 +8951,15 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "{__name__=~\"rustfs_cluster_erasure_set_.*\",job=~\"$job\"}",
-          "legendFormat": "{{__name__}}",
+          "expr": "rustfs:storage:current{source_metric=~\"rustfs_cluster_erasure_set_.*\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
+          "legendFormat": "{{source_metric}}",
           "range": true,
           "refId": "A"
         }
       ],
       "title": "Cluster Erasure Set (All)",
-      "type": "timeseries"
+      "type": "timeseries",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -11716,7 +11726,7 @@
           "editorMode": "code",
           "range": true,
           "refId": "A",
-          "expr": "max by (server, drive, pool_index, set_index, drive_index, state) (rustfs_system_drive_runtime_state{job=~\"$job\",server=~\"$server\",drive=~\"$drive\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_drive_runtime_state\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\",drive=~\"$drive\"}",
           "legendFormat": "{{server}} | {{drive}} | p{{pool_index}}/s{{set_index}}/d{{drive_index}} | {{state}}"
         },
         {
@@ -11727,12 +11737,13 @@
           "editorMode": "code",
           "range": true,
           "refId": "B",
-          "expr": "max by (server, drive, pool_index, set_index, drive_index) (rustfs_system_drive_offline_duration_seconds{job=~\"$job\",server=~\"$server\",drive=~\"$drive\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_drive_offline_duration_seconds\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\",drive=~\"$drive\"}",
           "legendFormat": "{{server}} | {{drive}} | offline seconds"
         }
       ],
-      "title": "Drive Runtime State and Offline Duration",
-      "type": "timeseries"
+      "title": "Observed Cluster Drive State and Offline Duration",
+      "type": "timeseries",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -11825,12 +11836,13 @@
           "editorMode": "code",
           "range": true,
           "refId": "A",
-          "expr": "sum by (server, drive, pool_index, set_index, drive_index, api) (rate(rustfs_system_drive_api_calls_total{job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}[$__rate_interval]))",
+          "expr": "rate(rustfs_system_drive_api_calls_total{collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}[$__rate_interval]) and ignoring(source_metric) rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}",
           "legendFormat": "{{server}} | {{drive}} | p{{pool_index}}/s{{set_index}}/d{{drive_index}} | {{api}}"
         }
       ],
       "title": "Drive API Calls by Operation",
-      "type": "timeseries"
+      "type": "timeseries",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -12376,6 +12388,50 @@
         "sort": 1,
         "type": "query"
       },
+      {
+        "current": {},
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${datasource}"
+        },
+        "definition": "label_values(rustfs:storage_snapshot:fresh, rustfs_cluster_id)",
+        "includeAll": false,
+        "label": "Storage cluster",
+        "multi": false,
+        "name": "storage_cluster",
+        "options": [],
+        "query": {
+          "qryType": 1,
+          "query": "label_values(rustfs:storage_snapshot:fresh, rustfs_cluster_id)",
+          "refId": "PrometheusVariableQueryEditor-storage_cluster"
+        },
+        "refresh": 2,
+        "regex": "",
+        "sort": 1,
+        "type": "query"
+      },
+      {
+        "current": {},
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${datasource}"
+        },
+        "definition": "query_result(rustfs:storage_snapshot:fresh{collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\"})",
+        "includeAll": false,
+        "label": "Cluster observer",
+        "multi": false,
+        "name": "storage_observer",
+        "options": [],
+        "query": {
+          "qryType": 3,
+          "query": "query_result(rustfs:storage_snapshot:fresh{collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\"})",
+          "refId": "PrometheusVariableQueryEditor-storage_observer"
+        },
+        "refresh": 2,
+        "regex": "/observer=\"([^\"]+)\"/",
+        "sort": 1,
+        "type": "query"
+      },
       {
         "allValue": ".*",
         "current": {
@@ -12386,7 +12442,7 @@
           "type": "prometheus",
           "uid": "${datasource}"
         },
-        "definition": "label_values(rustfs_system_drive_api_calls_total,api)",
+        "definition": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, api)",
         "includeAll": true,
         "label": "Drive API",
         "multi": true,
@@ -12394,7 +12450,7 @@
         "options": [],
         "query": {
           "qryType": 1,
-          "query": "label_values(rustfs_system_drive_api_calls_total,api)",
+          "query": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, api)",
           "refId": "PrometheusVariableQueryEditor-drive_api"
         },
         "refresh": 2,
@@ -12455,7 +12511,7 @@
           "text": "All",
           "value": "$__all"
         },
-        "definition": "label_values(rustfs_system_drive_used_bytes,drive)",
+        "definition": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, drive)",
         "includeAll": true,
         "label": "Drive",
         "multi": true,
@@ -12463,7 +12519,7 @@
         "options": [],
         "query": {
           "qryType": 1,
-          "query": "label_values(rustfs_system_drive_used_bytes,drive)",
+          "query": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, drive)",
           "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 2,

.docker/observability/prometheus-rules/rustfs-storage.yml

@@ -0,0 +1,575 @@
+# Preserve OTLP timestamps in the Collector. Apply timestamp() directly to
+# each raw selector, before label rewriting; functions such as label_replace()
+# replace the evaluation timestamp and would make a cached value look new.
+# Compare publication times at Prometheus millisecond precision so points
+# published and exported within the same millisecond are not withheld.
+groups:
+  - name: rustfs-storage-snapshots
+    interval: 15s
+    rules:
+      - record: rustfs:storage_snapshot:fresh
+        expr: |
+          rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"}
+          and ((time() - rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"}) <= rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"})
+          and ((time() - rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"}) >= 0)
+          and (rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"} > 0)
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_buckets_total
+        expr: |
+          rustfs_cluster_buckets_total{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_buckets_total{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_capacity_free_bytes
+        expr: |
+          rustfs_cluster_capacity_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_capacity_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_capacity_missing_drives
+        expr: |
+          rustfs_cluster_capacity_missing_drives{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_capacity_missing_drives{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_capacity_raw_total_bytes
+        expr: |
+          rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_capacity_stale_drives
+        expr: |
+          rustfs_cluster_capacity_stale_drives{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_capacity_stale_drives{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_capacity_usable_total_bytes
+        expr: |
+          rustfs_cluster_capacity_usable_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_capacity_usable_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_capacity_used_bytes
+        expr: |
+          rustfs_cluster_capacity_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_capacity_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_drive_capacity_observation_age_seconds
+        expr: |
+          rustfs_cluster_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_drive_capacity_observation_state
+        expr: |
+          rustfs_cluster_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_drive_free_bytes
+        expr: |
+          rustfs_cluster_drive_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_drive_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_drive_offline_duration_seconds
+        expr: |
+          rustfs_cluster_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_drive_present
+        expr: |
+          rustfs_cluster_drive_present{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_drive_present{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_drive_runtime_state
+        expr: |
+          rustfs_cluster_drive_runtime_state{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_drive_runtime_state{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_drive_total_bytes
+        expr: |
+          rustfs_cluster_drive_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_drive_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_drive_used_bytes
+        expr: |
+          rustfs_cluster_drive_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_drive_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_data_shards
+        expr: |
+          rustfs_cluster_erasure_set_data_shards{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_data_shards{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_healing_drives_count
+        expr: |
+          rustfs_cluster_erasure_set_healing_drives_count{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_healing_drives_count{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_health
+        expr: |
+          rustfs_cluster_erasure_set_health{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_health{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_online_drives_count
+        expr: |
+          rustfs_cluster_erasure_set_online_drives_count{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_online_drives_count{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_overall_health
+        expr: |
+          rustfs_cluster_erasure_set_overall_health{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_overall_health{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_overall_write_quorum
+        expr: |
+          rustfs_cluster_erasure_set_overall_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_overall_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_parity
+        expr: |
+          rustfs_cluster_erasure_set_parity{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_parity{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_read_health
+        expr: |
+          rustfs_cluster_erasure_set_read_health{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_read_health{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_read_quorum
+        expr: |
+          rustfs_cluster_erasure_set_read_quorum{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_read_quorum{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_read_tolerance
+        expr: |
+          rustfs_cluster_erasure_set_read_tolerance{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_read_tolerance{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_size
+        expr: |
+          rustfs_cluster_erasure_set_size{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_size{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_write_health
+        expr: |
+          rustfs_cluster_erasure_set_write_health{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_write_health{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_write_quorum
+        expr: |
+          rustfs_cluster_erasure_set_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_write_tolerance
+        expr: |
+          rustfs_cluster_erasure_set_write_tolerance{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_write_tolerance{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_health_drives_count
+        expr: |
+          rustfs_cluster_health_drives_count{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_health_drives_count{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_health_drives_offline_count
+        expr: |
+          rustfs_cluster_health_drives_offline_count{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_health_drives_offline_count{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_health_drives_online_count
+        expr: |
+          rustfs_cluster_health_drives_online_count{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_health_drives_online_count{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_objects_total
+        expr: |
+          rustfs_cluster_objects_total{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_objects_total{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_node_disk_free_bytes
+        expr: |
+          rustfs_node_disk_free_bytes{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_node_disk_free_bytes{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_node_disk_total_bytes
+        expr: |
+          rustfs_node_disk_total_bytes{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_node_disk_total_bytes{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_node_disk_used_bytes
+        expr: |
+          rustfs_node_disk_used_bytes{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_node_disk_used_bytes{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_api_calls_total
+        expr: |
+          rustfs_system_drive_api_calls_total{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_api_calls_total{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_api_latency_by_api_micros
+        expr: |
+          rustfs_system_drive_api_latency_by_api_micros{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_api_latency_by_api_micros{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_api_latency_micros
+        expr: |
+          rustfs_system_drive_api_latency_micros{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_api_latency_micros{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_availability_errors_total
+        expr: |
+          rustfs_system_drive_availability_errors_total{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_availability_errors_total{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_capacity_observation_age_seconds
+        expr: |
+          rustfs_system_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_capacity_observation_state
+        expr: |
+          rustfs_system_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_count
+        expr: |
+          rustfs_system_drive_count{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_count{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_deletes_total
+        expr: |
+          rustfs_system_drive_deletes_total{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_deletes_total{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_free_bytes
+        expr: |
+          rustfs_system_drive_free_bytes{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_free_bytes{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_free_inodes
+        expr: |
+          rustfs_system_drive_free_inodes{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_free_inodes{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_healing
+        expr: |
+          rustfs_system_drive_healing{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_healing{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_health
+        expr: |
+          rustfs_system_drive_health{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_health{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_info
+        expr: |
+          rustfs_system_drive_info{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_info{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_io_errors_total
+        expr: |
+          rustfs_system_drive_io_errors_total{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_io_errors_total{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_offline_count
+        expr: |
+          rustfs_system_drive_offline_count{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_offline_count{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_offline_duration_seconds
+        expr: |
+          rustfs_system_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_online_count
+        expr: |
+          rustfs_system_drive_online_count{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_online_count{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_perc_util
+        expr: |
+          rustfs_system_drive_perc_util{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_perc_util{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_present
+        expr: |
+          rustfs_system_drive_present{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_present{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_reads_await
+        expr: |
+          rustfs_system_drive_reads_await{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_reads_await{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_reads_kb_per_sec
+        expr: |
+          rustfs_system_drive_reads_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_reads_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_reads_per_sec
+        expr: |
+          rustfs_system_drive_reads_per_sec{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_reads_per_sec{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_runtime_state
+        expr: |
+          rustfs_system_drive_runtime_state{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_runtime_state{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_scanning
+        expr: |
+          rustfs_system_drive_scanning{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_scanning{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_timeout_errors_total
+        expr: |
+          rustfs_system_drive_timeout_errors_total{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_timeout_errors_total{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_total_bytes
+        expr: |
+          rustfs_system_drive_total_bytes{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_total_bytes{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_total_inodes
+        expr: |
+          rustfs_system_drive_total_inodes{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_total_inodes{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_used_bytes
+        expr: |
+          rustfs_system_drive_used_bytes{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_used_bytes{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_used_inodes
+        expr: |
+          rustfs_system_drive_used_inodes{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_used_inodes{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_waiting_io
+        expr: |
+          rustfs_system_drive_waiting_io{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_waiting_io{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_writes_await
+        expr: |
+          rustfs_system_drive_writes_await{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_writes_await{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_writes_kb_per_sec
+        expr: |
+          rustfs_system_drive_writes_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_writes_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_writes_per_sec
+        expr: |
+          rustfs_system_drive_writes_per_sec{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_writes_per_sec{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_writes_total
+        expr: |
+          rustfs_system_drive_writes_total{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_writes_total{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))

.docker/observability/tests/storage-rules.test.yml

@@ -0,0 +1,189 @@
+rule_files:
+- ../prometheus-rules/rustfs-storage.yml
+evaluation_interval: 15s
+tests:
+- name: owners, pools, clusters and observer views remain distinct in mixed versions
+  interval: 1m
+  input_series:
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",pool_index="0",set_index="0",drive_index="0"}
+    values: 100x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data",pool_index="0",set_index="0",drive_index="1"}
+    values: 100x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n2",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n2",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n2",collection_scope="local",job="rustfs",instance="collector",server="n2",drive="/data",pool_index="1",set_index="0",drive_index="0"}
+    values: 100x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n3",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n3",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n3",collection_scope="local",job="rustfs",instance="collector",server="n3",drive="/data",pool_index="1",set_index="0",drive_index="1"}
+    values: 100x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",pool_index="0",set_index="0",drive_index="0"}
+    values: 1000x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data",pool_index="0",set_index="0",drive_index="1"}
+    values: 1000x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n2",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n2",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n2",collection_scope="local",job="rustfs",instance="collector",server="n2",drive="/data",pool_index="1",set_index="0",drive_index="0"}
+    values: 1000x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n3",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n3",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n3",collection_scope="local",job="rustfs",instance="collector",server="n3",drive="/data",pool_index="1",set_index="0",drive_index="1"}
+    values: 1000x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="cluster",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="cluster",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id="a",observer="n0",collection_scope="cluster",job="rustfs",instance="collector"}
+    values: 400x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="cluster",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="cluster",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id="a",observer="n1",collection_scope="cluster",job="rustfs",instance="collector"}
+    values: 300x8
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="old",server="n0",drive="/data",job="rustfs",instance="collector"}
+    values: 999x8
+  promql_expr_test:
+  - expr: sum by (rustfs_cluster_id) (rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes",collection_scope="local"})
+    eval_time: 2m
+    exp_samples:
+    - labels: '{rustfs_cluster_id="a"}'
+      value: 400
+    - labels: '{rustfs_cluster_id="b"}'
+      value: 4000
+  - expr: count by (rustfs_cluster_id,pool_index) (rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes"})
+    eval_time: 2m
+    exp_samples:
+    - labels: '{rustfs_cluster_id="a",pool_index="0"}'
+      value: 2
+    - labels: '{rustfs_cluster_id="a",pool_index="1"}'
+      value: 2
+    - labels: '{rustfs_cluster_id="b",pool_index="0"}'
+      value: 2
+    - labels: '{rustfs_cluster_id="b",pool_index="1"}'
+      value: 2
+  - expr: sum(rustfs:storage:current{source_metric="rustfs_cluster_capacity_raw_total_bytes",rustfs_cluster_id="a",observer="n0"})
+    eval_time: 2m
+    exp_samples:
+    - labels: '{}'
+      value: 400
+  - expr: sum(rustfs:storage:current{source_metric="rustfs_cluster_capacity_raw_total_bytes",rustfs_cluster_id="a",observer="n1"})
+    eval_time: 2m
+    exp_samples:
+    - labels: '{}'
+      value: 300
+- name: removed values and identities cannot rejoin new snapshots; stalled sources expire
+  interval: 1m
+  input_series:
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0 60 _ _ _ _ _ _ _
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_info{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",disk_id="old"}
+    values: 1 1 _ _ _ _ _ _ _
+  - series: rustfs_system_drive_info{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",disk_id="new"}
+    values: _ _ 1 1 1 1 1 1 1
+  - series: rustfs_system_drive_waiting_io{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data"}
+    values: 7 7 _ _ _ _ _ _ _
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data"}
+    values: 100 100 _ _ _ _ _ _ _
+  promql_expr_test:
+  - expr: rustfs:storage:current{source_metric="rustfs_system_drive_info",disk_id="old"}
+    eval_time: 2m
+    exp_samples: []
+  - expr: rustfs:storage:current{source_metric="rustfs_system_drive_waiting_io"}
+    eval_time: 2m
+    exp_samples: []
+  - expr: count(rustfs:storage:current{source_metric="rustfs_system_drive_info",disk_id="new"})
+    eval_time: 2m
+    exp_samples:
+    - labels: '{}'
+      value: 1
+  - expr: rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes"}
+    eval_time: 5m
+    exp_samples: []
+  - expr: rustfs:storage_snapshot:fresh{observer="n1"}
+    eval_time: 5m
+    exp_samples: []
+- name: counters reset independently; filter versions before rate and sum rates across owners
+  interval: 1m
+  input_series:
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_api_calls_total{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",disk_id="n0-disk",api="read_all"}
+    values: 0 60 120 30 90 150 210
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_api_calls_total{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data",disk_id="n1-disk",api="read_all"}
+    values: 0+120x6
+  - series: rustfs_system_drive_api_calls_total{rustfs_cluster_id="a",server="n0",drive="/data",observer="old"}
+    values: 0+999x6
+  promql_expr_test:
+  - expr: sum(resets(rustfs_system_drive_api_calls_total{collection_scope="local"}[5m]))
+    eval_time: 5m
+    exp_samples:
+    - labels: '{}'
+      value: 1
+  - expr: sum(rate(rustfs_system_drive_api_calls_total{collection_scope="local"}[2m]) and ignoring(source_metric) rustfs:storage:current{source_metric="rustfs_system_drive_api_calls_total"})
+    eval_time: 5m
+    exp_samples:
+    - labels: '{}'
+      value: 3
+- name: completed slow collection has no validity budget
+  interval: 1m
+  input_series:
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="slow",collection_scope="cluster",job="rustfs",instance="collector"}
+    values: 0+60x3
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="slow",collection_scope="cluster",job="rustfs",instance="collector"}
+    values: '0x3'
+  promql_expr_test:
+  - expr: rustfs:storage_snapshot:fresh{observer="slow"}
+    eval_time: 2m
+    exp_samples: []
+- name: submillisecond publication and millisecond OTLP samples share a cutoff
+  interval: 1m
+  input_series:
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="submillisecond",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0.0009+60x3
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="submillisecond",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x3
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="submillisecond",collection_scope="local",job="rustfs",instance="collector",server="submillisecond",drive="/data"}
+    values: 100x3
+  promql_expr_test:
+  - expr: count(rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes",observer="submillisecond"})
+    eval_time: 2m15s
+    exp_samples:
+    - labels: '{}'
+      value: 1

.github/actions/quick-checks/action.yml

@@ -111,10 +111,6 @@ runs:
       shell: bash
       run: ./scripts/check_no_planning_docs.sh
 
-    - name: Check CI paths stay in sync
-      shell: bash
-      run: ./scripts/check_ci_paths_sync.sh
-
     - name: Check io_uring lane --lib precondition
       shell: bash
       run: ./scripts/check_uring_lane_lib_only.sh

.github/s3tests/s3tests.conf

@@ -55,6 +55,10 @@ secret_key = ${S3_SECRET_KEY}
 
 ## replace with key id obtained when secret is created, or delete if KMS not tested
 #kms_keyid = 01234567-89ab-cdef-0123-456789abcdef
+#kms_keyid2 = fedcba98-7654-3210-fedc-ba9876543210
+
+## Expected service default for SSE-KMS requests without a key id; empty means none
+#kms_default_keyid =
 
 ## Storage classes
 #storage_classes = "LUKEWARM, FROZEN"

.github/workflows/ci-docs-only.yml

@@ -1,79 +0,0 @@
-# Copyright 2026 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Reports the existing required checks for paths excluded by ci.yml.
-# Mixed PRs can trigger both workflows; their Quick Checks jobs use one shared
-# action to keep validation coverage aligned. Keep this paths list in sync with
-# ci.yml's pull_request.paths-ignore via scripts/check_ci_paths_sync.sh.
-
-name: Continuous Integration (docs only)
-
-on:
-  pull_request:
-    types: [ opened, synchronize, reopened ]
-    branches: [ main, release ]
-    paths:
-      - "**.md"
-      - "docs/**"
-      - "deploy/**"
-      - "scripts/dev_*.sh"
-      - "scripts/probe.sh"
-      - "LICENSE*"
-      - ".gitignore"
-      - ".dockerignore"
-      - "README*"
-      - "**/*.png"
-      - "**/*.jpg"
-      - "**/*.svg"
-      - ".github/workflows/build.yml"
-      - ".github/workflows/docker.yml"
-      - ".github/workflows/audit.yml"
-      - "flake.lock"
-
-permissions:
-  contents: read
-
-jobs:
-  quick-checks:
-    name: Quick Checks
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Run shared quick checks
-        uses: ./.github/actions/quick-checks
-
-  test-and-lint:
-    name: Test and Lint
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      # Docs-only PRs skip the full code CI, but they are exactly where a
-      # planning-type document could be slipped in (git add -f bypasses
-      # .gitignore). Run the guard here so the required "Test and Lint" check
-      # stays meaningful for docs-only changes.
-      - name: Check no planning docs committed
-        run: ./scripts/check_no_planning_docs.sh
-
-      - name: Satisfy required check for docs-only changes
-        run: echo "Docs-only change — code CI is skipped by paths-ignore; planning-docs guard passed, reporting success for the required 'Test and Lint' check."

.github/workflows/ci.yml

@@ -37,25 +37,6 @@ on:
   pull_request:
     types: [ opened, synchronize, reopened, closed ]
     branches: [ main, release ]
-    # Keep this list in sync with the `paths` list in ci-docs-only.yml, which
-    # reports the required "Test and Lint" check for PRs skipped here.
-    paths-ignore:
-      - "**.md"
-      - "docs/**"
-      - "deploy/**"
-      - "scripts/dev_*.sh"
-      - "scripts/probe.sh"
-      - "LICENSE*"
-      - ".gitignore"
-      - ".dockerignore"
-      - "README*"
-      - "**/*.png"
-      - "**/*.jpg"
-      - "**/*.svg"
-      - ".github/workflows/build.yml"
-      - ".github/workflows/docker.yml"
-      - ".github/workflows/audit.yml"
-      - "flake.lock"
   merge_group:
     types: [ checks_requested ]
   schedule:
@@ -88,6 +69,32 @@ jobs:
       - name: Explain cancellation run
         run: echo "PR closed; this run only cancels older runs in the same concurrency group."
 
+  classify-changes:
+    name: Select CI scope
+    if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    outputs:
+      mode: ${{ steps.scope.outputs.mode }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          fetch-depth: 2
+          persist-credentials: false
+      - name: Select scope using the base revision's policy
+        id: scope
+        env:
+          CI_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          if [[ "$GITHUB_EVENT_NAME" != "pull_request" ]]; then
+            printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
+          elif [[ "$CI_BASE_SHA" =~ ^[0-9a-f]{40}$ ]] && git show "$CI_BASE_SHA:scripts/ci_gate.py" > "$RUNNER_TEMP/ci-gate-base.py"; then
+            python3 -I "$RUNNER_TEMP/ci-gate-base.py" select
+          else
+            printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
+            echo "Base CI policy unavailable; running the full matrix."
+          fi
+
   typos:
     name: Typos
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
@@ -100,7 +107,7 @@ jobs:
       - name: Typos check with custom config file
         uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master
 
-  # Fail early with compile-free checks shared with docs-only CI.
+  # Fail early with compile-free checks for every pull request.
   quick-checks:
     name: Quick Checks
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
@@ -116,9 +123,9 @@ jobs:
         uses: ./.github/actions/quick-checks
 
   test-and-lint:
-    name: Test and Lint
-    if: github.event_name != 'pull_request' || github.event.action != 'closed'
-    needs: [ quick-checks ]
+    name: Workspace Test and Lint
+    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
     env:
@@ -289,45 +296,6 @@ jobs:
       - name: Run rebalance/decommission migration proofs
         run: ./scripts/check_migration_gate_count.sh
 
-      # Record the reason before this job completes as FAILURE. A separate
-      # dependent job cancels sibling lanes only after GitHub has preserved this
-      # required check's failure verdict.
-      - name: Annotate early-stop reason
-        if: >-
-          failure() && github.event_name == 'pull_request'
-          && github.event.pull_request.head.repo.full_name == github.repository
-        run: |
-          {
-            echo "## CI early-stop"
-            echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; a follow-up job will cancel sibling lanes to free runners."
-            echo "Sibling jobs showing **cancelled** were stopped by the early-stop follow-up, not by their own failure."
-          } >> "$GITHUB_STEP_SUMMARY"
-
-  # Preserve the required Test and Lint FAILURE verdict before stopping sibling
-  # lanes. Cancelling from inside test-and-lint changed its own conclusion to
-  # CANCELLED and hid the actionable failure in the PR checks UI.
-  cancel-after-test-and-lint-failure:
-    name: Cancel siblings after Test and Lint failure
-    if: >-
-      failure() && needs.test-and-lint.result == 'failure'
-      && github.event_name == 'pull_request'
-      && github.event.pull_request.head.repo.full_name == github.repository
-    needs: [ test-and-lint ]
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    permissions:
-      actions: write
-    steps:
-      - name: Cancel remaining jobs
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          curl -fsS -X POST \
-            -H "Authorization: Bearer ${GH_TOKEN}" \
-            -H "Accept: application/vnd.github+json" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel"
-
   # Dedicated serial lane for the ILM / lifecycle integration tests. These tests
   # drive the object layer through process-global singletons (the GLOBAL_ENV
   # ECStore, the global tier-config manager, background-expiry workers) and bind
@@ -340,8 +308,8 @@ jobs:
   # See rustfs/backlog#1148 (ilm-1) and #1155.
   test-ilm-integration-serial:
     name: ILM Integration (serial)
-    if: github.event_name != 'pull_request' || github.event.action != 'closed'
-    needs: [ quick-checks ]
+    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
     env:
@@ -408,8 +376,8 @@ jobs:
 
   test-and-lint-rio-v2:
     name: Test and Lint (rio-v2)
-    if: github.event_name != 'pull_request' || github.event.action != 'closed'
-    needs: [ quick-checks ]
+    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
     env:
@@ -449,8 +417,8 @@ jobs:
 
   connect-short-credential-boundary:
     name: Connect Short Credential Boundary
-    if: github.event_name != 'pull_request' || github.event.action != 'closed'
-    needs: [ quick-checks ]
+    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 60
     env:
@@ -507,8 +475,8 @@ jobs:
 
   test-and-lint-protocols:
     name: "Test and Lint (${{ matrix.features.name }})"
-    if: github.event_name != 'pull_request' || github.event.action != 'closed'
-    needs: [ quick-checks ]
+    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
     strategy:
@@ -561,8 +529,8 @@ jobs:
 
   build-rustfs-debug-binary:
     name: Build RustFS Debug Binary
-    if: github.event_name != 'pull_request' || github.event.action != 'closed'
-    needs: [ quick-checks ]
+    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 30
     env:
@@ -684,8 +652,8 @@ jobs:
     # job had neither, so each closed/merged PR really ran the whole io_uring
     # suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
     # 30662728539) and kept the cancellation run in progress for minutes.
-    if: github.event_name != 'pull_request' || github.event.action != 'closed'
-    needs: [ quick-checks ]
+    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    needs: [ quick-checks, classify-changes ]
     # GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
     # a container, applies no seccomp filter that would block io_uring_setup — so
     # the probe succeeds and the tests exercise the real UringBackend/FdCache/
@@ -1213,9 +1181,44 @@ jobs:
           if-no-files-found: ignore
           retention-days: 3
 
+  required-checks:
+    name: Test and Lint
+    if: always() && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    needs:
+      - classify-changes
+      - typos
+      - quick-checks
+      - test-and-lint
+      - test-ilm-integration-serial
+      - test-and-lint-rio-v2
+      - connect-short-credential-boundary
+      - test-and-lint-protocols
+      - build-rustfs-debug-binary
+      - uring-integration
+      - e2e-tests
+      - s3-implemented-tests
+      - s3-lifecycle-behavior-tests
+      - build-rustfs-debug-binary-rio-v2
+      - e2e-tests-rio-v2
+      - e2e-full
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Require the expected result of every CI lane
+        env:
+          CI_NEEDS: ${{ toJSON(needs) }}
+        shell: bash
+        run: python3 scripts/ci_gate.py verify
+
   alert-on-failure:
     name: Alert on scheduled failure
     needs:
+      - classify-changes
+      - connect-short-credential-boundary
+      - required-checks
       - typos
       - quick-checks
       - test-and-lint

.github/workflows/nightly-gnu.yml

@@ -19,17 +19,27 @@ on:
     - cron: "7 0 * * *"
       timezone: "Asia/Shanghai"
   workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Branch/ref to build and publish as the nightly (empty = scheduled source, see NIGHTLY_BUILD_REF)'
+        required: false
+        default: ''
 
 permissions:
   contents: read
 
+# Scheduled builds follow the NIGHTLY_BRANCH repo variable so the channel can
+# be pointed at e.g. `release` for the GA cycle and back to `main` afterwards
+# without touching this file. Manual runs take the `branch` input, falling
+# back to the branch the run was dispatched from.
 concurrency:
-  group: nightly-gnu-build-main-${{ github.event_name }}
+  group: nightly-gnu-build-${{ github.event_name }}-${{ github.event_name == 'schedule' && (vars.NIGHTLY_BRANCH || 'main') || (inputs.branch || github.ref_name) }}
   cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' }}
 
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: 1
+  NIGHTLY_BUILD_REF: ${{ github.event_name == 'schedule' && (vars.NIGHTLY_BRANCH || 'main') || (inputs.branch || github.ref_name) }}
 
 jobs:
   build:
@@ -43,6 +53,7 @@ jobs:
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
+          ref: ${{ env.NIGHTLY_BUILD_REF }}
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -152,13 +163,104 @@ jobs:
 
           fakeroot dpkg-deb --build "${PKG_DIR}"
           ls -lh "${DEB_FILE}"
+          echo "deb_date=${DEB_DATE}" >> "${GITHUB_OUTPUT}"
           echo "deb_file=${DEB_FILE}" >> "${GITHUB_OUTPUT}"
 
+      # Same packaging scheme as .github/workflows/package.yml (fpm), but from
+      # the locally built nightly binary instead of a release artifact, with a
+      # date-based version that mirrors the DEB.
+      - name: Build RPM package
+        id: rpm
+        shell: bash
+        env:
+          DEB_DATE: ${{ steps.deb.outputs.deb_date }}
+        run: |
+          set -euo pipefail
+
+          if ! command -v fpm >/dev/null 2>&1; then
+            SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+            ${SUDO} apt-get update -qq && ${SUDO} apt-get install -y -qq ruby ruby-dev build-essential rpm >/dev/null
+            ${SUDO} gem install fpm --no-document >/dev/null
+          fi
+
+          RPM_FILE="rustfs-nightly-${DEB_DATE}.rpm"
+          RPM_VERSION="0"
+          RPM_RELEASE="0.nightly.${DEB_DATE//-/.}"
+
+          echo "Building RPM: ${RPM_FILE} (version ${RPM_VERSION}-${RPM_RELEASE})"
+
+          # fpm wants the config file to exist before packaging.
+          mkdir -p ./tmp-pkg/etc/default
+          cat > ./tmp-pkg/etc/default/rustfs << 'ENVEOF'
+          # RustFS Environment Configuration
+          # See https://rustfs.com/docs/ for more information
+          # RUSTFS_VOLUMES=""
+          # RUSTFS_ROOT_USER=""
+          # RUSTFS_ROOT_PASSWORD=""
+          ENVEOF
+
+          fpm -s dir -t rpm \
+            --name rustfs \
+            --version "$RPM_VERSION" \
+            --iteration "$RPM_RELEASE" \
+            --architecture x86_64 \
+            --package "$RPM_FILE" \
+            --depends "glibc >= 2.31" \
+            --maintainer "RustFS Team <support@rustfs.com>" \
+            --description "High-performance distributed object storage" \
+            --url "https://rustfs.com" \
+            --license "Apache-2.0" \
+            --after-install <(cat << 'POSTINST'
+          #!/bin/bash
+          set -e
+          if ! getent passwd rustfs > /dev/null 2>&1; then
+              useradd -r -s /bin/false -d /opt/rustfs rustfs
+          fi
+          mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
+          chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
+          if [ -d /run/systemd/system ]; then
+              systemctl daemon-reload
+          fi
+          POSTINST
+            ) \
+            --before-remove <(cat << 'PRERM'
+          #!/bin/bash
+          set -e
+          if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
+              systemctl stop rustfs
+          fi
+          PRERM
+            ) \
+            --after-remove <(cat << 'POSTRM'
+          #!/bin/bash
+          set -e
+          if [ -d /run/systemd/system ]; then
+              systemctl daemon-reload
+          fi
+          POSTRM
+            ) \
+            --config-files /etc/default/rustfs \
+            "rustfs-nightly-${DEB_DATE}/usr/bin/rustfs=/usr/bin/rustfs" \
+            ./tmp-pkg/etc/default/rustfs=/etc/default/rustfs \
+            deploy/build/rustfs.service=/lib/systemd/system/rustfs.service \
+            LICENSE=/usr/share/doc/rustfs/LICENSE \
+            README.md=/usr/share/doc/rustfs/README.md
+
+          [[ -f "$RPM_FILE" ]] || { echo "RPM build failed"; exit 1; }
+          rpm -qpl "$RPM_FILE" | grep -Fx '/usr/bin/rustfs' >/dev/null
+          stat --printf='%n %s bytes\n' "$RPM_FILE"
+          echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
+
       - name: Upload DEB artifact
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
           name: ${{ steps.deb.outputs.deb_file }}
           path: ${{ steps.deb.outputs.deb_file }}
+      - name: Upload RPM artifact
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: ${{ steps.rpm.outputs.rpm_file }}
+          path: ${{ steps.rpm.outputs.rpm_file }}
           if-no-files-found: error
 
       # Persist the nightly deb on Cloudflare R2 (same channel as package.yml)
@@ -187,11 +289,10 @@ jobs:
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
+          # The candidate manifest must describe the tree that was actually
+          # built. With a ref override (NIGHTLY_BRANCH / dispatch input) that
+          # is not necessarily GITHUB_SHA, so always advertise HEAD.
           SOURCE_SHA="$(git rev-parse HEAD)"
-          if [[ "${SOURCE_SHA}" != "${GITHUB_SHA}" ]]; then
-            echo "Checkout SHA does not match the nightly build run" >&2
-            exit 1
-          fi
           DEB_SHA256="$(sha256sum "${DEB_FILE}" | cut -d ' ' -f 1)"
           CANDIDATE_KEY="artifacts/rustfs/packages/nightly/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${DEB_SHA256}/rustfs.deb"
           CANDIDATE_URL="https://dl.rustfs.com/${CANDIDATE_KEY}"
@@ -247,6 +348,20 @@ jobs:
           path: ${{ steps.publish.outputs.candidate_file }}
           if-no-files-found: error
 
+      # Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
+      # engineers can download and install the nightly directly. The branch is
+      # a single-commit orphan rewritten on every build, which keeps the repo
+      # small while the latest files stay reachable at stable raw URLs.
+      # Publish the deb/rpm pair as assets of the rolling `nightly` release on
+      # rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
+      - name: Publish packages to auto-testing release assets
+        env:
+          ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
+          RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
+          DEB_DATE: ${{ steps.deb.outputs.deb_date }}
+          BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
+        run: bash scripts/release/publish_nightly_assets.sh
   # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
   #
   # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
@@ -284,6 +399,7 @@ jobs:
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
+          ref: ${{ env.NIGHTLY_BUILD_REF }}
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -372,6 +488,7 @@ jobs:
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
+          ref: ${{ env.NIGHTLY_BUILD_REF }}
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup

.github/workflows/windows-filesystem.yml

@@ -16,7 +16,7 @@ name: Windows Filesystem Tests
 
 on:
   push:
-    branches: [ main ]
+    branches: [ main, release ]
     paths:
       - "crates/ecstore/src/disk/**"
       - "crates/ecstore/src/store/init_format.rs"
@@ -26,7 +26,7 @@ on:
       - ".github/actions/setup/**"
       - ".github/workflows/windows-filesystem.yml"
   pull_request:
-    branches: [ main ]
+    branches: [ main, release ]
     paths:
       - "crates/ecstore/src/disk/**"
       - "crates/ecstore/src/store/init_format.rs"

Cargo.lock

@@ -701,9 +701,9 @@ dependencies = [
 
 [[package]]
 name = "async-compression"
-version = "0.4.44"
+version = "0.4.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "515a1f282e33d55983c499d7e9e87082e81cbc32974825bf9032f928392d5844"
+checksum = "4f10dafd0c8d2e51ae9a748805777613ed0bbe17bf586b76c8311f45c020a32f"
 dependencies = [
  "compression-codecs",
  "compression-core",
@@ -1031,9 +1031,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-s3"
-version = "1.145.0"
+version = "1.146.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0e6320417a37c8a62f78b443d0b4cf628b57cd340a09b0eb56173d47cc94e93"
+checksum = "37b5ffaae346b9bd486ebdc3eb7053ec8ab8a1ad0f19a332eefeff036ed559e6"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -1735,18 +1735,18 @@ dependencies = [
 
 [[package]]
 name = "bon"
-version = "3.10.0"
+version = "3.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e3fac94a66da67200398458a25412bcc3f9b6443b5119a6cad9cf3ccfcd8cc6"
+checksum = "60eafe0d77c3a2fc292c1d1346c3041b33c0a108085a2afabf672b70f69dbbc9"
 dependencies = [
  "bon-macros",
 ]
 
 [[package]]
 name = "bon-macros"
-version = "3.10.0"
+version = "3.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4654961ad0494e4774c5c60b4cb4cd0ae9b9d92d039d901638b1dba97ebebf5"
+checksum = "bd0f9631d8aaaee112c41985d675ef269e02acbd4f33122836af4f0c5f699ff6"
 dependencies = [
  "darling 0.24.1",
  "ident_case",
@@ -2220,7 +2220,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47"
 dependencies = [
  "unicode-segmentation",
- "unicode-width 0.2.2",
+ "unicode-width",
 ]
 
 [[package]]
@@ -2238,9 +2238,9 @@ dependencies = [
 
 [[package]]
 name = "compression-codecs"
-version = "0.4.39"
+version = "0.4.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fe67f2944eef52fc7b106b8c9450d243a88701a0c065f7f57235e76abaed7df"
+checksum = "58a6d0db8759036a783bc7c3f7a07f8cef3bf9470eb1db3bc86e8bcd1c5d0fe8"
 dependencies = [
  "brotli 8.0.4",
  "bzip2",
@@ -2249,8 +2249,8 @@ dependencies = [
  "liblzma",
  "lz4",
  "memchr",
- "zstd 0.13.3",
- "zstd-safe 7.3.0",
+ "zstd 0.14.0",
+ "zstd-safe 8.0.0",
 ]
 
 [[package]]
@@ -2408,6 +2408,12 @@ version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
+[[package]]
+name = "core_detect"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48"
+
 [[package]]
 name = "cpp_demangle"
 version = "0.5.1"
@@ -3927,16 +3933,6 @@ dependencies = [
  "dirs-sys",
 ]
 
-[[package]]
-name = "dirs-next"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
-dependencies = [
- "cfg-if",
- "dirs-sys-next",
-]
-
 [[package]]
 name = "dirs-sys"
 version = "0.5.0"
@@ -3945,21 +3941,10 @@ checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab"
 dependencies = [
  "libc",
  "option-ext",
- "redox_users 0.5.2",
+ "redox_users",
  "windows-sys 0.61.2",
 ]
 
-[[package]]
-name = "dirs-sys-next"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
-dependencies = [
- "libc",
- "redox_users 0.4.6",
- "winapi",
-]
-
 [[package]]
 name = "dispatch2"
 version = "0.3.1"
@@ -4208,11 +4193,17 @@ checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
 
 [[package]]
 name = "encoding_rs"
-version = "0.8.35"
+version = "0.8.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
+checksum = "7b5ef0006ac9ab233c38522f5ae99cae3625151de8f706cacee1cba4b8e2832a"
 dependencies = [
  "cfg-if",
+ "core_detect",
+ "multiversion",
+ "multiversion_no_op",
+ "rustversion",
+ "scopeguard",
+ "simdutf8",
 ]
 
 [[package]]
@@ -5071,9 +5062,9 @@ dependencies = [
 
 [[package]]
 name = "hashlink"
-version = "0.12.1"
+version = "0.12.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32069d97bb81e38fa67eab65e3393bf804bb85969f2bc06bf13f64aef5aba248"
+checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb"
 dependencies = [
  "hashbrown 0.17.1",
 ]
@@ -5287,9 +5278,9 @@ dependencies = [
 
 [[package]]
 name = "hotpath"
-version = "0.25.0"
+version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ec7782e005cabd5eaf350febde384cd799faa3a0e624587aa8759c240e0b592"
+checksum = "366c5db5a8e4643bdde121a1d6f35cfa035e7850b8b8ea94c6960a139d44e93a"
 dependencies = [
  "arc-swap",
  "async-channel",
@@ -5298,6 +5289,7 @@ dependencies = [
  "crossbeam-channel",
  "flate2",
  "futures-channel",
+ "futures-core",
  "futures-util",
  "hdrhistogram",
  "hotpath-macros",
@@ -5306,9 +5298,8 @@ dependencies = [
  "object 0.36.7",
  "parking_lot",
  "pin-project-lite",
- "prettytable-rs",
  "quanta",
- "regex",
+ "regex-lite",
  "reqwest",
  "reqwest-middleware",
  "rustc-demangle",
@@ -5410,9 +5401,9 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
 
 [[package]]
 name = "hybrid-array"
-version = "0.4.14"
+version = "0.4.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b"
+checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17"
 dependencies = [
  "ctutils",
  "subtle",
@@ -5711,9 +5702,9 @@ checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96"
 
 [[package]]
 name = "io-uring"
-version = "0.7.14"
+version = "0.7.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d64d8ca234d152948ceaede1f419b6a83983a5ecccaac05fb337a809c96d3aa6"
+checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb"
 dependencies = [
  "bitflags 2.13.1",
  "cfg-if",
@@ -5751,17 +5742,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "is-terminal"
-version = "0.4.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
-dependencies = [
- "hermit-abi",
- "libc",
- "windows-sys 0.61.2",
-]
-
 [[package]]
 name = "is_debug"
 version = "1.1.0"
@@ -6021,9 +6001,9 @@ checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37"
 
 [[package]]
 name = "lapin"
-version = "4.10.0"
+version = "4.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fd20e01fd92597ca352ca7ceed3c589851ebad279dfcada48aa4d24fd3a7caa"
+checksum = "24b99d8cdfe3f6223f9f75e35e7510a8662767bee27c1cf870d503fc3bb2428a"
 dependencies = [
  "amq-protocol",
  "async-rs",
@@ -6724,6 +6704,33 @@ version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084"
 
+[[package]]
+name = "multiversion"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4ca4bea16ffc3f443cf7d866912118196bfef4c6a1556ca00f9f9b00bb43f7c"
+dependencies = [
+ "multiversion-macros",
+]
+
+[[package]]
+name = "multiversion-macros"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d416831a7317ef4b08bee00b69cbbb9c8763da7959a7026244d6266869f9c83"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn 3.0.5",
+]
+
+[[package]]
+name = "multiversion_no_op"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d"
+
 [[package]]
 name = "murmur3"
 version = "0.5.2"
@@ -7709,9 +7716,9 @@ dependencies = [
 
 [[package]]
 name = "pe-unwind-info"
-version = "0.6.0"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97f6fccfd2d9d2df765ca23ff85fe5cc437fb0e6d3e164e4d3cbe09d14780c93"
+checksum = "e33c6dbf1a8fb7f71742cd70f5e9f0986e60b2d19dc0b28d9ca0d1323259274a"
 dependencies = [
  "arrayvec",
  "bitflags 2.13.1",
@@ -8187,19 +8194,6 @@ dependencies = [
  "syn 3.0.5",
 ]
 
-[[package]]
-name = "prettytable-rs"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a"
-dependencies = [
- "encode_unicode",
- "is-terminal",
- "lazy_static",
- "term",
- "unicode-width 0.1.14",
-]
-
 [[package]]
 name = "primefield"
 version = "0.14.0"
@@ -8519,24 +8513,24 @@ dependencies = [
 
 [[package]]
 name = "qrcode-core"
-version = "2.0.0"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79de8ed9b3c65c866e0ca79ba3aa8b05264e5183b0f894f1ea43b3a53146d657"
+checksum = "5a5c0d85fe467f79319e9f0d0ffe43f8c32defae67263f78d57e33d8b4a638bf"
 
 [[package]]
 name = "qrcode-decode"
-version = "2.0.0"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41c307d82a6b418dc99004a68cdd621c0606874faae8645036eef9168ec5aef4"
+checksum = "04cf729caefa8fd0919f8d111a4e6531942ac48742e7649ed4c9c159b41a6fda"
 dependencies = [
  "qrcode-core",
 ]
 
 [[package]]
 name = "qrcode-eps"
-version = "2.0.0"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1cf7d12b9e539644309c628db9436be7024e0fbe16f440109ce92d7c9787688"
+checksum = "d289a6212464685d1e598c3692e02189a096b2f4abe1acdb1f4fba6a932809d2"
 dependencies = [
  "qrcode-core",
  "qrcode-render",
@@ -8544,9 +8538,19 @@ dependencies = [
 
 [[package]]
 name = "qrcode-html"
-version = "2.0.0"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fb7d790f2b34ce1edbafdf7a6802e73694aa94798be275c63acb780a39c1e6c"
+checksum = "5a14b98d4aa6278f7e7218ad901837a85cef040a262127d0c4b6f8d3bd51d46e"
+dependencies = [
+ "qrcode-core",
+ "qrcode-render",
+]
+
+[[package]]
+name = "qrcode-image"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7923bee1dc13ef96b9778e9299d803ae9f5de38971cb8e55c141022cc296618"
 dependencies = [
  "qrcode-core",
  "qrcode-render",
@@ -8554,15 +8558,15 @@ dependencies = [
 
 [[package]]
 name = "qrcode-parse"
-version = "2.0.0"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84f6484a2329e87b5a4b21c983d68bee32a442087287c0679efe782689ee469a"
+checksum = "0726e7e2acd6335b63936ba1d784361765313f53274942efb0636407e882fdc9"
 
 [[package]]
 name = "qrcode-pdf"
-version = "2.0.0"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "deda8c86193091e31856024288e986869559ace2a71b7b477f59b4caea4dabf4"
+checksum = "41ed9a51cab153e441963f8ec64ffc980d7a40bfd7fa8f6e0edd290424cda206"
 dependencies = [
  "qrcode-core",
  "qrcode-render",
@@ -8570,9 +8574,9 @@ dependencies = [
 
 [[package]]
 name = "qrcode-pic"
-version = "2.0.0"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03bda277eb6e11b06e0ebe0cb1b519d71467a3f1d7f6408336af9a979edb1091"
+checksum = "038be675ee01aeae82f20eea4b043054f8232e14d3ccc7836803510ad3ff6524"
 dependencies = [
  "qrcode-core",
  "qrcode-render",
@@ -8580,23 +8584,24 @@ dependencies = [
 
 [[package]]
 name = "qrcode-render"
-version = "2.0.0"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a40771febec53bdf7de3b3de6549950a7c9f54eef58477d99f4de2fd9f114f9"
+checksum = "7af1ddfd0f2496c28276913e26289e8fb245cab16ece7efcf9df46e2027dd3c8"
 dependencies = [
  "qrcode-core",
 ]
 
 [[package]]
 name = "qrcode-rs"
-version = "2.0.0"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a62237103e62d01195453a4b34d56a35f59cf8c8ca3fa8ddcfca84336d5c4a2a"
+checksum = "246e31e4a82d59d4714c589e4912f27bc67f5fa205cc4e3bdf916ccc640af010"
 dependencies = [
  "qrcode-core",
  "qrcode-decode",
  "qrcode-eps",
  "qrcode-html",
+ "qrcode-image",
  "qrcode-parse",
  "qrcode-pdf",
  "qrcode-pic",
@@ -8606,9 +8611,9 @@ dependencies = [
 
 [[package]]
 name = "qrcode-svg"
-version = "2.0.0"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "513f1dd2a0e11d1bcc4dd3268887cdad059ed8ab5269d45ce57aada1256c5405"
+checksum = "19f641cac9a021d31948437c740baec47243c03586c3230c839b76df9e4df084"
 dependencies = [
  "qrcode-core",
  "qrcode-render",
@@ -8981,17 +8986,6 @@ dependencies = [
  "bitflags 2.13.1",
 ]
 
-[[package]]
-name = "redox_users"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"
-dependencies = [
- "getrandom 0.2.17",
- "libredox",
- "thiserror 1.0.69",
-]
-
 [[package]]
 name = "redox_users"
 version = "0.5.2"
@@ -9084,11 +9078,11 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
 
 [[package]]
 name = "reqwest"
-version = "0.13.4"
+version = "0.13.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
+checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029"
 dependencies = [
- "base64 0.22.1",
+ "base64 0.23.1",
  "bytes",
  "encoding_rs",
  "futures-channel",
@@ -9310,9 +9304,9 @@ dependencies = [
 
 [[package]]
 name = "russh"
-version = "0.63.2"
+version = "0.63.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e134e2480f4e86f83e4aa45b4c0a9723f84beaffa694c54bdf056e74efdd7dd"
+checksum = "036204edbd199552a5b3832f63c60dcdf395dc44c7f06b4af1c0e8139cc11bce"
 dependencies = [
  "aes 0.9.3",
  "aws-lc-rs",
@@ -9393,9 +9387,9 @@ dependencies = [
 
 [[package]]
 name = "russh-sftp"
-version = "2.4.0"
+version = "3.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9de67aace74530a29086db0671fa200c470a58eb380081f28ad512ffb0c5356b"
+checksum = "093197e526668d92bba562e2bbbe98d1af9831bf080b619c736316ca1fa35101"
 dependencies = [
  "bitflags 2.13.1",
  "bytes",
@@ -9928,6 +9922,7 @@ dependencies = [
  "async-trait",
  "base64-simd",
  "bytes",
+ "chrono",
  "crc-fast",
  "futures",
  "hotpath",
@@ -11085,9 +11080,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.23.43"
+version = "0.23.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06"
+checksum = "6725596c3f2c3a0aef021139e145d4eafe314a6623e4680ca83852b2c67ab2ba"
 dependencies = [
  "aws-lc-rs",
  "log",
@@ -11603,11 +11598,11 @@ dependencies = [
 
 [[package]]
 name = "serde_with"
-version = "3.22.0"
+version = "3.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a"
+checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4"
 dependencies = [
- "base64 0.22.1",
+ "base64 0.23.1",
  "bs58",
  "chrono",
  "hex",
@@ -11624,14 +11619,14 @@ dependencies = [
 
 [[package]]
 name = "serde_with_macros"
-version = "3.22.0"
+version = "3.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46"
+checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc"
 dependencies = [
- "darling 0.23.0",
+ "darling 0.24.1",
  "proc-macro2",
  "quote",
- "syn 2.0.119",
+ "syn 3.0.5",
 ]
 
 [[package]]
@@ -12195,9 +12190,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
 [[package]]
 name = "suppaftp"
-version = "11.0.0"
+version = "12.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46c5095831abc0d7944a2d50d6ec6abcd75b9d165d9377deb3e45798cae2343a"
+checksum = "9c6978f1b04fc86f122f767b88a9d1d89e80cfe3dc97d670598507d713c4ce48"
 dependencies = [
  "async-trait",
  "chrono",
@@ -12416,17 +12411,6 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
-[[package]]
-name = "term"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"
-dependencies = [
- "dirs-next",
- "rustversion",
- "winapi",
-]
-
 [[package]]
 name = "termcolor"
 version = "1.4.1"
@@ -13193,12 +13177,6 @@ version = "1.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
 
-[[package]]
-name = "unicode-width"
-version = "0.1.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
-
 [[package]]
 name = "unicode-width"
 version = "0.2.2"
@@ -13999,18 +13977,18 @@ dependencies = [
 
 [[package]]
 name = "zerocopy"
-version = "0.8.56"
+version = "0.8.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb"
+checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.56"
+version = "0.8.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1"
+checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc"
 dependencies = [
  "proc-macro2",
  "quote",

Cargo.toml

@@ -145,7 +145,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.5" }
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
 mysql_async = { default-features = false, version = "0.37.1" }
-async-compression = { version = "0.4.44" }
+async-compression = { version = "0.4.46" }
 async-recursion = "1.1.1"
 async-trait = "0.1.92"
 async-nats = { version = "0.50.0", default-features = false }
@@ -156,7 +156,7 @@ futures-lite = "2.6.1"
 futures-util = "0.3.34"
 pollster = "1.0.1"
 pulsar = { default-features = false, version = "6.9.0" }
-lapin = { default-features = false, version = "4.10.0" }
+lapin = { default-features = false, version = "4.11.0" }
 hyper = { version = "1.11.1" }
 hyper-rustls = { default-features = false, version = "0.27.9" }
 hyper-util = { version = "0.1.20" }
@@ -164,7 +164,7 @@ http = "1.5.0"
 http-body = "1.1.0"
 http-body-util = "0.1.5"
 minlz = "1.2.3"
-reqwest = "0.13.4"
+reqwest = "0.13.5"
 rustfs-kafka-async = { version = "1.3.1" }
 socket2 = { version = "0.6.5" }
 tokio = { version = "1.53.1" }
@@ -211,7 +211,7 @@ openidconnect = { default-features = false, version = "4.0" }
 pbkdf2 = "0.13.0"
 p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
 rsa = { version = "=0.10.0-rc.18" }
-rustls = { default-features = false, version = "0.23.43" }
+rustls = { default-features = false, version = "0.23.44" }
 rustls-native-certs = "0.8"
 rustls-pki-types = "1.15.1"
 x509-parser = "0.18.1"
@@ -245,7 +245,7 @@ atomic_enum = "0.3.0"
 aws-config = { version = "1.12.0" }
 aws-credential-types = { version = "1.3.0" }
 aws-sdk-kms = { default-features = false, version = "1.118.0" }
-aws-sdk-s3 = { default-features = false, version = "1.145.0" }
+aws-sdk-s3 = { default-features = false, version = "1.146.0" }
 aws-sdk-sts = { default-features = false, version = "1.114.0" }
 aws-smithy-async = { version = "1.3.0" }
 aws-smithy-http-client = { default-features = false, version = "1.4.0" }
@@ -297,7 +297,7 @@ percent-encoding = "2.3.2"
 # Server-side QR rendering for TOTP enrollment, so neither the console nor the
 # CLI needs its own QR encoder. No default features: the image/render backends
 # pull in an image stack this only needs SVG and text output from.
-qrcode-rs = { version = "2.0.0", default-features = false, features = ["std", "svg"] }
+qrcode-rs = { version = "2.1.0", default-features = false, features = ["std", "svg"] }
 pin-project-lite = "0.2.17"
 pretty_assertions = "1.4.1"
 rand = { version = "0.10.2" }
@@ -362,10 +362,10 @@ pyroscope = { version = "2.1.1" }
 # FTP and SFTP
 libunftp = { version = "0.23.0" }
 unftp-core = "0.1.0"
-suppaftp = { version = "11.0.0" }
+suppaftp = { version = "12.0.0" }
 rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
-russh = { version = "0.63.2" }
-russh-sftp = "2.4.0"
+russh = { version = "0.63.3" }
+russh-sftp = "3.0.0"
 
 # WebDAV
 dav-server = "0.11.0"
@@ -373,7 +373,7 @@ dav-server = "0.11.0"
 # Performance Analysis and Memory Profiling
 rustfs-mimalloc = { version = "0.5.3" }
 # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
-hotpath = { version = "=0.25.0", default-features = false }
+hotpath = { version = "0.25.1", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.48" }
 

README.md

@@ -211,7 +211,10 @@ For developers who want to build RustFS Docker images from source with multi-arc
 
 ```bash
 # Build multi-architecture images locally
-./docker-buildx.sh --build-arg RELEASE=latest
+./docker-buildx.sh
+
+# Build a single-platform image locally
+./docker-buildx.sh -p linux/amd64
 
 # Build and push to registry
 ./docker-buildx.sh --push

README_ZH.md

@@ -167,7 +167,10 @@ docker compose -f docker-compose-simple.yml up -d
 
 ```bash
 # 在本地构建多架构镜像
-./docker-buildx.sh --build-arg RELEASE=latest
+./docker-buildx.sh
+
+# 在本地构建单平台镜像
+./docker-buildx.sh -p linux/amd64
 
 # 构建并推送到仓库
 ./docker-buildx.sh --push

crates/data-usage/src/data_usage.rs

@@ -593,6 +593,20 @@ pub struct DataUsageSnapshotIdentity {
     pub scanner_epoch: Option<u64>,
 }
 
+#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
+pub struct DataUsageSegmentInvalidationProof {
+    #[serde(default)]
+    pub process_epoch: String,
+    #[serde(default)]
+    pub generation_start: u64,
+    #[serde(default)]
+    pub generation_end: u64,
+    #[serde(default)]
+    pub producer_identity_coverage_complete: bool,
+    #[serde(default)]
+    pub cold_zero_walk_oracle: bool,
+}
+
 #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
 pub struct DataUsageSnapshotSetState {
     pub pool_index: u64,
@@ -607,6 +621,8 @@ pub struct DataUsageSnapshotSetState {
     pub complete: bool,
     #[serde(default)]
     pub tombstone: bool,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub segment_invalidation_proof: Option<DataUsageSegmentInvalidationProof>,
 }
 
 impl DataUsageInfo {
@@ -3073,6 +3089,7 @@ mod tests {
             scan_plan_digest: Some([1; 32]),
             complete: false,
             tombstone: false,
+            segment_invalidation_proof: None,
         }];
         assert!(observed_data_usage_is_newer(&partial, &authoritative));
     }
@@ -3095,6 +3112,7 @@ mod tests {
                     scan_plan_digest: Some([1; 32]),
                     complete: true,
                     tombstone: false,
+                    segment_invalidation_proof: None,
                 },
                 DataUsageSnapshotSetState {
                     pool_index: 1,
@@ -3104,6 +3122,7 @@ mod tests {
                     scan_plan_digest: Some([2; 32]),
                     complete: false,
                     tombstone: false,
+                    segment_invalidation_proof: None,
                 },
             ],
             ..Default::default()
@@ -3113,6 +3132,42 @@ mod tests {
         assert!(partial.is_valid_partial_snapshot());
     }
 
+    #[test]
+    fn set_state_segment_invalidation_proof_is_additive() {
+        #[derive(Deserialize)]
+        struct LegacySetState {
+            pool_index: u64,
+            set_index: u64,
+            complete: bool,
+        }
+
+        let proof = DataUsageSegmentInvalidationProof {
+            process_epoch: "scanner-process".to_string(),
+            generation_start: 3,
+            generation_end: 5,
+            producer_identity_coverage_complete: true,
+            cold_zero_walk_oracle: true,
+        };
+        let state = DataUsageSnapshotSetState {
+            pool_index: 1,
+            set_index: 2,
+            scanner_cycle: Some(9),
+            scanner_epoch: Some(4),
+            scan_plan_digest: Some([7; 32]),
+            complete: true,
+            tombstone: false,
+            segment_invalidation_proof: Some(proof.clone()),
+        };
+        let encoded = rmp_serde::to_vec_named(&state).expect("set state should encode with additive proof");
+        let legacy: LegacySetState = rmp_serde::from_slice(&encoded).expect("legacy readers should ignore proof metadata");
+        assert_eq!(legacy.pool_index, 1);
+        assert_eq!(legacy.set_index, 2);
+        assert!(legacy.complete);
+
+        let decoded: DataUsageSnapshotSetState = rmp_serde::from_slice(&encoded).expect("new readers should restore proof");
+        assert_eq!(decoded.segment_invalidation_proof, Some(proof));
+    }
+
     #[test]
     fn completeness_marker_requires_a_snapshot_timestamp() {
         let untimestamped = DataUsageInfo {

crates/e2e_test/README.md

@@ -30,6 +30,8 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
 
+The external-tool `storage_metric_ownership_test` validates the OTLP/Collector/Prometheus path, including a rolling upgrade and node failures. See the [storage metrics guide](../../docs/operations/storage-metrics.md) for its required binaries and focused command.
+
 ## How to run
 
 All commands assume repo root. `cargo test` triggers an on-demand build of the

crates/e2e_test/src/cluster_concurrency_test.rs

@@ -268,6 +268,7 @@ async fn test_bucket_cors_write_is_visible_on_peer_before_response() -> Result<(
     let rule = CorsRule::builder()
         .allowed_methods("GET")
         .allowed_origins("https://example.com")
+        .allowed_headers("*")
         .build()?;
     let configuration = CorsConfiguration::builder().cors_rules(rule).build()?;
 
@@ -288,6 +289,60 @@ async fn test_bucket_cors_write_is_visible_on_peer_before_response() -> Result<(
     assert_eq!(rules[0].allowed_methods(), ["GET"]);
     assert_eq!(rules[0].allowed_origins(), ["https://example.com"]);
 
+    let http = reqwest::Client::builder().no_proxy().build()?;
+    let url = format!("http://{}/{}", cluster.nodes[1].address, BUCKET_METADATA_RELOAD_BUCKET);
+    let without_headers = http
+        .request(reqwest::Method::OPTIONS, &url)
+        .header("Origin", "https://example.com")
+        .header("Access-Control-Request-Method", "GET")
+        .send()
+        .await?;
+    assert!(without_headers.status().is_success());
+    assert!(!without_headers.headers().contains_key("access-control-allow-headers"));
+    assert!(
+        without_headers
+            .headers()
+            .get("vary")
+            .and_then(|value| value.to_str().ok())
+            .is_some_and(|value| value.contains("Access-Control-Request-Headers")),
+        "a cached header-free preflight must not suppress a later requested header grant"
+    );
+    let preflight = http
+        .request(reqwest::Method::OPTIONS, &url)
+        .header("Origin", "https://example.com")
+        .header("Access-Control-Request-Method", "GET")
+        .header("Access-Control-Request-Headers", "X-Another-Header, x-could-be-anything")
+        .send()
+        .await?;
+    assert!(preflight.status().is_success(), "peer preflight should succeed: {preflight:?}");
+    assert_eq!(
+        preflight
+            .headers()
+            .get("access-control-allow-headers")
+            .and_then(|value| value.to_str().ok()),
+        Some("x-another-header,x-could-be-anything"),
+        "a wildcard rule must return only the headers requested by this preflight"
+    );
+    assert!(
+        preflight
+            .headers()
+            .get("vary")
+            .and_then(|value| value.to_str().ok())
+            .is_some_and(|value| value.contains("Access-Control-Request-Headers")),
+        "preflight caches must distinguish the requested header list"
+    );
+    let denied = http
+        .request(reqwest::Method::OPTIONS, &url)
+        .header("Origin", "https://disallowed.example.com")
+        .header("Access-Control-Request-Method", "GET")
+        .header("Access-Control-Request-Headers", "x-another-header")
+        .send()
+        .await?;
+    assert!(
+        !denied.headers().contains_key("access-control-allow-headers"),
+        "a rejected origin must not receive the requested header grant"
+    );
+
     writer
         .delete_bucket_cors()
         .bucket(BUCKET_METADATA_RELOAD_BUCKET)

crates/e2e_test/src/distributed/harness.rs

@@ -962,27 +962,6 @@ pub(crate) async fn wait_for_rebalance_active(
     }
 }
 
-pub(crate) async fn wait_for_rebalance_running_with_progress(
-    cluster: &RustFSTestClusterEnvironment,
-    expected_id: &str,
-    timeout: Duration,
-) -> TestResult {
-    let deadline = Instant::now() + timeout;
-    loop {
-        let status = rebalance_status_json(cluster).await?;
-        if rebalance_running_with_progress(&status, expected_id)? {
-            return Ok(());
-        }
-        if Instant::now() >= deadline {
-            return Err(format!(
-                "rebalance did not become active with non-zero progress within {timeout:?}; last status: {status}"
-            )
-            .into());
-        }
-        sleep(Duration::from_millis(100)).await;
-    }
-}
-
 pub(crate) async fn wait_for_rebalance_complete(
     cluster: &RustFSTestClusterEnvironment,
     expected_id: &str,

crates/e2e_test/src/distributed/heal_test.rs

@@ -26,6 +26,7 @@ use std::collections::{BTreeMap, HashSet};
 use std::io::{Read, Write};
 use std::path::{Path, PathBuf};
 use std::time::Duration;
+use tokio::time::{Instant, sleep};
 
 const EC84_NODE_COUNT: usize = 3;
 const EC84_DRIVES_PER_NODE: usize = 4;
@@ -33,6 +34,8 @@ const EC84_DATA_BLOCKS: usize = 8;
 const EC84_PARITY_BLOCKS: usize = 4;
 const EC84_TARGET_DRIVE_RESTART_CASE: &str = "ec84-target-drive-restart";
 const EC84_TARGET_DRIVE_RESTART_ORACLE: &str = "ec84-target-drive-restart.json";
+const EC84_HEAL_CONTROL_READY_TIMEOUT: Duration = Duration::from_secs(45);
+const EC84_HEAL_CONTROL_RETRY_DELAY: Duration = Duration::from_millis(250);
 
 #[derive(Clone)]
 struct ExpectedShard {
@@ -211,6 +214,29 @@ fn assert_replaced_drive_empty(drive: &Path, bucket: &str, keys: &[String]) -> T
     Ok(())
 }
 
+fn is_cluster_heal_coordination_unavailable(error: &(dyn std::error::Error + Send + Sync)) -> bool {
+    let message = error.to_string();
+    message.contains("500 Internal Server Error") && message.contains("cluster heal coordination unavailable")
+}
+
+async fn start_ec84_root_heal_when_control_ready(
+    heal_url: &str,
+    heal_body: &str,
+    access_key: &str,
+    secret_key: &str,
+) -> TestResult {
+    let deadline = Instant::now() + EC84_HEAL_CONTROL_READY_TIMEOUT;
+    loop {
+        match signed_admin_post(heal_url, Some(heal_body), access_key, secret_key).await {
+            Ok(_) => return Ok(()),
+            Err(error) if is_cluster_heal_coordination_unavailable(error.as_ref()) && Instant::now() < deadline => {
+                sleep(EC84_HEAL_CONTROL_RETRY_DELAY).await;
+            }
+            Err(error) => return Err(error),
+        }
+    }
+}
+
 async fn put_large_inventory(client: &Client, bucket: &str) -> TestResult<Vec<ExpectedShard>> {
     let mut expected = Vec::new();
     for index in 0..4 {
@@ -304,7 +330,7 @@ async fn three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_res
     let heal_body =
         r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
     let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[0].url);
-    signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?;
+    start_ec84_root_heal_when_control_ready(&heal_url, heal_body, &dist.cluster.access_key, &dist.cluster.secret_key).await?;
 
     wait_until(
         Duration::from_secs(120),
@@ -365,3 +391,23 @@ async fn three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_res
 
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn cluster_heal_coordination_retry_is_exact() {
+        let retryable: Box<dyn std::error::Error + Send + Sync> =
+            "admin POST failed: 500 Internal Server Error cluster heal coordination unavailable".into();
+        assert!(is_cluster_heal_coordination_unavailable(retryable.as_ref()));
+
+        let other_internal: Box<dyn std::error::Error + Send + Sync> =
+            "admin POST failed: 500 Internal Server Error unrelated".into();
+        assert!(!is_cluster_heal_coordination_unavailable(other_internal.as_ref()));
+
+        let wrong_status: Box<dyn std::error::Error + Send + Sync> =
+            "admin POST failed: 503 Service Unavailable cluster heal coordination unavailable".into();
+        assert!(!is_cluster_heal_coordination_unavailable(wrong_status.as_ref()));
+    }
+}

crates/e2e_test/src/distributed/s3_during_data_movement_test.rs

@@ -14,9 +14,9 @@
 
 use super::harness::{
     DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
-    decommission_status_json, put_inventory_retrying, rebalance_running_with_progress, rebalance_status_json,
-    retrying_get_equals, retrying_put, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
-    wait_for_decommission_running_with_progress, wait_for_rebalance_complete, wait_for_rebalance_running_with_progress,
+    decommission_status_json, put_inventory_retrying, rebalance_active, rebalance_status_json, retrying_get_equals, retrying_put,
+    start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
+    wait_for_decommission_running_with_progress, wait_for_rebalance_active, wait_for_rebalance_complete,
 };
 use crate::common::init_logging;
 use std::time::Duration;
@@ -67,7 +67,10 @@ async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResu
     assert_inventory(&live, &bucket, &inventory).await?;
 
     let rebalance_id = start_rebalance(&dist.cluster).await?;
-    wait_for_rebalance_running_with_progress(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
+    // The status API reads persisted progress, whose first periodic save is
+    // after 30 seconds. A shorter run can remain at zero until completion.
+    // Require Started around the S3 operations and nonzero progress at completion.
+    wait_for_rebalance_active(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
     retrying_put(
         &live,
         &bucket,
@@ -84,11 +87,26 @@ async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResu
         Duration::from_secs(30),
     )
     .await?;
+    let listed = live.list_objects_v2().bucket(&bucket).send().await?;
+    assert!(
+        listed
+            .contents()
+            .iter()
+            .any(|object| object.key() == Some("during-rebalance.bin")),
+        "list during rebalance missed the newly written key"
+    );
     let status = rebalance_status_json(&dist.cluster).await?;
-    if !rebalance_running_with_progress(&status, &rebalance_id)? {
+    if !rebalance_active(&status, &rebalance_id)? {
         return Err(format!("rebalance did not remain active across the S3 operations: {status}").into());
     }
     wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
-    assert_inventory(&dist.client(1)?, &bucket, &inventory).await?;
+    let after = dist.client(1)?;
+    assert_inventory(&after, &bucket, &inventory).await?;
+    for (key, body) in [
+        ("during-decommission.bin", b"written-while-decommissioning".as_slice()),
+        ("during-rebalance.bin", b"written-while-rebalancing".as_slice()),
+    ] {
+        retrying_get_equals(&after, &bucket, key, body, Duration::from_secs(30)).await?;
+    }
     Ok(())
 }

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -62,18 +62,38 @@ mod tests {
         unclean_shutdown_marker: bool,
         topology: EvidenceTopology,
         storage_class_standard: Option<&'static str>,
-        erasure_set_drive_count: Option<&'static str>,
+        erasure_set_drive_count: Option<usize>,
+        outage_target_manifest_required: bool,
     }
 
     #[derive(Clone, Copy)]
     struct EvidenceTopology {
         nodes: usize,
         drives_per_node: usize,
+        layout: EvidenceTopologyLayout,
+    }
+
+    #[derive(Clone, Copy)]
+    enum EvidenceTopologyLayout {
+        SinglePool,
+        PerNodePools,
     }
 
     impl EvidenceTopology {
         const fn new(nodes: usize, drives_per_node: usize) -> Self {
-            Self { nodes, drives_per_node }
+            Self {
+                nodes,
+                drives_per_node,
+                layout: EvidenceTopologyLayout::SinglePool,
+            }
+        }
+
+        const fn per_node_pools(nodes: usize, drives_per_node: usize) -> Self {
+            Self {
+                nodes,
+                drives_per_node,
+                layout: EvidenceTopologyLayout::PerNodePools,
+            }
         }
 
         fn total_drives(self) -> usize {
@@ -81,7 +101,23 @@ mod tests {
         }
 
         fn cluster_topology(self) -> ClusterTopology {
-            ClusterTopology::single_pool_multidrive(self.nodes, self.drives_per_node)
+            match self.layout {
+                EvidenceTopologyLayout::SinglePool => ClusterTopology::single_pool_multidrive(self.nodes, self.drives_per_node),
+                EvidenceTopologyLayout::PerNodePools => {
+                    ClusterTopology::per_node_pools(self.drives_per_node, (0..self.nodes).map(|node| vec![node]).collect())
+                }
+            }
+        }
+
+        fn pool_count(self) -> usize {
+            match self.layout {
+                EvidenceTopologyLayout::SinglePool => 1,
+                EvidenceTopologyLayout::PerNodePools => self.nodes,
+            }
+        }
+
+        fn set_count(self, erasure_set_drive_count: Option<usize>) -> usize {
+            self.total_drives() / erasure_set_drive_count.unwrap_or_else(|| self.total_drives())
         }
     }
 
@@ -93,6 +129,7 @@ mod tests {
         topology: EvidenceTopology::new(4, 1),
         storage_class_standard: None,
         erasure_set_drive_count: None,
+        outage_target_manifest_required: true,
     };
 
     const BACKGROUND_TARGET_CRASH_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
@@ -103,6 +140,7 @@ mod tests {
         topology: EvidenceTopology::new(4, 1),
         storage_class_standard: None,
         erasure_set_drive_count: None,
+        outage_target_manifest_required: true,
     };
 
     const BACKGROUND_TARGET_RESTART_EC84_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
@@ -112,7 +150,8 @@ mod tests {
         unclean_shutdown_marker: false,
         topology: EvidenceTopology::new(3, 4),
         storage_class_standard: Some("EC:4"),
-        erasure_set_drive_count: Some("12"),
+        erasure_set_drive_count: Some(12),
+        outage_target_manifest_required: true,
     };
 
     const BACKGROUND_TARGET_CRASH_EC84_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
@@ -122,7 +161,30 @@ mod tests {
         unclean_shutdown_marker: true,
         topology: EvidenceTopology::new(3, 4),
         storage_class_standard: Some("EC:4"),
-        erasure_set_drive_count: Some("12"),
+        erasure_set_drive_count: Some(12),
+        outage_target_manifest_required: true,
+    };
+
+    const BACKGROUND_TARGET_RESTART_EC84_MULTI_SET_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
+        id: "background-target-restart-ec8-4-multi-set",
+        oracle: "background-target-restart-ec8-4-multi-set.json",
+        evidence: "process-restart",
+        unclean_shutdown_marker: false,
+        topology: EvidenceTopology::new(3, 8),
+        storage_class_standard: Some("EC:4"),
+        erasure_set_drive_count: Some(12),
+        outage_target_manifest_required: true,
+    };
+
+    const BACKGROUND_TARGET_CRASH_EC84_MULTI_POOL_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
+        id: "background-target-crash-ec8-4-multi-pool",
+        oracle: "background-target-crash-ec8-4-multi-pool.json",
+        evidence: "process-crash-restart",
+        unclean_shutdown_marker: true,
+        topology: EvidenceTopology::per_node_pools(3, 12),
+        storage_class_standard: Some("EC:4"),
+        erasure_set_drive_count: Some(12),
+        outage_target_manifest_required: false,
     };
 
     struct RestartEvidenceContext {
@@ -1041,6 +1103,26 @@ mod tests {
         .await?
     }
 
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        timeout(
+            Duration::from_secs(600),
+            run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetRestartEc84MultiSet),
+        )
+        .await?
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_cluster_root_heal_recovers_ec84_shards_across_multi_pool_after_background_target_crash()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        timeout(
+            Duration::from_secs(600),
+            run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetCrashEc84MultiPool),
+        )
+        .await?
+    }
+
     #[tokio::test(flavor = "multi_thread")]
     async fn test_cluster_root_heal_recovers_remote_shards_after_coordinator_restart() -> Result<(), Box<dyn Error + Send + Sync>>
     {
@@ -1080,6 +1162,8 @@ mod tests {
         BackgroundTargetCrash,
         BackgroundTargetRestartEc84,
         BackgroundTargetCrashEc84,
+        BackgroundTargetRestartEc84MultiSet,
+        BackgroundTargetCrashEc84MultiPool,
         BackgroundCoordinatorRestart,
         TargetEndpointBlackhole,
     }
@@ -1091,6 +1175,8 @@ mod tests {
             InterruptionScenario::BackgroundTargetCrash => Some(BACKGROUND_TARGET_CRASH_EVIDENCE),
             InterruptionScenario::BackgroundTargetRestartEc84 => Some(BACKGROUND_TARGET_RESTART_EC84_EVIDENCE),
             InterruptionScenario::BackgroundTargetCrashEc84 => Some(BACKGROUND_TARGET_CRASH_EC84_EVIDENCE),
+            InterruptionScenario::BackgroundTargetRestartEc84MultiSet => Some(BACKGROUND_TARGET_RESTART_EC84_MULTI_SET_EVIDENCE),
+            InterruptionScenario::BackgroundTargetCrashEc84MultiPool => Some(BACKGROUND_TARGET_CRASH_EC84_MULTI_POOL_EVIDENCE),
             _ => None,
         };
         let evidence_run = match evidence_case {
@@ -1104,12 +1190,18 @@ mod tests {
             InterruptionScenario::BackgroundTargetCrash => (true, 1, "background_target_crash"),
             InterruptionScenario::BackgroundTargetRestartEc84 => (true, 1, "background_target_restart_ec8_4"),
             InterruptionScenario::BackgroundTargetCrashEc84 => (true, 1, "background_target_crash_ec8_4"),
+            InterruptionScenario::BackgroundTargetRestartEc84MultiSet => (true, 1, "background_target_restart_ec8_4_multi_set"),
+            InterruptionScenario::BackgroundTargetCrashEc84MultiPool => (true, 1, "background_target_crash_ec8_4_multi_pool"),
             InterruptionScenario::BackgroundCoordinatorRestart => (true, 0, "coordinator_restart"),
             InterruptionScenario::TargetEndpointBlackhole => (false, 1, "target_endpoint_blackhole"),
         };
         let topology = evidence_case
             .map(|case| case.topology)
             .unwrap_or_else(|| EvidenceTopology::new(4, 1));
+        let erasure_set_drive_count = evidence_case
+            .and_then(|case| case.erasure_set_drive_count)
+            .unwrap_or_else(|| topology.total_drives());
+        let outage_target_manifest_required = evidence_case.map(|case| case.outage_target_manifest_required).unwrap_or(true);
         init_logging();
         info!(
             event = "heal_interruption_started",
@@ -1128,8 +1220,10 @@ mod tests {
             cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", storage_class);
         }
         if let Some(erasure_set_drive_count) = evidence_case.and_then(|case| case.erasure_set_drive_count) {
-            cluster.set_env("RUSTFS_ERASURE_SET_DRIVE_COUNT", erasure_set_drive_count);
+            cluster.set_env("RUSTFS_ERASURE_SET_DRIVE_COUNT", erasure_set_drive_count.to_string());
         }
+        // Capture physical baselines after the PUT rename fanout has drained.
+        cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
         // Heal control uses the first lexicographically sorted grid host.
         // Keep that coordinator distinct from the remote target at index 1.
         cluster.nodes.sort_by(|left, right| left.url.cmp(&right.url));
@@ -1268,6 +1362,9 @@ mod tests {
             }
             for (drive_index, drive) in node.data_dirs.iter().enumerate() {
                 let census = census_object_version_on_disk(Path::new(drive), bucket, outage_key, None)?;
+                if !census.has_xl_meta {
+                    continue;
+                }
                 assert!(
                     census.is_complete(),
                     "online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}"
@@ -1276,7 +1373,7 @@ mod tests {
                     format!("online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}")
                 })?;
                 assert!(
-                    (1..=topology.total_drives()).contains(&erasure_index),
+                    (1..=erasure_set_drive_count).contains(&erasure_index),
                     "online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}"
                 );
                 assert!(
@@ -1285,19 +1382,26 @@ mod tests {
                 );
             }
         }
-        assert_eq!(
-            outage_peer_erasure_indices.len(),
-            topology.total_drives().saturating_sub(cluster.nodes[1].data_dirs.len()),
-            "every online drive must contribute one unique outage-object erasure index"
+        assert!(
+            !outage_peer_erasure_indices.is_empty() && outage_peer_erasure_indices.len() <= erasure_set_drive_count,
+            "outage-object must occupy one non-empty erasure set"
         );
-        let missing_outage_erasure_indices = (1..=topology.total_drives())
+        if outage_target_manifest_required {
+            let min_online_data_shards = erasure_set_drive_count.saturating_sub(4);
+            assert!(
+                outage_peer_erasure_indices.len() >= min_online_data_shards,
+                "online drives in the selected erasure set must retain at least the EC data quorum"
+            );
+        }
+        let missing_outage_erasure_indices = (1..=erasure_set_drive_count)
             .filter(|index| !outage_peer_erasure_indices.contains(index))
             .collect::<HashSet<_>>();
-        assert_eq!(
-            missing_outage_erasure_indices.len(),
-            cluster.nodes[1].data_dirs.len(),
-            "the stopped node must account for every missing outage-object erasure index"
-        );
+        if outage_target_manifest_required {
+            assert!(
+                !missing_outage_erasure_indices.is_empty(),
+                "the stopped target must account for at least one missing outage-object erasure index"
+            );
+        }
 
         let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
         if !background_enabled {
@@ -1569,7 +1673,9 @@ mod tests {
         } else {
             if matches!(
                 scenario,
-                InterruptionScenario::BackgroundTargetRestart | InterruptionScenario::BackgroundTargetRestartEc84
+                InterruptionScenario::BackgroundTargetRestart
+                    | InterruptionScenario::BackgroundTargetRestartEc84
+                    | InterruptionScenario::BackgroundCoordinatorRestart
             ) {
                 cluster.stop_node_gracefully(interruption_node).await?;
             } else {
@@ -1594,7 +1700,9 @@ mod tests {
                 unclean_shutdown_marker_observed = Some(marker_exists);
                 let expected_marker = matches!(
                     scenario,
-                    InterruptionScenario::BackgroundTargetCrash | InterruptionScenario::BackgroundTargetCrashEc84
+                    InterruptionScenario::BackgroundTargetCrash
+                        | InterruptionScenario::BackgroundTargetCrashEc84
+                        | InterruptionScenario::BackgroundTargetCrashEc84MultiPool
                 );
                 assert!(
                     marker_exists == expected_marker,
@@ -1634,9 +1742,10 @@ mod tests {
             .unwrap_or(180);
         let heal_deadline = Instant::now() + Duration::from_secs(heal_timeout_secs);
         loop {
-            if metadata_count(&replaced_disk, bucket, &expected_manifests) == expected_manifests.len()
-                && object_metadata_exists_on_disk(&replaced_disk, bucket, outage_key)
-            {
+            let baseline_recovered = metadata_count(&replaced_disk, bucket, &expected_manifests) == expected_manifests.len();
+            let outage_recovered =
+                !outage_target_manifest_required || object_metadata_exists_on_disk(&replaced_disk, bucket, outage_key);
+            if baseline_recovered && outage_recovered {
                 let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
                 let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
                 let pool_metadata_matches = match &expected_pool_metadata {
@@ -1646,7 +1755,10 @@ mod tests {
                     }
                     None => true,
                 };
-                if matching == expected_manifests.len() && outage_census.is_complete() && pool_metadata_matches {
+                if matching == expected_manifests.len()
+                    && (!outage_target_manifest_required || outage_census.is_complete())
+                    && pool_metadata_matches
+                {
                     break;
                 }
             }
@@ -1691,17 +1803,19 @@ mod tests {
             );
         }
         let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
-        assert!(
-            outage_census.is_complete(),
-            "outage object must have a complete target shard: {outage_census:?}"
-        );
-        assert_eq!(
-            outage_census
-                .erasure_index
-                .filter(|index| missing_outage_erasure_indices.contains(index)),
-            outage_census.erasure_index,
-            "the outage object must be rebuilt into one of the stopped node's missing erasure slots"
-        );
+        if outage_target_manifest_required {
+            assert!(
+                outage_census.is_complete(),
+                "outage object must have a complete target shard: {outage_census:?}"
+            );
+            assert_eq!(
+                outage_census
+                    .erasure_index
+                    .filter(|index| missing_outage_erasure_indices.contains(index)),
+                outage_census.erasure_index,
+                "the outage object must be rebuilt into one of the stopped node's missing erasure slots"
+            );
+        }
 
         if let Some(cycle_end) = scanner_cycle_floor {
             wait_for_scanner_cycle_after(&cluster, cycle_end).await?;
@@ -1767,33 +1881,22 @@ mod tests {
         let task_status_body = signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key).await?;
         let task_status: serde_json::Value = serde_json::from_str(&task_status_body)
             .map_err(|err| format!("heal task status is not JSON ({err}): {task_status_body}"))?;
+        if task_status["summary"].as_str() != Some("finished") {
+            return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
+        }
         if interruption_node == 0 {
-            // Admin tasks are process-local. Physical and queue convergence
-            // above establish recovery; a lost task must not report success.
-            assert_eq!(
-                task_status["summary"].as_str(),
-                Some("notFound"),
-                "interrupted task status: {task_status}"
-            );
-            assert_eq!(
-                task_status["detail"].as_str(),
-                Some("heal task not found or expired"),
-                "interrupted admin task must be explicitly unavailable: {task_status}"
-            );
+            // Restart recovery must finish the original durable root request.
             info!(
                 event = "heal_interruption_recovered",
                 component = "e2e_test",
                 subsystem = "heal",
                 interruption_node,
                 interruption_kind,
-                task_state = "not_found",
-                "Physical recovery completed after coordinator restart"
+                task_state = "finished",
+                "Original root heal completed after coordinator restart"
             );
             return Ok(());
         }
-        if task_status["summary"].as_str() != Some("finished") {
-            return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
-        }
 
         if let Some(evidence_context) = evidence_run {
             let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id();
@@ -1810,6 +1913,14 @@ mod tests {
                 "binary_sha256": evidence_context.run.binary.sha256,
                 "test_binary_sha256": evidence_context.run.test_binary.sha256,
                 "topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
+                "erasure_set_drive_count": erasure_set_drive_count,
+                "sets": topology.set_count(evidence_context.case.erasure_set_drive_count),
+                "pools": topology.pool_count(),
+                "outage_target_manifest_required": outage_target_manifest_required,
+                "distributed_ec_invalidation": true,
+                "peer_count": cluster.nodes.len(),
+                "same_window_remote_proof": true,
+                "all_peers_bound_to_generation_window": true,
                 "pid_before": target_pid, "pid_after": restarted_pid,
                 "unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
                 "objects": evidence_objects, "node_listings": node_listings,

crates/e2e_test/src/kms/kms_fault_recovery_test.rs

@@ -109,10 +109,10 @@ async fn test_kms_key_directory_unavailable() -> Result<(), Box<dyn std::error::
         .await;
 
     let unavailable_error = put_result2.expect_err("a missing Local KMS key directory must reject encrypted writes");
-    assert_eq!(unavailable_error.raw_response().map(|response| response.status().as_u16()), Some(500));
+    assert_eq!(unavailable_error.raw_response().map(|response| response.status().as_u16()), Some(503));
     assert_eq!(
         unavailable_error.as_service_error().and_then(ProvideErrorMetadata::code),
-        Some("InternalError")
+        Some("ServiceUnavailable")
     );
     let unavailable_absence = s3_client
         .get_object()

crates/e2e_test/src/lib.rs

@@ -12,6 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#[cfg(test)]
+mod storage_metric_ownership_test;
+
 mod reliant;
 mod storage_api;
 

crates/e2e_test/src/multipart_auth_test.rs

@@ -15,6 +15,7 @@
 //! Regression coverage for anonymous access on multipart control APIs.
 
 use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+use crate::kms::common::LocalKMSTestEnvironment;
 use async_compression::tokio::write::{BzEncoder, Lz4Encoder, XzEncoder};
 use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
 use aws_sdk_s3::operation::head_object::HeadObjectOutput;
@@ -1465,10 +1466,10 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_s3() -> Result<(), B
 async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
-    let mut env = RustFSTestEnvironment::new().await?;
-    let master_key = local_sse_master_key_value();
-    env.start_rustfs_server_with_env(vec![], &[(LOCAL_SSE_MASTER_KEY_ENV, master_key.as_str())])
-        .await?;
+    let mut kms_env = LocalKMSTestEnvironment::new().await?;
+    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
+    kms_env.wait_for_kms_ready().await?;
+    let env = &kms_env.base_env;
 
     let bucket = "anon-post-default-sse-kms";
     let object_key = "post-default-sse-kms-object.txt";
@@ -1484,7 +1485,7 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(),
                 .apply_server_side_encryption_by_default(
                     ServerSideEncryptionByDefault::builder()
                         .sse_algorithm(ServerSideEncryption::AwsKms)
-                        .kms_master_key_id("test-key")
+                        .kms_master_key_id(default_key_id)
                         .build()
                         .expect("default encryption rule should build"),
                 )

crates/e2e_test/src/namespace_lock_quorum_test.rs

@@ -17,6 +17,7 @@ use aws_sdk_s3::Client;
 use aws_sdk_s3::error::SdkError;
 use bytes::Bytes;
 use std::sync::Arc;
+use std::time::{Duration, Instant};
 use tokio::sync::Barrier;
 use tracing::{info, warn};
 
@@ -25,6 +26,307 @@ const KEY: &str = "thumb/79/concurrent-overwrite.jpg";
 
 type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
 
+async fn assert_degraded_cluster_publication_guard_errors_are_retryable() -> TestResult {
+    let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
+    cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2");
+    cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
+    cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
+    cluster.set_env("RUST_LOG", "warn");
+    cluster.start().await?;
+    cluster.create_test_bucket(BUCKET).await?;
+    let clients: Vec<_> = cluster
+        .create_all_clients()?
+        .into_iter()
+        .map(|client| {
+            Client::from_conf(
+                client
+                    .config()
+                    .to_builder()
+                    .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
+                    .build(),
+            )
+        })
+        .collect();
+
+    for alive in (1..=4).rev() {
+        if alive < 4 {
+            cluster.stop_node(alive)?;
+        }
+        for (node, client) in clients.iter().take(alive).enumerate() {
+            let key = format!("publication-put-{alive}-{node}");
+            let put = client
+                .put_object()
+                .bucket(BUCKET)
+                .key(&key)
+                .body(Bytes::from_static(b"publication guard regression").into())
+                .send()
+                .await;
+            if alive >= 3 {
+                put?;
+            } else {
+                let err = put.expect_err("PUT must reject writes without a write quorum");
+                assert_eq!(
+                    err.raw_response().map(|response| response.status().as_u16()),
+                    Some(503),
+                    "PUT with {alive} nodes alive, requested through node {node}: {err:?}"
+                );
+                assert_eq!(
+                    err.as_service_error().and_then(|error| error.meta().code()),
+                    Some("ServiceUnavailable"),
+                    "PUT with {alive} nodes alive, requested through node {node}: {err:?}"
+                );
+            }
+
+            let multipart_key = format!("publication-multipart-{alive}-{node}");
+            let multipart = client
+                .create_multipart_upload()
+                .bucket(BUCKET)
+                .key(&multipart_key)
+                .send()
+                .await;
+            if alive >= 3 {
+                let upload = multipart?;
+                let upload_id = upload
+                    .upload_id()
+                    .expect("successful multipart initialization must return an upload ID");
+                client
+                    .abort_multipart_upload()
+                    .bucket(BUCKET)
+                    .key(&multipart_key)
+                    .upload_id(upload_id)
+                    .send()
+                    .await?;
+            } else {
+                let err = multipart.expect_err("multipart initialization must reject writes without a write quorum");
+                assert_eq!(
+                    err.raw_response().map(|response| response.status().as_u16()),
+                    Some(503),
+                    "CreateMultipartUpload with {alive} nodes alive, requested through node {node}: {err:?}"
+                );
+                assert_eq!(
+                    err.as_service_error().and_then(|error| error.meta().code()),
+                    Some("ServiceUnavailable"),
+                    "CreateMultipartUpload with {alive} nodes alive, requested through node {node}: {err:?}"
+                );
+            }
+        }
+    }
+
+    cluster.stop();
+    cluster.start().await?;
+    for client in &clients {
+        for alive in [1, 3, 4] {
+            for node in 0..alive {
+                let key = format!("publication-put-{alive}-{node}");
+                let get = client.get_object().bucket(BUCKET).key(key).send().await;
+                if alive >= 3 {
+                    assert_eq!(
+                        get?.body.collect().await?.into_bytes().as_ref(),
+                        b"publication guard regression",
+                        "acknowledged writes must survive restart"
+                    );
+                } else {
+                    let err = get.expect_err("a rejected publication guard must not publish an object");
+                    assert_eq!(err.as_service_error().and_then(|error| error.meta().code()), Some("NoSuchKey"));
+                }
+            }
+        }
+        let uploads = client.list_multipart_uploads().bucket(BUCKET).send().await?;
+        assert!(
+            uploads
+                .uploads()
+                .iter()
+                .all(|upload| upload.key() != Some("publication-multipart-1-0")),
+            "a rejected publication guard must not publish a multipart upload"
+        );
+    }
+    Ok(())
+}
+
+async fn assert_quorum_object_body(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
+    let body = client
+        .get_object()
+        .bucket(bucket)
+        .key(key)
+        .send()
+        .await?
+        .body
+        .collect()
+        .await?
+        .into_bytes();
+    assert_eq!(body.as_ref(), expected, "quorum read returned incorrect contents for {key}");
+    Ok(())
+}
+
+async fn wait_for_quorum_read_admission(clients: &[Client], bucket: &str) -> TestResult {
+    // SIGKILL can orphan a granted lease. Wait for shared metadata-lock
+    // admission before asserting the stable quorum boundary; cold bodies
+    // remain unread throughout this readiness probe.
+    let deadline =
+        tokio::time::Instant::now() + rustfs_lock::fast_lock::DEFAULT_LOCK_TIMEOUT + std::time::Duration::from_secs(15);
+    loop {
+        let mut ready = true;
+        for client in clients {
+            for key in ["warm-small", "warm-large"] {
+                match client.head_object().bucket(bucket).key(key).send().await {
+                    Ok(_) => {}
+                    Err(error) if error.raw_response().is_some_and(|response| response.status().as_u16() == 503) => {
+                        ready = false;
+                        break;
+                    }
+                    Err(error) => return Err(error.into()),
+                }
+            }
+            if !ready {
+                break;
+            }
+        }
+        if ready {
+            return Ok(());
+        }
+        if tokio::time::Instant::now() >= deadline {
+            return Err(format!("read quorum did not become available after lease convergence for {bucket}").into());
+        }
+        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
+    }
+}
+
+#[tokio::test]
+async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult {
+    crate::common::init_logging();
+
+    for (node_count, parity) in [(4, 2), (6, 3), (6, 2)] {
+        let read_quorum = node_count - parity;
+        let write_quorum = read_quorum + usize::from(read_quorum == parity);
+        let mut cluster = RustFSTestClusterEnvironment::new(node_count).await?;
+        cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{parity}"));
+        // Wait for every seed fanout before removing any physical shard.
+        cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
+        cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
+        cluster.set_env("RUST_LOG", "warn,rustfs_lock=debug");
+        cluster.start().await?;
+
+        let clients = cluster
+            .create_all_clients()?
+            .into_iter()
+            .map(|client| {
+                Client::from_conf(
+                    client
+                        .config()
+                        .to_builder()
+                        .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
+                        .build(),
+                )
+            })
+            .collect::<Vec<_>>();
+        let bucket = format!("read-quorum-{node_count}-{parity}");
+        clients[0].create_bucket().bucket(&bucket).send().await?;
+        let small = b"read quorum is derived from the erasure layout".to_vec();
+        let large = (0..1_048_576)
+            .map(|index| u8::try_from(index % 251).expect("bounded payload byte"))
+            .collect::<Vec<_>>();
+        for (key, body) in [
+            ("warm-small", &small),
+            ("warm-large", &large),
+            ("cold-small", &small),
+            ("cold-large", &large),
+            ("below-quorum", &large),
+        ] {
+            clients[node_count - 1]
+                .put_object()
+                .bucket(&bucket)
+                .key(key)
+                .body(Bytes::copy_from_slice(body).into())
+                .send()
+                .await?;
+        }
+        for node in &cluster.nodes {
+            for key in ["warm-small", "warm-large", "cold-small", "cold-large", "below-quorum"] {
+                let census =
+                    crate::chaos::census_object_version_on_disk(std::path::Path::new(&node.data_dir), &bucket, key, None)?;
+                assert!(census.is_complete(), "seed shard must be complete before fault injection: {census:?}");
+                assert_eq!(census.data_blocks, Some(read_quorum));
+                assert_eq!(census.parity_blocks, Some(parity));
+            }
+        }
+        for client in &clients {
+            assert_quorum_object_body(client, &bucket, "warm-small", &small).await?;
+            assert_quorum_object_body(client, &bucket, "warm-large", &large).await?;
+        }
+
+        for offline_node in (read_quorum..node_count).rev() {
+            cluster.stop_node(offline_node)?;
+            wait_for_quorum_read_admission(&clients[..offline_node], &bucket).await?;
+            for client in clients.iter().take(offline_node) {
+                client.head_bucket().bucket(&bucket).send().await?;
+                assert_quorum_object_body(client, &bucket, "warm-large", &large).await?;
+            }
+        }
+
+        // Exercise more than the five-second positive bucket-validation TTL.
+        // Every sample must succeed; polling must not hide a transient failure.
+        let validation_deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(6);
+        loop {
+            for client in clients.iter().take(read_quorum) {
+                assert_quorum_object_body(client, &bucket, "warm-small", &small).await?;
+                assert_quorum_object_body(client, &bucket, "warm-large", &large).await?;
+                let listing = client.list_objects_v2().bucket(&bucket).send().await?;
+                for key in ["warm-small", "warm-large", "cold-small", "cold-large", "below-quorum"] {
+                    assert!(listing.contents().iter().any(|entry| entry.key() == Some(key)), "listing omitted {key}");
+                }
+            }
+            if tokio::time::Instant::now() >= validation_deadline {
+                break;
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(250)).await;
+        }
+        for client in clients.iter().take(read_quorum) {
+            assert_quorum_object_body(client, &bucket, "cold-small", &small).await?;
+            assert_quorum_object_body(client, &bucket, "cold-large", &large).await?;
+        }
+
+        let write = clients[0]
+            .put_object()
+            .bucket(&bucket)
+            .key("quorum-write")
+            .body(Bytes::copy_from_slice(&small).into())
+            .send()
+            .await;
+        if read_quorum >= write_quorum {
+            write?;
+        } else {
+            let error = write.expect_err("a read quorum must not authorize a write that needs more votes");
+            assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
+        }
+
+        cluster.stop_node(read_quorum - 1)?;
+        for client in clients.iter().take(read_quorum - 1) {
+            match client.get_object().bucket(&bucket).key("below-quorum").send().await {
+                Ok(response) => assert!(
+                    response.body.collect().await.is_err(),
+                    "fewer than {read_quorum} valid fragments must not reconstruct an uncached object"
+                ),
+                Err(error) => assert_eq!(
+                    error.as_service_error().and_then(|error| error.meta().code()),
+                    Some("ServiceUnavailable"),
+                    "a quorum loss must not be mistaken for a missing object"
+                ),
+            }
+        }
+
+        for node in 0..read_quorum - 1 {
+            cluster.stop_node(node)?;
+        }
+        cluster.start().await?;
+        for client in &clients {
+            assert_quorum_object_body(client, &bucket, "warm-large", &large).await?;
+            assert_quorum_object_body(client, &bucket, "below-quorum", &large).await?;
+        }
+    }
+
+    Ok(())
+}
+
 async fn put_object(client: Client, payload: Vec<u8>, writer_id: usize) -> Result<(), String> {
     client
         .put_object()
@@ -58,6 +360,9 @@ async fn test_concurrent_cluster_overwrites_do_not_fail_namespace_lock_quorum()
     // Keep the regression focused on false quorum-loss errors, not ordinary lock
     // wait exhaustion under a heavily contended same-key overwrite workload.
     cluster.set_env("RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT", "20");
+    cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2");
+    cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
+    cluster.set_env("RUSTFS_HEALTH_MINIMAL_RESPONSE_ENABLE", "false");
     cluster.start().await?;
     cluster.create_test_bucket(BUCKET).await?;
 
@@ -116,6 +421,181 @@ async fn test_concurrent_cluster_overwrites_do_not_fail_namespace_lock_quorum()
     );
 
     clients[0].delete_object().bucket(BUCKET).key(KEY).send().await?;
+    assert_node_readiness_tracks_quorum(&mut cluster).await?;
+    Ok(())
+}
+
+async fn assert_node_readiness_tracks_quorum(cluster: &mut RustFSTestClusterEnvironment) -> TestResult {
+    let clients: Vec<_> = cluster
+        .create_all_clients()?
+        .into_iter()
+        .map(|client| {
+            Client::from_conf(
+                client
+                    .config()
+                    .to_builder()
+                    .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
+                    .build(),
+            )
+        })
+        .collect();
+    let http = reqwest::Client::builder()
+        .no_proxy()
+        .timeout(Duration::from_secs(3))
+        .build()?;
+    let seed_key = "readiness-seed";
+    let seed_body = b"readiness quorum regression";
+    clients[0]
+        .put_object()
+        .bucket(BUCKET)
+        .key(seed_key)
+        .body(Bytes::from_static(seed_body).into())
+        .send()
+        .await?;
+
+    for (phase, survivors) in [4, 3, 2, 1, 4].into_iter().enumerate() {
+        if phase == 4 {
+            cluster.stop();
+            cluster.start().await?;
+        } else if survivors < 4 {
+            cluster.stop_node(survivors)?;
+        }
+        let write_ready = survivors >= 3;
+        let read_quorum = survivors >= 2;
+        let expected_status = if write_ready { 200 } else { 503 };
+        for (idx, client) in clients.iter().enumerate().take(survivors) {
+            let url = &cluster.nodes[idx].url;
+            let deadline = Instant::now() + Duration::from_secs(30);
+            // Poll health before issuing S3 I/O: idle remote disk handles must
+            // not remain evidence of quorum after their host becomes unreachable.
+            let payload = loop {
+                let response = http.get(format!("{url}/health/ready")).send().await?;
+                let status = response.status().as_u16();
+                let payload: serde_json::Value = response.json().await?;
+                if status == expected_status
+                    && payload["ready"] == write_ready
+                    && payload["details"]["storage"]["ready"] == write_ready
+                    && payload["details"]["storage"]["readQuorum"] == read_quorum
+                    && payload["details"]["storage"]["writeQuorum"] == write_ready
+                    && payload["details"]["poolMetadata"]["ready"] == true
+                    && payload["details"]["iam"]["ready"] == true
+                    && payload["details"]["lock"]["ready"] == write_ready
+                {
+                    break payload;
+                }
+                assert!(Instant::now() < deadline, "node {idx}, survivors={survivors}: HTTP {status}, {payload}");
+                tokio::time::sleep(Duration::from_millis(200)).await;
+            };
+            assert_eq!(payload["details"]["storage"]["readinessScope"], "write_quorum_and_pool_metadata");
+            assert_eq!(payload["details"]["storage"]["source"], "local_runtime");
+            assert_eq!(
+                payload["details"]["storage"]["status"],
+                if write_ready { "connected" } else { "disconnected" }
+            );
+            if !write_ready {
+                assert!(
+                    payload["degradedReasons"]
+                        .as_array()
+                        .expect("degraded reasons")
+                        .iter()
+                        .any(|reason| reason == "storage_and_lock_unavailable")
+                );
+            }
+
+            for path in ["/health/ready", "/minio/health/ready"] {
+                let head = http.head(format!("{url}{path}")).send().await?;
+                assert_eq!(head.status().as_u16(), expected_status, "HEAD {path}, survivors={survivors}");
+                assert!(head.bytes().await?.is_empty());
+                let response = http.get(format!("{url}{path}")).send().await?;
+                assert_eq!(response.status().as_u16(), expected_status);
+                let body: serde_json::Value = response.json().await?;
+                assert_eq!(body["details"]["storage"], payload["details"]["storage"]);
+                assert_eq!(body["details"]["poolMetadata"], payload["details"]["poolMetadata"]);
+            }
+            let live = http.get(format!("{url}/health/live")).send().await?;
+            assert_eq!(live.status().as_u16(), 200);
+            assert!(live.json::<serde_json::Value>().await?.get("details").is_none());
+            for (path, storage_ready, scope) in [
+                ("/minio/health/cluster", write_ready, "write_quorum_and_pool_metadata"),
+                ("/minio/health/cluster/read", read_quorum, "read_quorum"),
+            ] {
+                let deadline = Instant::now() + Duration::from_secs(30);
+                // Cluster read/write reports have independent caches; allow
+                // each observation to expire before comparing stable states.
+                let body = loop {
+                    let response = http.get(format!("{url}{path}")).send().await?;
+                    let status = response.status().as_u16();
+                    let body: serde_json::Value = response.json().await?;
+                    if status == expected_status
+                        && body["details"]["storage"]["ready"] == storage_ready
+                        && body["details"]["lock"]["ready"] == write_ready
+                    {
+                        break body;
+                    }
+                    assert!(Instant::now() < deadline, "{path}, survivors={survivors}: HTTP {status}, {body}");
+                    tokio::time::sleep(Duration::from_millis(200)).await;
+                };
+                assert_eq!(body["details"]["storage"]["readinessScope"], scope);
+            }
+
+            let put = client
+                .put_object()
+                .bucket(BUCKET)
+                .key(format!("readiness-phase-{phase}-node-{idx}"))
+                .body(Bytes::from_static(seed_body).into())
+                .send()
+                .await;
+            let put_status = if write_ready {
+                put.expect("a ready node must accept the PUT");
+                200
+            } else {
+                let error = put.expect_err("subquorum node must reject PUT");
+                assert!(
+                    error.raw_response().is_some_and(|response| response.status().as_u16() >= 500),
+                    "unexpected PUT failure: {error:?}"
+                );
+                error.raw_response().expect("PUT error response").status().as_u16()
+            };
+            let get = client.get_object().bucket(BUCKET).key(seed_key).send().await;
+            let get_status = match get {
+                Ok(object) => {
+                    assert_eq!(object.body.collect().await?.into_bytes().as_ref(), seed_body);
+                    200
+                }
+                Err(error) => {
+                    assert!(!write_ready, "GET must succeed on a ready cluster: {error:?}");
+                    let status = error
+                        .raw_response()
+                        .expect("GET should have an HTTP response")
+                        .status()
+                        .as_u16();
+                    assert!(status >= 500, "unexpected GET failure: {error:?}");
+                    status
+                }
+            };
+            let list = client.list_objects_v2().bucket(BUCKET).send().await;
+            let list_status = match list {
+                Ok(result) => {
+                    assert!(result.contents().iter().any(|object| object.key() == Some(seed_key)));
+                    200
+                }
+                Err(error) => {
+                    assert!(!write_ready, "listing must succeed on a ready cluster: {error:?}");
+                    let status = error
+                        .raw_response()
+                        .expect("LIST should have an HTTP response")
+                        .status()
+                        .as_u16();
+                    assert!(status >= 500, "unexpected listing failure: {error:?}");
+                    status
+                }
+            };
+            eprintln!(
+                "readiness matrix: survivors={survivors}, node={idx}, ready={write_ready}, read_quorum={read_quorum}, PUT={put_status}, GET={get_status}, LIST={list_status}"
+            );
+        }
+    }
+    cluster.stop();
     Ok(())
 }
 
@@ -125,6 +605,8 @@ async fn test_concurrent_cluster_overwrites_do_not_fail_namespace_lock_quorum()
 /// Before the fix, `map_namespace_lock_error` wrapped lock timeout/conflict errors as
 /// `StorageError::other(...)` → `StorageError::Io(...)`, which fell through to
 /// `S3ErrorCode::InternalError` (500) in the error mapping.
+/// Also checks PUT and multipart initialization when node failures prevent
+/// acquiring a table publication guard.
 #[tokio::test]
 async fn test_concurrent_put_same_key_never_returns_500() -> TestResult {
     crate::common::init_logging();
@@ -227,5 +709,6 @@ async fn test_concurrent_put_same_key_never_returns_500() -> TestResult {
     );
 
     clients[0].delete_object().bucket(BUCKET).key(KEY).send().await?;
-    Ok(())
+    cluster.stop();
+    assert_degraded_cluster_publication_guard_errors_are_retryable().await
 }

crates/e2e_test/src/on_demand_migration/interaction_test.rs

@@ -1137,7 +1137,12 @@ async fn test_odm_admin_config_is_redacted_and_status_counts_match_the_source()
         let miss = env.raw_get(bucket, miss_key).await?;
         assert_eq!(miss.status, 404, "{}", String::from_utf8_lossy(&miss.body));
     }
-    assert!(env.wait_local_listed(bucket, hit_key, SETTLE).await?);
+    let (listed, _, _) = tokio::try_join!(
+        env.wait_local_listed(bucket, hit_key, SETTLE),
+        env.wait_for_status_counter(bucket, "/counters/pulled_objects_total/inline", 1, SETTLE),
+        env.wait_for_status_counter(bucket, "/counters/pulled_bytes_total", body.len() as u64, SETTLE),
+    )?;
+    assert!(listed);
 
     let status = env.status_json(bucket).await?;
     assert_eq!(status.pointer("/configured").and_then(Value::as_bool), Some(true), "{status}");

crates/e2e_test/src/storage_metric_ownership_test.rs

@@ -0,0 +1,335 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Native OTLP -> Collector -> Prometheus contract, including a rolling upgrade.
+
+use crate::common::RustFSTestClusterEnvironment;
+use aws_sdk_s3::primitives::ByteStream;
+use serde_json::Value;
+use std::fs::{self, File};
+use std::net::TcpListener;
+use std::path::{Path, PathBuf};
+use std::process::{Child, Command, Stdio};
+use std::time::{Duration, Instant};
+
+type TestResult<T = ()> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
+
+struct ToolProcess(Child);
+impl Drop for ToolProcess {
+    fn drop(&mut self) {
+        let _ = self.0.kill();
+        let _ = self.0.wait();
+    }
+}
+
+fn required_binary(name: &str) -> TestResult<PathBuf> {
+    let path = PathBuf::from(std::env::var(name).map_err(|_| format!("{name} must name a pinned executable"))?);
+    if !path.is_file() {
+        return Err(format!("{name} does not name a file: {}", path.display()).into());
+    }
+    Ok(path)
+}
+
+fn free_port() -> TestResult<u16> {
+    Ok(TcpListener::bind("127.0.0.1:0")?.local_addr()?.port())
+}
+
+fn start_tool(binary: &Path, args: &[String], log: &Path) -> TestResult<ToolProcess> {
+    let log = File::create(log)?;
+    Ok(ToolProcess(
+        Command::new(binary)
+            .args(args)
+            .env("NO_PROXY", "127.0.0.1,localhost")
+            .env_remove("HTTP_PROXY")
+            .env_remove("HTTPS_PROXY")
+            .stdout(Stdio::from(log.try_clone()?))
+            .stderr(Stdio::from(log))
+            .spawn()?,
+    ))
+}
+
+async fn query(client: &reqwest::Client, base: &str, expression: &str) -> TestResult<Value> {
+    let mut url = reqwest::Url::parse(&format!("{base}/api/v1/query"))?;
+    url.query_pairs_mut().append_pair("query", expression);
+    let result: Value = client.get(url).send().await?.error_for_status()?.json().await?;
+    if result["status"] != "success" {
+        return Err(format!("PromQL failed: {result}").into());
+    }
+    Ok(result["data"]["result"].clone())
+}
+
+async fn await_count(client: &reqwest::Client, base: &str, selector: &str, expected: u64) -> TestResult {
+    let deadline = Instant::now() + Duration::from_secs(120);
+    loop {
+        let result = query(client, base, &format!("count({selector}) or vector(0)")).await;
+        if let Ok(rows) = &result
+            && rows[0]["value"][1].as_str().and_then(|value| value.parse::<u64>().ok()) == Some(expected)
+        {
+            println!("PASS count={expected}: {selector}");
+            return Ok(());
+        }
+        if Instant::now() >= deadline {
+            return Err(format!("expected {expected} for {selector}; last result: {result:?}").into());
+        }
+        tokio::time::sleep(Duration::from_millis(250)).await;
+    }
+}
+
+async fn validate_dashboard_queries(client: &reqwest::Client, base: &str, observer: &str) -> TestResult {
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../.docker/observability/grafana/dashboards/rustfs.json");
+    let dashboard: Value = serde_json::from_str(&fs::read_to_string(path)?)?;
+    for name in ["storage_cluster", "storage_observer"] {
+        let variable = dashboard["templating"]["list"]
+            .as_array()
+            .ok_or("dashboard variables")?
+            .iter()
+            .find(|variable| variable["name"] == name)
+            .ok_or("storage selection variable")?;
+        assert_eq!(variable["multi"], false, "storage views must select one {name}");
+        assert_eq!(variable["includeAll"], false, "storage views must select one {name}");
+    }
+    let mut pending = dashboard["panels"]
+        .as_array()
+        .ok_or("dashboard panels")?
+        .iter()
+        .collect::<Vec<_>>();
+    let mut checked = 0;
+    while let Some(panel) = pending.pop() {
+        if let Some(children) = panel["panels"].as_array() {
+            pending.extend(children);
+        }
+        for target in panel["targets"].as_array().into_iter().flatten() {
+            let Some(expression) = target["expr"].as_str() else { continue };
+            if !expression.contains("rustfs:storage:current") {
+                continue;
+            }
+            if expression.contains("collection_scope=\"cluster\"") {
+                assert!(
+                    expression.contains("observer=\"$storage_observer\""),
+                    "global views must select one observer: {expression}"
+                );
+            }
+            let mut expression = expression.to_string();
+            for (name, value) in [
+                ("$__rate_interval", "5m"),
+                ("$storage_cluster", "metrics-e2e"),
+                ("$storage_observer", observer),
+                ("$drive_api", ".*"),
+                ("$server", ".*"),
+                ("$drive", ".*"),
+                ("$job", "rustfs"),
+            ] {
+                expression = expression.replace(name, value);
+            }
+            query(client, base, &expression).await?;
+            checked += 1;
+        }
+    }
+    assert!(checked > 0, "the storage dashboard queries must be exercised");
+    println!("PASS: {checked} storage dashboard queries against the live pipeline");
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[ignore = "external tools: pinned Collector, Prometheus, previous release and current RustFS binaries"]
+async fn storage_metric_ownership_pipeline() -> TestResult {
+    let baseline = required_binary("RUSTFS_METRICS_BASELINE_BINARY")?;
+    let current = required_binary("CARGO_BIN_EXE_rustfs")?;
+    let collector = required_binary("RUSTFS_OTELCOL_BINARY")?;
+    let prometheus = required_binary("RUSTFS_PROMETHEUS_BINARY")?;
+    let temp = tempfile::Builder::new().prefix("rustfs-storage-metrics-").tempdir()?;
+    let work = if let Ok(path) = std::env::var("RUSTFS_METRICS_E2E_ARTIFACTS") {
+        let path = PathBuf::from(path);
+        fs::create_dir_all(&path)?;
+        tempfile::Builder::new().prefix("storage-run-").tempdir_in(path)?.keep()
+    } else {
+        temp.path().to_path_buf()
+    };
+    println!("Metrics pipeline logs: {}", work.display());
+    let otlp = free_port()?;
+    let scrape = free_port()?;
+    let prom = free_port()?;
+    let collector_config = work.join("collector.yaml");
+    fs::write(
+        &collector_config,
+        format!(
+            r#"receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: 127.0.0.1:{otlp}
+exporters:
+  prometheus:
+    endpoint: 127.0.0.1:{scrape}
+    send_timestamps: true
+    metric_expiration: 5m
+    resource_to_telemetry_conversion:
+      enabled: true
+service:
+  telemetry:
+    metrics:
+      level: none
+    logs:
+      level: warn
+  pipelines:
+    metrics:
+      receivers: [otlp]
+      exporters: [prometheus]
+"#
+        ),
+    )?;
+    let rules = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("../../.docker/observability/prometheus-rules/rustfs-storage.yml")
+        .canonicalize()?;
+    let prom_config = work.join("prometheus.yaml");
+    fs::write(
+        &prom_config,
+        format!(
+            r#"global:
+  scrape_interval: 1s
+  evaluation_interval: 1s
+rule_files:
+  - '{}'
+scrape_configs:
+  - job_name: rustfs
+    static_configs:
+      - targets: ['127.0.0.1:{scrape}']
+"#,
+            rules.display()
+        ),
+    )?;
+    // Use the shipped expressions, with only the test evaluation interval shortened.
+    let test_rules = work.join("storage-rules.yaml");
+    fs::write(&test_rules, fs::read_to_string(&rules)?.replace("interval: 15s", "interval: 1s"))?;
+    fs::write(
+        &prom_config,
+        fs::read_to_string(&prom_config)?.replace(&rules.display().to_string(), &test_rules.display().to_string()),
+    )?;
+    let _collector = start_tool(
+        &collector,
+        &[format!("--config={}", collector_config.display())],
+        &work.join("collector.log"),
+    )?;
+    let _prometheus = start_tool(
+        &prometheus,
+        &[
+            format!("--config.file={}", prom_config.display()),
+            format!("--web.listen-address=127.0.0.1:{prom}"),
+            format!("--storage.tsdb.path={}", work.join("prometheus-data").display()),
+        ],
+        &work.join("prometheus.log"),
+    )?;
+    let http = reqwest::Client::builder()
+        .no_proxy()
+        .timeout(Duration::from_secs(5))
+        .build()?;
+    let prom_url = format!("http://127.0.0.1:{prom}");
+    await_count(&http, &prom_url, "up{job=\"rustfs\"} == 1", 1).await?;
+
+    let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
+    cluster.set_env("NO_PROXY", "127.0.0.1,localhost");
+    cluster.set_env("RUSTFS_OBS_METRIC_ENDPOINT", format!("http://127.0.0.1:{otlp}/v1/metrics"));
+    cluster.set_env("OTEL_RESOURCE_ATTRIBUTES", "rustfs.cluster.id=metrics-e2e");
+    cluster.set_env("RUSTFS_OBS_METER_INTERVAL", "2");
+    cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "true");
+    cluster.set_env("RUSTFS_OBS_LOGS_EXPORT_ENABLED", "false");
+    cluster.set_env("RUSTFS_OBS_TRACES_EXPORT_ENABLED", "false");
+    cluster.set_env("RUSTFS_METRICS_NODE_INTERVAL", "2");
+    cluster.set_env("RUSTFS_METRICS_CLUSTER_INTERVAL", "5");
+    for index in 0..4 {
+        // Four localhost processes otherwise share the startup resource IP.
+        // Distinct test host IDs model the four hosts in a distributed deployment.
+        cluster.set_node_env(
+            index,
+            "OTEL_RESOURCE_ATTRIBUTES",
+            format!("rustfs.cluster.id=metrics-e2e,host.id=metrics-node-{index}"),
+        )?;
+        cluster.set_node_capture_log_path(index, work.join(format!("node-{index}.log")).display().to_string())?;
+    }
+    cluster.start_with_binary(&baseline).await?;
+    // Reproduce the original four observers x four global drives before fixing it.
+    await_count(&http, &prom_url, "rustfs_system_drive_total_bytes{collection_scope=\"\",drive!=\"\"}", 16).await?;
+    let local = "rustfs:storage:current{source_metric=\"rustfs_system_drive_total_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"metrics-e2e\"}";
+    for index in 0..4 {
+        cluster.stop_node_gracefully(index).await?;
+        cluster.start_node_from_binary(index, &current).await?;
+        await_count(&http, &prom_url, local, u64::try_from(index + 1)?).await?;
+    }
+    let rows = query(&http, &prom_url, local).await?;
+    for row in rows.as_array().ok_or("expected a metric vector")? {
+        assert_eq!(
+            row["metric"]["observer"], row["metric"]["server"],
+            "a node must only export its own detailed drives"
+        );
+    }
+    let observer = cluster.nodes[0].address.clone();
+    let inventory = format!(
+        "rustfs:storage:current{{source_metric=\"rustfs_cluster_drive_present\",collection_scope=\"cluster\",rustfs_cluster_id=\"metrics-e2e\",observer=\"{observer}\"}}"
+    );
+    await_count(&http, &prom_url, &inventory, 4).await?;
+    cluster.create_test_bucket("metrics-ownership").await?;
+    let client = cluster.create_s3_client(0)?;
+    for index in 0..8 {
+        client
+            .put_object()
+            .bucket("metrics-ownership")
+            .key(format!("object-{index}"))
+            .body(ByteStream::from(vec![7_u8; 4096]))
+            .send()
+            .await?;
+    }
+    await_count(
+        &http,
+        &prom_url,
+        "count by (server) (rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\"})",
+        4,
+    ).await?;
+    let counters = query(
+        &http,
+        &prom_url,
+        "rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\"}",
+    )
+    .await?;
+    assert!(
+        !counters.as_array().ok_or("expected counters")?.is_empty(),
+        "exercise actual storage counters"
+    );
+    for row in counters.as_array().ok_or("expected counters")? {
+        assert_eq!(row["metric"]["observer"], row["metric"]["server"]);
+        assert!(
+            !row["metric"]["disk_id"].as_str().unwrap_or_default().is_empty(),
+            "counters must carry physical disk identity"
+        );
+    }
+    validate_dashboard_queries(&http, &prom_url, &observer).await?;
+    for index in (1..4).rev() {
+        cluster.stop_node(index)?;
+        // The Collector stays alive; cached samples must not keep stopped owners fresh.
+        await_count(&http, &prom_url, local, u64::try_from(index)?).await?;
+        await_count(&http, &prom_url, &inventory, 4).await?;
+        let unavailable = format!(
+            "rustfs:storage:current{{source_metric=\"rustfs_cluster_drive_runtime_state\",collection_scope=\"cluster\",rustfs_cluster_id=\"metrics-e2e\",observer=\"{observer}\",state=~\"offline|unknown|suspect\"}} == 1"
+        );
+        await_count(&http, &prom_url, &unavailable, u64::try_from(4 - index)?).await?;
+    }
+    cluster.stop();
+    await_count(&http, &prom_url, local, 0).await?;
+    cluster.start_with_binary(&current).await?;
+    await_count(&http, &prom_url, local, 4).await?;
+    await_count(&http, &prom_url, &inventory, 4).await?;
+    let restored = client.get_object().bucket("metrics-ownership").key("object-0").send().await?;
+    assert_eq!(restored.body.collect().await?.into_bytes().as_ref(), vec![7_u8; 4096].as_slice());
+    println!("PASS: baseline duplication; four rolling upgrades; owner identity; counters; 4 -> 3 -> 2 -> 1 -> 0 -> 4 recovery");
+    Ok(())
+}

crates/ecstore/src/api/mod.rs

@@ -38,6 +38,16 @@ pub mod bucket {
     }
 
     pub mod lifecycle {
+        pub mod legacy_transition_state_reconcile {
+            pub use crate::bucket::lifecycle::legacy_transition_state_reconcile::{
+                LegacyTransitionStateCopyRepresentation, LegacyTransitionStateMetadataAlias, LegacyTransitionStateReconcileError,
+                LegacyTransitionStateReconcileOutcome, LegacyTransitionStateReconcileReadiness,
+                LegacyTransitionStateReconcileRequest, LegacyTransitionStateReconcileResponse,
+                LegacyTransitionStateReconcileSelector, LegacyTransitionStateSetRepresentation, LegacyTransitionStateSource,
+                LegacyTransitionStateTarget,
+            };
+        }
+
         pub mod bucket_lifecycle_audit {
             pub use crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
         }
@@ -260,18 +270,19 @@ pub mod bucket {
             ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
             ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
             ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
-            ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
-            VersionPurgeStatusType, XferStats, assign_site_replication_rule_priorities, commit_force_delete_intent,
-            complete_force_delete_intent, delete_replication_state_from_config, delete_replication_version_id,
-            get_global_replication_pool, get_global_replication_stats, get_proxy_targets, init_background_replication,
+            ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, ScannerDirtyUsageMutationObserver,
+            ScannerDirtyUsageMutationSource, TargetReplicationResyncStatus, VersionPurgeStatusType, XferStats,
+            assign_site_replication_rule_priorities, commit_force_delete_intent, complete_force_delete_intent,
+            delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
+            get_global_replication_stats, get_proxy_targets, init_background_replication,
             invalid_replication_config_status_field, is_site_replication_role, is_site_replication_rule,
             merge_incoming_replication_config, merge_user_replication_config, persist_force_delete_intent,
             read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map,
             replication_target_arn_deployment_id, replication_target_arns, resync_start_conflict_id,
-            should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info,
-            should_use_existing_delete_replication_source, site_replication_rule_deployment_id,
-            unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
-            version_purge_status_to_filemeta,
+            set_scanner_dirty_usage_mutation_observer, should_remove_replication_target, should_schedule_delete_replication,
+            should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
+            site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
+            validate_replication_config_target_arns, version_purge_status_to_filemeta,
         };
     }
 
@@ -366,6 +377,14 @@ pub mod config {
 }
 
 pub mod data_usage {
+    #[cfg(feature = "test-util")]
+    pub use crate::data_movement::SourceCleanupDeleteBarrier;
+    #[cfg(feature = "test-util")]
+    pub use crate::data_movement::scanner_backlog::test_util::NativeScannerPauseBacklogWriteFault;
+    pub use crate::data_movement::scanner_backlog::{
+        MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementPlan, ScannerPauseBacklogRetirementPlanner,
+        ScannerPauseBacklogRetirementReplica, register_scanner_pause_backlog_retirement_planner,
+    };
     pub use crate::data_usage::{
         DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
         init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -966,6 +966,11 @@ async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel:
     if let Some(err) = first_error {
         return Err(err);
     }
+    runtime_sources::notify_scanner_dirty_usage_mutation(
+        &oi.bucket,
+        &oi.name,
+        runtime_sources::ScannerDirtyUsageMutationSource::TierExpiration,
+    );
     Ok(true)
 }
 
@@ -4749,6 +4754,11 @@ async fn expire_transitioned_object_with_lock_lost_signal(
                 // Drop any cached restored-copy body so it does not sit resident
                 // until TTL after the copy is expired (ODC-26).
                 crate::object_api::notify_object_mutation(&oi.bucket, &oi.name).await;
+                runtime_sources::notify_scanner_dirty_usage_mutation(
+                    &oi.bucket,
+                    &oi.name,
+                    runtime_sources::ScannerDirtyUsageMutationSource::TierExpiration,
+                );
                 //audit_log_lifecycle(*oi, ILMExpiry, tags, traceFn);
                 Ok(dobj)
             }
@@ -4784,6 +4794,11 @@ async fn expire_transitioned_object_with_lock_lost_signal(
     // The transitioned version is gone; evict any cached body for this object
     // so it does not linger until TTL (ODC-26).
     crate::object_api::notify_object_mutation(&oi.bucket, &oi.name).await;
+    runtime_sources::notify_scanner_dirty_usage_mutation(
+        &oi.bucket,
+        &oi.name,
+        runtime_sources::ScannerDirtyUsageMutationSource::TierExpiration,
+    );
 
     //audit_log_lifecycle(oi, ILMExpiry, tags);
 

crates/ecstore/src/bucket/lifecycle/mod.rs

@@ -18,6 +18,7 @@ mod config_boundary;
 pub mod core;
 mod durable_namespace;
 pub mod evaluator;
+pub mod legacy_transition_state_reconcile;
 pub mod manual_transition_job;
 mod metadata_boundary;
 pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs, get_lifecycle_config};

crates/ecstore/src/bucket/lifecycle/runtime_boundary.rs

@@ -20,6 +20,7 @@ use tokio_util::sync::CancellationToken;
 
 use crate::bucket::lifecycle::bucket_lifecycle_ops::{ExpiryState, TransitionState};
 use crate::runtime::sources;
+pub(crate) use crate::runtime::sources::ScannerDirtyUsageMutationSource;
 use crate::services::tier::tier::TierConfigMgr;
 use crate::store::ECStore;
 
@@ -54,3 +55,7 @@ pub(crate) fn deployment_id() -> Option<String> {
 pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<BucketLifecycleConfiguration> {
     sources::bucket_lifecycle_config(bucket).await
 }
+
+pub(crate) fn notify_scanner_dirty_usage_mutation(bucket: &str, object: &str, source: ScannerDirtyUsageMutationSource) {
+    sources::notify_scanner_dirty_usage_mutation(bucket, object, source);
+}

crates/ecstore/src/bucket/metadata_sys.rs

@@ -1681,9 +1681,13 @@ impl BucketMetadataSys {
         expected: Option<&Arc<BucketMetadata>>,
         namespace_guard: &rustfs_lock::NamespaceLockGuard,
     ) -> Result<()> {
-        if !self
-            .bucket_exists(bucket, namespace_guard, "bucket metadata existence check")
-            .await?
+        if !await_bucket_namespace_operation(
+            Some(namespace_guard),
+            bucket,
+            "bucket metadata heal existence check",
+            self.api.bucket_exists_for_heal(bucket),
+        )
+        .await?
         {
             if matches!(mode, MetadataLoadMode::Refresh) {
                 let _publish_guard = self

crates/ecstore/src/bucket/replication/mod.rs

@@ -92,3 +92,6 @@ pub use replication_target_boundary::SsecPassthroughCapability;
 pub use replication_target_boundary::VersionIdentityCapability;
 pub use replication_target_boundary::{ObjectLockIntegrity, object_lock_put_integrity};
 pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
+pub use runtime_boundary::{
+    ScannerDirtyUsageMutationObserver, ScannerDirtyUsageMutationSource, set_scanner_dirty_usage_mutation_observer,
+};

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -3843,6 +3843,13 @@ async fn persist_replication_state_if_current<S: ReplicationStorage>(
     match storage.put_object_metadata(&roi.bucket, &roi.name, &write_opts).await {
         Ok(updated) => {
             *object_info = updated;
+            if mode == ReplicationStatusWritebackMode::Update {
+                runtime_sources::notify_scanner_dirty_usage_mutation(
+                    &roi.bucket,
+                    &roi.name,
+                    runtime_sources::ScannerDirtyUsageMutationSource::Replication,
+                );
+            }
             Ok(ReplicationStatePersistOutcome::Updated)
         }
         Err(Error::PreconditionFailed) => Ok(ReplicationStatePersistOutcome::Superseded),

crates/ecstore/src/bucket/replication/runtime_boundary.rs

@@ -19,6 +19,9 @@ use super::replication_pool::DynReplicationPool;
 use super::replication_state::ReplicationStats;
 use super::replication_storage_boundary::ReplicationObjectStore;
 use crate::runtime::sources;
+pub use crate::runtime::sources::{
+    ScannerDirtyUsageMutationObserver, ScannerDirtyUsageMutationSource, set_scanner_dirty_usage_mutation_observer,
+};
 
 pub(crate) fn object_store_handle() -> Option<Arc<ReplicationObjectStore>> {
     sources::object_store_handle()
@@ -43,3 +46,7 @@ pub(crate) fn replication_runtime_initialized() -> bool {
 pub(crate) fn bucket_monitor() -> Option<Arc<ReplicationBucketMonitor>> {
     sources::bucket_monitor()
 }
+
+pub(crate) fn notify_scanner_dirty_usage_mutation(bucket: &str, object: &str, source: ScannerDirtyUsageMutationSource) {
+    sources::notify_scanner_dirty_usage_mutation(bucket, object, source);
+}

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -51,10 +51,10 @@ use rustfs_protos::proto_gen::node_service::{
     LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ReplacementRecoveryStatusRequest,
     ScannerActivityRequest, ScannerActivityResponse, ScannerDirtyUsageSnapshotRequest, ScannerDirtyUsageSnapshotResponse,
     ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse,
-    ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageEntry, ServerInfoRequest, SignalServiceRequest,
-    SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, TierDailyStatsRequest,
-    TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse, TierMutationFailureClass,
-    TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
+    ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse, ScannerScopedDirtyUsageEntry, ServerInfoRequest,
+    SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest,
+    TierDailyStatsRequest, TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse,
+    TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
     tier_mutation_control_service_client::TierMutationControlServiceClient,
 };
 pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
@@ -289,6 +289,20 @@ fn scanner_scoped_dirty_usage_ack_payload(
     Ok(payload)
 }
 
+fn scanner_scoped_dirty_usage_ack_response_matches(
+    request: &ScannerScopedDirtyUsageAckRequest,
+    response: &ScannerScopedDirtyUsageAckResponse,
+) -> bool {
+    let cleared_within_request = u64::try_from(request.entries.len())
+        .is_ok_and(|entry_count| response.cleared <= entry_count && (!request.probe_only || response.cleared == 0));
+    response.protocol_version == rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
+        && response.owner_id == request.owner_id
+        && response.instance_id == request.instance_id
+        && response.max_entries == rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_ENTRIES
+        && response.max_request_bytes == rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
+        && cleared_within_request
+}
+
 fn scanner_scoped_dirty_usage_ack_reconciled(activity: &ScannerPeerActivity, expected_instance_id: &str) -> bool {
     activity.instance_id == expected_instance_id && activity.dirty_usage_pending == Some(false)
 }
@@ -2255,13 +2269,7 @@ impl PeerRestClient {
                     let body = canonical_scoped_dirty_usage_response(&canonical, &response)
                         .map_err(|_| Error::other("scoped dirty usage capability response is too large"))?;
                     verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
-                    if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
-                        || response.owner_id != payload.owner_id
-                        || response.instance_id != payload.instance_id
-                        || response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
-                        || response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
-                        || response.cleared != 0
-                    {
+                    if !scanner_scoped_dirty_usage_ack_response_matches(&payload, &response) {
                         return Err(Error::other("scoped dirty usage capability response does not match request"));
                     }
                     if !response.supported {
@@ -2297,13 +2305,7 @@ impl PeerRestClient {
                 let body = canonical_scoped_dirty_usage_response(&canonical, &response)
                     .map_err(|_| Error::other("scoped dirty usage acknowledgement response is too large"))?;
                 verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
-                if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
-                    || response.owner_id != payload.owner_id
-                    || response.instance_id != payload.instance_id
-                    || response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
-                    || response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
-                    || !response.supported
-                {
+                if !scanner_scoped_dirty_usage_ack_response_matches(&payload, &response) || !response.supported {
                     return Err(Error::other("scoped dirty usage acknowledgement response does not match request"));
                 }
             }
@@ -3118,6 +3120,60 @@ mod tests {
         }
     }
 
+    #[test]
+    fn scanner_scoped_dirty_usage_ack_response_bounds_cleared_entries_to_request() {
+        use rustfs_protos::scoped_dirty_usage::{
+            SCOPED_DIRTY_USAGE_MAX_ENTRIES, SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES, SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
+        };
+
+        let mut request = scanner_scoped_dirty_usage_ack_payload(
+            "33333333-3333-3333-3333-333333333333",
+            "0123456789abcdef0123456789abcdef",
+            false,
+            vec![
+                ScannerScopedDirtyUsageEntry {
+                    bucket: "archive".to_string(),
+                    bucket_incarnation: Uuid::from_u128(0x11111111111111111111111111111111).as_bytes().to_vec().into(),
+                    generation: 3,
+                },
+                ScannerScopedDirtyUsageEntry {
+                    bucket: "photos".to_string(),
+                    bucket_incarnation: Uuid::from_u128(0x22222222222222222222222222222222).as_bytes().to_vec().into(),
+                    generation: 7,
+                },
+            ],
+        )
+        .expect("two ordered entries should form a valid scoped ACK request");
+        let mut response = ScannerScopedDirtyUsageAckResponse {
+            protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
+            owner_id: request.owner_id.clone(),
+            instance_id: request.instance_id.clone(),
+            supported: true,
+            max_entries: SCOPED_DIRTY_USAGE_MAX_ENTRIES,
+            max_request_bytes: SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES,
+            cleared: 1,
+            response_proof: Bytes::new(),
+        };
+
+        assert!(scanner_scoped_dirty_usage_ack_response_matches(&request, &response));
+        response.cleared = 2;
+        assert!(scanner_scoped_dirty_usage_ack_response_matches(&request, &response));
+        response.cleared = 3;
+        assert!(
+            !scanner_scoped_dirty_usage_ack_response_matches(&request, &response),
+            "a peer cannot clear more entries than the signed request contains"
+        );
+
+        request.probe_only = true;
+        response.cleared = 1;
+        assert!(
+            !scanner_scoped_dirty_usage_ack_response_matches(&request, &response),
+            "a capability probe cannot report a mutation"
+        );
+        response.cleared = 0;
+        assert!(scanner_scoped_dirty_usage_ack_response_matches(&request, &response));
+    }
+
     #[test]
     fn scanner_scoped_dirty_usage_ack_reconciliation_requires_same_clean_instance() {
         let activity = |instance_id: &str, pending| ScannerPeerActivity {

crates/ecstore/src/core/pools_test.rs

@@ -5256,16 +5256,35 @@ mod decommission_lock_order_tests {
 
     #[test]
     #[serial_test::serial]
-    fn scanner_backlog_native_replica_reconciles_capacity_and_cleans_source() {
-        run_large_stack_current_thread_async_test("scanner-backlog-reconcile", async || {
+    fn data_movement_existing_replica_reconciles_capacity_and_cleans_source() {
+        data_movement_existing_replica_reconciles_capacity_case(false);
+    }
+
+    #[test]
+    #[serial_test::serial]
+    fn data_movement_existing_replica_outside_reservation_uses_reserved_target() {
+        data_movement_existing_replica_reconciles_capacity_case(true);
+    }
+
+    fn data_movement_existing_replica_reconciles_capacity_case(existing_outside_reservation: bool) {
+        run_large_stack_current_thread_async_test("reserved-replica-reconcile", async move || {
             let (_temp_dirs, store, other_store) =
                 test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await;
-            let object = "buckets/.scanner-pause-backlog.json";
+            let object = "buckets/reserved-replica-routing.json";
             let body = br#"{"schemaVersion":1,"generation":2}"#.to_vec();
             let old_body = br#"{"schemaVersion":1,"generation":1}"#.to_vec();
             let source_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(20);
-            let target_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(10);
-            for (pool_index, payload, mod_time) in [(0, body.clone(), source_time), (2, old_body, target_time)] {
+            let target_time = source_time;
+            let target_pool_index = if existing_outside_reservation { 1 } else { 2 };
+            let mut replicas = vec![(0, body.clone(), source_time), (target_pool_index, old_body, target_time)];
+            if existing_outside_reservation {
+                replicas.push((
+                    2,
+                    br#"{"schemaVersion":1,"generation":3}"#.to_vec(),
+                    source_time + time::Duration::seconds(10),
+                ));
+            }
+            for (pool_index, payload, mod_time) in replicas.iter().cloned() {
                 store.pools[pool_index]
                     .put_object(
                         RUSTFS_META_BUCKET,
@@ -5278,13 +5297,19 @@ mod decommission_lock_order_tests {
                         },
                     )
                     .await
-                    .expect("seed native scanner replicas with independent write times");
+                    .expect("seed existing replicas with independent write times");
             }
             let layout = DecommissionErasureLayout { data: 1, parity: 0 };
             let target_total = body.len() * 8;
             let capacities = vec![
                 DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len() * 2, body.len() * 2),
-                DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
+                DecommissionPoolCapacityInfo::for_test(
+                    1,
+                    layout,
+                    if existing_outside_reservation { target_total } else { 0 },
+                    target_total,
+                    if existing_outside_reservation { 0 } else { target_total },
+                ),
                 DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
             ];
             set_decommission_capacity_info_overrides_for_test(store.id, vec![capacities.clone()]);
@@ -5293,6 +5318,17 @@ mod decommission_lock_order_tests {
                 .await
                 .expect("activate the source reservation");
             let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
+            let reserved_snapshot = store.pool_meta.read().await.clone();
+            let reservation = reserved_snapshot.pools[0]
+                .decommission
+                .as_ref()
+                .and_then(|info| info.capacity_reservation.as_ref())
+                .expect("active source reservation");
+            assert_eq!(
+                reservation.targets.iter().map(|target| target.pool_index).collect::<Vec<_>>(),
+                vec![target_pool_index],
+                "the fixture must reserve exactly one target"
+            );
             let source_reader = store.pools[0]
                 .get_object_reader(
                     RUSTFS_META_BUCKET,
@@ -5314,12 +5350,91 @@ mod decommission_lock_order_tests {
                 RUSTFS_META_BUCKET.to_string(),
                 source_reader,
                 None,
-                "scanner_backlog_conflict",
+                "reserved_replica_conflict",
                 Some(owner),
             )
             .await
-            .expect_err("a different older native ledger must retain its source and capacity intent");
+            .expect_err("a different older existing record must retain its source and capacity intent");
             assert!(conflict.to_string().contains("Precondition failed"), "unexpected conflict: {conflict}");
+            let reserved_snapshot = store.pool_meta.read().await.clone();
+            let mut selection_opts = ObjectOptions {
+                data_movement: true,
+                src_pool_idx: 0,
+                ..Default::default()
+            };
+            assert_eq!(
+                store
+                    .select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &selection_opts, true)
+                    .await
+                    .expect("selection without a capacity owner retains existing-replica routing"),
+                2
+            );
+            owner.apply_to(&mut selection_opts);
+            for stale_owner in [
+                DecommissionCapacityOwner {
+                    owner_nonce: uuid::Uuid::new_v4(),
+                    ..owner
+                },
+                DecommissionCapacityOwner {
+                    generation: owner.generation + 1,
+                    ..owner
+                },
+            ] {
+                let mut stale_opts = selection_opts.clone();
+                stale_owner.apply_to(&mut stale_opts);
+                assert!(
+                    matches!(
+                        store
+                            .select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &stale_opts, true)
+                            .await,
+                        Err(crate::error::Error::DecommissionCapacityBlocked { .. })
+                    ),
+                    "a stale owner must not fall back to another target"
+                );
+            }
+            {
+                let mut meta = store.pool_meta.write().await;
+                meta.pools[0]
+                    .decommission
+                    .as_mut()
+                    .unwrap()
+                    .capacity_reservation
+                    .as_mut()
+                    .unwrap()
+                    .expires_at = time::OffsetDateTime::now_utc() - time::Duration::seconds(1);
+            }
+            assert!(
+                matches!(
+                    store
+                        .select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &selection_opts, true)
+                        .await,
+                    Err(crate::error::Error::DecommissionCapacityBlocked { .. })
+                ),
+                "an expired owner must not fall back to another target"
+            );
+            *store.pool_meta.write().await = reserved_snapshot.clone();
+            if !existing_outside_reservation {
+                {
+                    let mut meta = store.pool_meta.write().await;
+                    let target = &mut meta.pools[0]
+                        .decommission
+                        .as_mut()
+                        .unwrap()
+                        .capacity_reservation
+                        .as_mut()
+                        .unwrap()
+                        .targets[0];
+                    target.consumed_physical_bytes = target.reserved_physical_bytes;
+                }
+                assert_eq!(
+                    store
+                        .select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &selection_opts, true)
+                        .await
+                        .expect("an existing reserved replica can still be selected after capacity was consumed"),
+                    target_pool_index
+                );
+                *store.pool_meta.write().await = reserved_snapshot;
+            }
             let mut persisted = crate::core::pools::PoolMeta::default();
             persisted
                 .load_no_lock_from_replicas(store.pools.clone())
@@ -5336,11 +5451,24 @@ mod decommission_lock_order_tests {
                     .pending_target_physical_bytes,
                 body.len()
             );
-            let previous = store.pools[2]
+            for (pool_index, payload, mod_time) in &replicas {
+                let mut reader = store.pools[*pool_index]
+                    .get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
+                    .await
+                    .expect("a refused existing record replacement must preserve every replica");
+                assert_eq!(reader.object_info.mod_time, Some(*mod_time));
+                let mut actual = Vec::new();
+                reader
+                    .read_to_end(&mut actual)
+                    .await
+                    .expect("read the unchanged existing record");
+                assert_eq!(&actual, payload);
+            }
+            let previous = store.pools[target_pool_index]
                 .get_object_info(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
                 .await
-                .expect("read the native writer's CAS revision");
-            let replacement = store.pools[2]
+                .expect("read the existing writer's CAS revision");
+            let replacement = store.pools[target_pool_index]
                 .put_object(
                     RUSTFS_META_BUCKET,
                     object,
@@ -5356,7 +5484,7 @@ mod decommission_lock_order_tests {
                     },
                 )
                 .await
-                .expect("native scanner CAS converges the payload without a migration marker");
+                .expect("existing CAS converges the payload without a migration marker");
             assert!(!data_movement::is_owned_data_movement_target(&replacement));
             *other_store.pool_meta.write().await = persisted;
             set_decommission_capacity_info_overrides_for_test(other_store.id, vec![capacities]);
@@ -5374,7 +5502,7 @@ mod decommission_lock_order_tests {
             )
             .await
             .expect("replica conflict recovery must be bounded")
-            .expect("identical native replica should finish migration on the reloaded node");
+            .expect("identical existing replica should finish migration on the reloaded node");
             let mut reconciled = crate::core::pools::PoolMeta::default();
             reconciled
                 .load_no_lock_from_replicas(other_store.pools.clone())
@@ -5404,14 +5532,14 @@ mod decommission_lock_order_tests {
                 .await
                 .expect_err("the source should be cleaned only after equivalent-target capacity reconciliation");
             assert!(crate::error::is_err_object_not_found(&missing));
-            let mut target_reader = other_store.pools[2]
+            let mut target_reader = other_store.pools[target_pool_index]
                 .get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
                 .await
                 .expect("the surviving replica should remain readable");
             assert_eq!(
                 target_reader.object_info.mod_time,
                 Some(target_time),
-                "recovery must not overwrite the native target"
+                "recovery must not overwrite the existing target"
             );
             let mut actual = Vec::new();
             target_reader
@@ -5419,6 +5547,20 @@ mod decommission_lock_order_tests {
                 .await
                 .expect("read surviving ledger bytes");
             assert_eq!(actual, body);
+            if existing_outside_reservation {
+                let (_, outside_body, outside_time) = replicas.last().expect("unreserved existing replica");
+                let mut outside = other_store.pools[2]
+                    .get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
+                    .await
+                    .expect("migration must leave the unreserved existing replica intact");
+                assert_eq!(outside.object_info.mod_time, Some(*outside_time));
+                let mut actual = Vec::new();
+                outside
+                    .read_to_end(&mut actual)
+                    .await
+                    .expect("read the untouched unreserved replica");
+                assert_eq!(&actual, outside_body);
+            }
         });
     }
 

crates/ecstore/src/data_movement/mod.rs

@@ -15,6 +15,7 @@
 // #730: data-movement migration keeps staged cleanup helpers until copy paths converge.
 
 pub(crate) mod backpressure;
+pub(crate) mod scanner_backlog;
 
 use crate::core::pools::{DecommissionCapacityOwner, decommission_capacity_mutation_id};
 use crate::error::{
@@ -984,24 +985,6 @@ fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target:
             .is_some_and(|(source_time, target_time)| target_time > source_time)
 }
 
-fn is_equivalent_scanner_backlog_replica(source: &ObjectInfo, target: &ObjectInfo, compare_part_checksums: bool) -> bool {
-    // Scanner publishes this exact payload to surviving sets with CAS. Each
-    // set assigns its own write time; that timestamp is not a ledger generation.
-    // Accept only an identical, known unversioned identity, never a different
-    // record based on timestamp ordering or a similarly named user object.
-    source.bucket == crate::disk::RUSTFS_META_BUCKET
-        && target.bucket == source.bucket
-        && source.name == "buckets/.scanner-pause-backlog.json"
-        && target.name == source.name
-        && is_unversioned_data_movement_object(source)
-        && is_unversioned_data_movement_object(target)
-        && !source.delete_marker
-        && source.mod_time.is_some()
-        && target.mod_time.is_some()
-        && source.etag.as_ref().is_some_and(|etag| !etag.is_empty())
-        && is_equivalent_data_movement_object_identity(source, target, false, compare_part_checksums)
-}
-
 fn is_data_movement_upload_takeover_target(source: &ObjectInfo, target: &ObjectInfo, compare_part_checksums: bool) -> bool {
     let identity = data_movement_upload_identity(source);
     source.mod_time.is_some()
@@ -1217,7 +1200,7 @@ struct SourceCleanupDeleteBarrierState {
     dead_code,
     reason = "installed by set_disk object tests behind `--features test-util` (backlog#1823)"
 )]
-pub(crate) struct SourceCleanupDeleteBarrier {
+pub struct SourceCleanupDeleteBarrier {
     state: Arc<SourceCleanupDeleteBarrierState>,
 }
 
@@ -1231,7 +1214,7 @@ static SOURCE_CLEANUP_DELETE_BARRIERS: std::sync::OnceLock<std::sync::Mutex<Vec<
     reason = "installed by set_disk object tests behind `--features test-util` (backlog#1823)"
 )]
 impl SourceCleanupDeleteBarrier {
-    pub(crate) fn install(bucket: &str, object: &str) -> Self {
+    pub fn install(bucket: &str, object: &str) -> Self {
         let state = Arc::new(SourceCleanupDeleteBarrierState {
             bucket: bucket.to_string(),
             object: object.to_string(),
@@ -1254,7 +1237,7 @@ impl SourceCleanupDeleteBarrier {
         Self { state }
     }
 
-    pub(crate) async fn wait_until_paused(&self) {
+    pub async fn wait_until_paused(&self) {
         tokio::time::timeout(StdDuration::from_secs(30), self.state.arrived.notified())
             .await
             .expect("source cleanup should reach the pre-delete barrier");
@@ -1270,7 +1253,7 @@ impl SourceCleanupDeleteBarrier {
         self.state.is_paused.load(Ordering::Acquire)
     }
 
-    pub(crate) fn release(&self) {
+    pub fn release(&self) {
         self.state.release.notify_one();
     }
 }
@@ -1449,7 +1432,8 @@ fn resolve_data_movement_overwrite_resume_result_for(
     target_pool_idx: usize,
     compare_part_checksums: bool,
 ) -> Result<bool> {
-    if !should_check_data_movement_overwrite_resume(err)
+    if scanner_backlog::is_scanner_pause_backlog(&source.bucket, &source.name)
+        || !should_check_data_movement_overwrite_resume(err)
         || !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx)
     {
         return Ok(false);
@@ -1471,9 +1455,7 @@ fn resolve_data_movement_overwrite_resume_result_for(
         return Ok(true);
     }
 
-    Ok(matches!(err, Error::PreconditionFailed)
-        && (is_equivalent_scanner_backlog_replica(source, &target, compare_part_checksums)
-            || is_superseding_unversioned_data_movement_object(source, &target)))
+    Ok(matches!(err, Error::PreconditionFailed) && is_superseding_unversioned_data_movement_object(source, &target))
 }
 
 #[derive(Clone, Copy)]
@@ -1521,9 +1503,27 @@ fn data_movement_part_stage_error(
     bucket: &str,
     object: &str,
     part_number: usize,
-    err: impl std::fmt::Display,
+    err: Error,
 ) -> Error {
-    Error::other(format!("{op_label}: {stage} failed for {bucket}/{object} part {part_number}: {err}"))
+    let rendered = format!("{op_label}: {stage} failed for {bucket}/{object} part {part_number}: {err}");
+    if matches!(&err, Error::DecommissionCapacityBlocked { .. }) {
+        return data_movement_context_error(rendered, err);
+    }
+    // A missing target part is not evidence that the source can be deleted.
+    // Keep other part errors opaque to the source-cleanup classifiers.
+    Error::other(rendered)
+}
+
+#[cfg(test)]
+pub(crate) fn data_movement_part_stage_error_for_test(
+    op_label: &str,
+    stage: &str,
+    bucket: &str,
+    object: &str,
+    part_number: usize,
+    err: Error,
+) -> Error {
+    data_movement_part_stage_error(op_label, stage, bucket, object, part_number, err)
 }
 
 fn is_data_movement_part_read_error(err: &Error) -> bool {
@@ -1628,6 +1628,9 @@ async fn migrate_object_inner(
     capacity_owner: Option<DecommissionCapacityOwner>,
     mutation_fence: Option<DecommissionFixedReadAnchor>,
 ) -> Result<()> {
+    if scanner_backlog::is_scanner_pause_backlog(&bucket, &rd.object_info.name) {
+        return Err(Error::other("scanner pause backlog requires native retirement handoff"));
+    }
     let mut mutation_fence = mutation_fence;
     let object_info = rd.object_info.clone();
     let capacity_owner = capacity_owner.map(|owner| {
@@ -2428,8 +2431,15 @@ mod tests {
         let err =
             data_movement_part_stage_error("rebalance_object", "put_object_part", "bucket-a", "object-a", 7, Error::SlowDown);
         let message = err.to_string();
-        assert!(message.contains("rebalance_object: put_object_part failed for bucket-a/object-a part 7"));
-        assert!(message.contains(Error::SlowDown.to_string().as_str()));
+        assert_eq!(
+            message,
+            Error::other(format!(
+                "rebalance_object: put_object_part failed for bucket-a/object-a part 7: {}",
+                Error::SlowDown
+            ))
+            .to_string()
+        );
+        assert!(data_movement_stage_source(&err).is_none());
     }
 
     #[test]
@@ -3329,16 +3339,25 @@ mod tests {
     }
 
     #[test]
-    fn test_scanner_backlog_resume_accepts_identical_native_replica_with_older_write_time() {
+    fn test_scanner_backlog_resume_requires_native_cohort_proof_even_for_identical_payload() {
         let (source, target) = scanner_backlog_replica_pair();
         assert!(!is_owned_data_movement_target(&target), "native scanner writes are not migration copies");
         assert!(!is_equivalent_data_movement_object(&source, &target));
         assert!(
-            scanner_backlog_precondition_resumes(&source, target),
-            "identical ledger payloads have replica-local write times, not distinct committed generations"
+            !scanner_backlog_precondition_resumes(&source, target),
+            "a single identical replica cannot prove native cohort authority"
         );
     }
 
+    #[test]
+    fn test_scanner_backlog_resume_rejects_newer_timestamp_and_full_single_replica_identity() {
+        let (source, mut target) = scanner_backlog_replica_pair();
+        target.mod_time = source.mod_time.map(|time| time + time::Duration::SECOND);
+        target.etag = Some("different-native-ledger".to_string());
+        assert!(!scanner_backlog_precondition_resumes(&source, target));
+        assert!(!scanner_backlog_precondition_resumes(&source, source.clone()));
+    }
+
     #[test]
     fn test_scanner_backlog_resume_rejects_changed_payload_or_metadata() {
         let (source, target) = scanner_backlog_replica_pair();

crates/ecstore/src/data_movement/scanner_backlog.rs

@@ -0,0 +1,292 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::disk::RUSTFS_META_BUCKET;
+use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_found};
+use crate::object_api::ObjectOptions;
+use crate::object_api::{ObjectInfo, PutObjReader, WriteCompletion};
+use crate::set_disk::SetDisks;
+use crate::storage_api_contracts::object::HTTPPreconditions;
+use crate::storage_api_contracts::object::ObjectIO as _;
+use futures::future::join_all;
+use http::HeaderMap;
+use std::sync::{Arc, OnceLock};
+use tokio::io::AsyncReadExt;
+
+pub const MAX_SCANNER_PAUSE_BACKLOG_BYTES: u64 = 64 * 1024;
+pub(crate) const SCANNER_PAUSE_BACKLOG_PATH: &str = "buckets/.scanner-pause-backlog.json";
+
+/// A bounded, storage-fenced native replica. Only a confirmed missing object
+/// has no payload; read failures never enter the Scanner verifier.
+pub struct ScannerPauseBacklogRetirementReplica {
+    pub pool_index: usize,
+    pub set_index: usize,
+    pub data: Option<Vec<u8>>,
+}
+
+/// Native records for a membership handoff. Existing durable ledgers are
+/// preserved; an empty native bootstrap may initialize its first ledger.
+pub struct ScannerPauseBacklogRetirementPlan {
+    pub seed_record: Option<Vec<u8>>,
+    pub commit_record: Vec<u8>,
+    pub stable_record: Vec<u8>,
+}
+
+pub type ScannerPauseBacklogRetirementPlanner =
+    fn(usize, &[ScannerPauseBacklogRetirementReplica]) -> std::result::Result<Option<ScannerPauseBacklogRetirementPlan>, String>;
+
+static RETIREMENT_PLANNER: OnceLock<ScannerPauseBacklogRetirementPlanner> = OnceLock::new();
+
+/// Install the stateless native record planner before storage starts workers.
+/// The scanner runtime switch does not control this storage safety check.
+pub fn register_scanner_pause_backlog_retirement_planner(planner: ScannerPauseBacklogRetirementPlanner) {
+    RETIREMENT_PLANNER.get_or_init(|| planner);
+}
+
+pub(crate) fn is_scanner_pause_backlog(bucket: &str, object: &str) -> bool {
+    bucket == RUSTFS_META_BUCKET && object == SCANNER_PAUSE_BACKLOG_PATH
+}
+
+pub(crate) struct ScannerPauseBacklogRetirementRead {
+    pub replica: ScannerPauseBacklogRetirementReplica,
+    pub etag: Option<String>,
+}
+
+impl ScannerPauseBacklogRetirementRead {
+    pub(crate) fn preconditions(&self) -> HTTPPreconditions {
+        match &self.etag {
+            Some(etag) => HTTPPreconditions {
+                if_match: Some(etag.clone()),
+                ..Default::default()
+            },
+            None => HTTPPreconditions {
+                if_none_match: Some("*".to_string()),
+                ..Default::default()
+            },
+        }
+    }
+}
+
+async fn read_replica(set: Arc<SetDisks>) -> Result<ScannerPauseBacklogRetirementRead> {
+    let mut replica = ScannerPauseBacklogRetirementReplica {
+        pool_index: set.pool_index,
+        set_index: set.set_index,
+        data: None,
+    };
+    let reader = match set
+        .get_object_reader(
+            RUSTFS_META_BUCKET,
+            SCANNER_PAUSE_BACKLOG_PATH,
+            None,
+            HeaderMap::new(),
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+    {
+        Ok(reader) => reader,
+        Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
+            return Ok(ScannerPauseBacklogRetirementRead { replica, etag: None });
+        }
+        Err(err) => return Err(err),
+    };
+    let info = &reader.object_info;
+    if info.version_id.is_some_and(|version| !version.is_nil())
+        || info.delete_marker
+        || info.is_dir
+        || info.etag.as_ref().is_none_or(String::is_empty)
+        || info.size < 0
+        || info.size > MAX_SCANNER_PAUSE_BACKLOG_BYTES as i64
+    {
+        return Err(Error::other("scanner pause backlog retirement found an unsupported replica identity"));
+    }
+    let etag = info.etag.clone();
+    let expected_size = info.size as usize;
+    let mut data = Vec::new();
+    reader
+        .take(MAX_SCANNER_PAUSE_BACKLOG_BYTES + 1)
+        .read_to_end(&mut data)
+        .await?;
+    if data.len() != expected_size || data.len() > MAX_SCANNER_PAUSE_BACKLOG_BYTES as usize {
+        return Err(Error::other("scanner pause backlog retirement replica has an invalid payload length"));
+    }
+    replica.data = Some(data);
+    Ok(ScannerPauseBacklogRetirementRead { replica, etag })
+}
+
+/// The caller retains the fixed object write lock and durable topology read
+/// fence through both this snapshot and physical source cleanup.
+pub(crate) async fn read_scanner_pause_backlog_retirement_replicas(
+    source_pool_index: usize,
+    source_set_index: usize,
+    sets: Vec<Arc<SetDisks>>,
+) -> Result<Vec<ScannerPauseBacklogRetirementRead>> {
+    let replicas = join_all(sets.into_iter().map(read_replica))
+        .await
+        .into_iter()
+        .collect::<Result<Vec<_>>>()?;
+    if !replicas.iter().any(|read| {
+        read.replica.pool_index == source_pool_index && read.replica.set_index == source_set_index && read.replica.data.is_some()
+    }) {
+        return Err(Error::other("scanner pause backlog retirement current source replica is missing"));
+    }
+    Ok(replicas)
+}
+
+pub(crate) fn plan_scanner_pause_backlog_retirement(
+    source_pool_index: usize,
+    replicas: &[ScannerPauseBacklogRetirementRead],
+) -> Result<Option<ScannerPauseBacklogRetirementPlan>> {
+    let planner = RETIREMENT_PLANNER
+        .get()
+        .ok_or_else(|| Error::other("scanner pause backlog native retirement planner is unavailable"))?;
+    let snapshots = replicas
+        .iter()
+        .map(|read| ScannerPauseBacklogRetirementReplica {
+            pool_index: read.replica.pool_index,
+            set_index: read.replica.set_index,
+            data: read.replica.data.clone(),
+        })
+        .collect::<Vec<_>>();
+    planner(source_pool_index, &snapshots).map_err(Error::other)
+}
+
+/// The native writer and retirement handoff use the same conditional, full-tail
+/// write. Their callers retain object and durable membership fences until return.
+pub(crate) async fn persist_native_scanner_pause_backlog_replica(
+    set: Arc<SetDisks>,
+    data: Vec<u8>,
+    preconditions: HTTPPreconditions,
+    mut opts: ObjectOptions,
+    _phase: &'static str,
+) -> Result<ObjectInfo> {
+    if data.len() > MAX_SCANNER_PAUSE_BACKLOG_BYTES as usize {
+        return Err(Error::other("scanner pause backlog exceeds its size bound"));
+    }
+    opts.max_parity = true;
+    opts.write_completion = WriteCompletion::TailDrained;
+    opts.http_preconditions = Some(preconditions);
+    #[cfg(feature = "test-util")]
+    let fault = test_util::matching_write(&set, _phase)?;
+    let result = set
+        .put_object(RUSTFS_META_BUCKET, SCANNER_PAUSE_BACKLOG_PATH, &mut PutObjReader::from_vec(data), &opts)
+        .await;
+    #[cfg(feature = "test-util")]
+    if result.is_ok()
+        && let Some(fault) = fault
+    {
+        fault.arrived.notify_one();
+        fault.release.notified().await;
+    }
+    result
+}
+
+#[cfg(feature = "test-util")]
+pub mod test_util {
+    use super::*;
+    use std::sync::Mutex;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use tokio::sync::Notify;
+
+    #[derive(Debug, thiserror::Error)]
+    #[error("injected native scanner backlog {phase} write failure")]
+    struct InjectedWriteFailure {
+        phase: &'static str,
+    }
+
+    pub(super) struct WriteFault {
+        set: Arc<SetDisks>,
+        phase: &'static str,
+        remaining: AtomicUsize,
+        fail_before_write: bool,
+        pub(super) arrived: Notify,
+        pub(super) release: Notify,
+    }
+
+    static WRITE_FAULTS: Mutex<Vec<Arc<WriteFault>>> = Mutex::new(Vec::new());
+
+    /// Scope a one-shot fault to the actual set instance, so other stores and
+    /// concurrent tests keep using the ordinary native persistence path.
+    pub struct NativeScannerPauseBacklogWriteFault {
+        state: Arc<WriteFault>,
+    }
+
+    impl NativeScannerPauseBacklogWriteFault {
+        fn install(set: Arc<SetDisks>, phase: &'static str, nth: usize, fail_before_write: bool) -> Self {
+            assert!(nth > 0);
+            let state = Arc::new(WriteFault {
+                set,
+                phase,
+                remaining: AtomicUsize::new(nth),
+                fail_before_write,
+                arrived: Notify::new(),
+                release: Notify::new(),
+            });
+            let mut faults = WRITE_FAULTS.lock().unwrap();
+            assert!(
+                !faults
+                    .iter()
+                    .any(|fault| Arc::ptr_eq(&fault.set, &state.set) && fault.phase == phase)
+            );
+            faults.push(Arc::clone(&state));
+            Self { state }
+        }
+
+        pub fn fail_before_write(set: Arc<SetDisks>, phase: &'static str, nth: usize) -> Self {
+            Self::install(set, phase, nth, true)
+        }
+
+        pub fn pause_after_write(set: Arc<SetDisks>, phase: &'static str) -> Self {
+            Self::install(set, phase, 1, false)
+        }
+
+        pub async fn wait_until_paused(&self) {
+            self.state.arrived.notified().await;
+        }
+
+        pub fn release(&self) {
+            self.state.release.notify_one();
+        }
+    }
+
+    impl Drop for NativeScannerPauseBacklogWriteFault {
+        fn drop(&mut self) {
+            self.release();
+            WRITE_FAULTS.lock().unwrap().retain(|fault| !Arc::ptr_eq(fault, &self.state));
+        }
+    }
+
+    pub(super) fn matching_write(set: &Arc<SetDisks>, phase: &'static str) -> Result<Option<Arc<WriteFault>>> {
+        let fault = WRITE_FAULTS
+            .lock()
+            .unwrap()
+            .iter()
+            .find(|fault| Arc::ptr_eq(&fault.set, set) && fault.phase == phase)
+            .cloned();
+        let Some(fault) = fault else { return Ok(None) };
+        if fault
+            .remaining
+            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| remaining.checked_sub(1))
+            != Ok(1)
+        {
+            return Ok(None);
+        }
+        if fault.fail_before_write {
+            return Err(Error::other(InjectedWriteFailure { phase }));
+        }
+        Ok(Some(fault))
+    }
+}

crates/ecstore/src/data_usage/mod.rs

@@ -3397,6 +3397,7 @@ mod tests {
             scan_plan_digest: Some([1; 32]),
             complete: false,
             tombstone: false,
+            segment_invalidation_proof: None,
         }];
         partial.buckets_usage.insert(
             "bucket".to_string(),
@@ -3469,6 +3470,7 @@ mod tests {
                 scan_plan_digest: Some([1; 32]),
                 complete: true,
                 tombstone: false,
+                segment_invalidation_proof: None,
             }],
             ..Default::default()
         };

crates/ecstore/src/disk/error.rs

@@ -627,7 +627,13 @@ impl From<tokio::task::JoinError> for DiskError {
 impl Clone for DiskError {
     fn clone(&self) -> Self {
         match self {
-            DiskError::Io(io_error) => DiskError::Io(std::io::Error::new(io_error.kind(), io_error.to_string())),
+            DiskError::Io(io_error) => DiskError::Io(
+                rustfs_rio::clone_internode_http_io_error(io_error)
+                    .and_then(std::io::Error::into_inner)
+                    // The helper derives a kind from the source; Clone must retain the original outer kind.
+                    .map(|source| std::io::Error::new(io_error.kind(), source))
+                    .unwrap_or_else(|| std::io::Error::new(io_error.kind(), io_error.to_string())),
+            ),
             DiskError::MaxVersionsExceeded => DiskError::MaxVersionsExceeded,
             DiskError::Unexpected => DiskError::Unexpected,
             DiskError::CorruptedFormat => DiskError::CorruptedFormat,
@@ -1265,6 +1271,49 @@ mod tests {
         assert!(!bad_request.is_retryable_internode_write_failure());
     }
 
+    #[test]
+    fn test_internode_http_clone_preserves_retryability_status_and_context() {
+        use http::StatusCode;
+        use rustfs_rio::InternodeHttpErrorKind::{ConnectionRefused, ConnectionReset, HttpStatus, Unknown};
+
+        for (kind, retryable) in [
+            (ConnectionRefused, true),
+            (ConnectionReset, true),
+            (HttpStatus(StatusCode::TOO_MANY_REQUESTS), true),
+            (HttpStatus(StatusCode::SERVICE_UNAVAILABLE), true),
+            (HttpStatus(StatusCode::CONFLICT), true),
+            (Unknown, false),
+            (HttpStatus(StatusCode::BAD_REQUEST), false),
+            (HttpStatus(StatusCode::INTERNAL_SERVER_ERROR), false),
+        ] {
+            let original = DiskError::from(rustfs_rio::new_test_internode_http_io_error(kind));
+            assert_eq!(original.internode_http_error_kind(), Some(kind));
+            assert_eq!(original.is_retryable_internode_write_failure(), retryable);
+
+            let cloned = original.clone();
+            assert_eq!(cloned, original, "clone must preserve the error bucket for {kind:?}");
+            assert_eq!(
+                cloned.is_retryable_internode_write_failure(),
+                retryable,
+                "clone changed retryability for {kind:?}"
+            );
+            assert_eq!(cloned.internode_http_error_kind(), Some(kind));
+            if let HttpStatus(status) = kind {
+                assert!(cloned.is_internode_http_status(status.as_u16()));
+            }
+            let DiskError::Io(io_error) = &cloned else {
+                panic!("unmarked internode error must remain Io: {cloned:?}");
+            };
+            let source = io_error
+                .get_ref()
+                .and_then(|source| source.downcast_ref::<InternodeHttpError>())
+                .expect("clone must retain the structured internode error");
+            assert_eq!(source.context().method(), "PUT");
+            assert_eq!(source.context().target(), "/rustfs/rpc/put_file_stream");
+            assert_eq!(source.context().operation(), Some(INTERNODE_OPERATION_PUT_FILE_STREAM));
+        }
+    }
+
     #[tokio::test]
     async fn read_stream_conflict_is_not_a_retryable_put_file_failure() {
         use tokio::io::{AsyncReadExt, AsyncWriteExt};
@@ -1309,11 +1358,57 @@ mod tests {
                 !error.is_retryable_internode_write_failure(),
                 "read-operation 409 must not trigger put-file retry"
             );
+            let cloned = error.clone();
+            let reduced = crate::disk::error_reduce::reduce_write_quorum_errs(&[Some(error)], &[], 1)
+                .expect("the read conflict must remain the dominant error");
+            for preserved in [&cloned, &reduced] {
+                assert!(
+                    !preserved.is_retryable_internode_write_failure(),
+                    "cloning or reducing a read conflict must not turn it into a PUT retry"
+                );
+                assert!(preserved.is_internode_http_status(409));
+                let DiskError::Io(io_error) = preserved else {
+                    panic!("read conflict must remain Io: {preserved:?}");
+                };
+                let source = io_error
+                    .get_ref()
+                    .and_then(|source| source.downcast_ref::<InternodeHttpError>())
+                    .expect("read conflict must retain its request context");
+                assert_eq!(source.context().method(), "GET");
+                assert_eq!(source.context().target(), "/rustfs/rpc/read_file_stream");
+                assert_eq!(
+                    source.context().operation(),
+                    Some(rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_READ_FILE_STREAM)
+                );
+            }
         })
         .await
         .expect("isolated read-conflict test must finish within its budget");
     }
 
+    #[test]
+    fn test_internode_http_clone_preserves_outer_io_kind_and_message() {
+        let source = rustfs_rio::new_test_internode_http_io_error(InternodeHttpErrorKind::ConnectionReset)
+            .into_inner()
+            .expect("the internode helper must provide a typed source");
+        let original_io = io::Error::new(io::ErrorKind::InvalidData, source);
+        let message = original_io.to_string();
+        let original = DiskError::from(original_io);
+        assert_eq!(original.internode_http_error_kind(), Some(InternodeHttpErrorKind::ConnectionReset));
+        assert!(original.is_retryable_internode_write_failure());
+
+        let cloned = original.clone();
+        let reduced = crate::disk::error_reduce::reduce_write_quorum_errs(&[Some(original)], &[], 1)
+            .expect("the wrapped internode error must remain the dominant error");
+        for preserved in [&cloned, &reduced] {
+            let DiskError::Io(io_error) = preserved else {
+                panic!("the wrapped error must remain Io: {preserved:?}");
+            };
+            assert_eq!(io_error.kind(), io::ErrorKind::InvalidData);
+            assert_eq!(io_error.to_string(), message);
+        }
+    }
+
     #[test]
     fn test_internode_missing_errors_preserve_disk_error_types() {
         let file_missing = DiskError::from(rustfs_rio::new_test_remote_file_not_found_http_io_error());
@@ -1325,6 +1420,17 @@ mod tests {
         assert_eq!(file_missing, DiskError::FileNotFound);
         assert_eq!(volume_missing, DiskError::VolumeNotFound);
         assert!(matches!(unmarked_server_error, DiskError::Io(_)));
+        for missing in [file_missing, volume_missing] {
+            assert_eq!(missing.clone(), missing);
+            assert_eq!(
+                crate::disk::error_reduce::reduce_write_quorum_errs(
+                    &[Some(missing.clone()), Some(missing.clone()), None],
+                    &[],
+                    2
+                ),
+                Some(missing)
+            );
+        }
     }
 
     #[test]

crates/ecstore/src/disk/error_reduce.rs

@@ -226,6 +226,78 @@ mod tests {
         assert_eq!(res, Some(quorum_err));
     }
 
+    #[test]
+    fn test_write_quorum_reduction_preserves_internode_http_identity() {
+        use http::StatusCode;
+        use rustfs_rio::InternodeHttpErrorKind::{ConnectionRefused, HttpStatus, Unknown};
+
+        for (kind, retryable) in [
+            (ConnectionRefused, true),
+            (HttpStatus(StatusCode::SERVICE_UNAVAILABLE), true),
+            (HttpStatus(StatusCode::CONFLICT), true),
+            (Unknown, false),
+            (HttpStatus(StatusCode::BAD_REQUEST), false),
+        ] {
+            // Construct both producer errors independently: the reducer owns the first clone.
+            let first = Error::from(rustfs_rio::new_test_internode_http_io_error(kind));
+            let second = Error::from(rustfs_rio::new_test_internode_http_io_error(kind));
+            assert_eq!(first.internode_http_error_kind(), Some(kind));
+            assert_eq!(second.internode_http_error_kind(), Some(kind));
+            assert_eq!(first.is_retryable_internode_write_failure(), retryable);
+            let errors = [Some(first), Some(second), None];
+            let reduced = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, 2)
+                .expect("two equal producer errors must dominate one successful write");
+
+            assert_eq!(Some(&reduced), errors[0].as_ref());
+            assert_eq!(
+                reduced.is_retryable_internode_write_failure(),
+                retryable,
+                "quorum reduction changed retryability for {kind:?}"
+            );
+            assert_eq!(reduced.internode_http_error_kind(), Some(kind));
+            if let HttpStatus(status) = kind {
+                assert!(reduced.is_internode_http_status(status.as_u16()));
+            }
+            let Error::Io(io_error) = &reduced else {
+                panic!("the dominant error must remain Io: {reduced:?}");
+            };
+            let source = io_error
+                .get_ref()
+                .and_then(|source| source.downcast_ref::<rustfs_rio::InternodeHttpError>())
+                .expect("quorum reduction must retain the structured internode error");
+            assert_eq!(source.context().method(), "PUT");
+            assert_eq!(source.context().target(), "/rustfs/rpc/put_file_stream");
+            assert_eq!(
+                source.context().operation(),
+                Some(rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_PUT_FILE_STREAM)
+            );
+        }
+    }
+
+    #[test]
+    fn test_clone_and_write_quorum_do_not_promote_non_retryable_errors() {
+        use http::StatusCode;
+        use rustfs_rio::InternodeHttpErrorKind::{HttpStatus, Unknown};
+
+        for original in [
+            Error::from(rustfs_rio::new_test_internode_http_io_error(Unknown)),
+            Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(StatusCode::BAD_REQUEST))),
+            Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(StatusCode::FORBIDDEN))),
+            Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(StatusCode::NOT_FOUND))),
+            Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(
+                StatusCode::INTERNAL_SERVER_ERROR,
+            ))),
+            err_io("internode connection reset: PUT /rustfs/rpc/put_file_stream"),
+        ] {
+            assert!(!original.is_retryable_internode_write_failure());
+            let cloned = original.clone();
+            let reduced =
+                reduce_write_quorum_errs(&[Some(original)], &[], 1).expect("a non-retryable error must remain an error");
+            assert!(!cloned.is_retryable_internode_write_failure());
+            assert!(!reduced.is_retryable_internode_write_failure());
+        }
+    }
+
     #[test]
     fn test_count_errs() {
         let e1 = err_io("a");

crates/ecstore/src/disk/local.rs

@@ -219,6 +219,7 @@ fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, r
     }
 }
 
+#[cfg(test)]
 async fn write_metadata_rollback_backup(object_dir: &Path, rollback_dir: Uuid, data: &[u8]) -> Result<()> {
     write_delete_rollback_file(object_dir, rollback_dir, STORAGE_FORMAT_FILE_BACKUP, data, None).await
 }
@@ -250,6 +251,7 @@ async fn write_delete_rollback_file(
     Ok(())
 }
 
+#[cfg(test)]
 async fn restore_metadata_backup(
     object_dir: &Path,
     xl_path: &Path,
@@ -277,15 +279,6 @@ async fn restore_metadata_backup_with_namespace_owner(
     Ok(())
 }
 
-async fn restore_delete_rollback(
-    object_dir: &Path,
-    xl_path: &Path,
-    rollback_dir: Uuid,
-    publication_root: &os::PublicationRoot,
-) -> Result<()> {
-    restore_delete_rollback_with_namespace_owner(object_dir, xl_path, rollback_dir, publication_root, None).await
-}
-
 async fn restore_delete_rollback_with_namespace_owner(
     object_dir: &Path,
     xl_path: &Path,
@@ -5857,6 +5850,8 @@ impl LocalDisk {
         check_path_length(file_path.to_string_lossy().as_ref())?;
 
         let xl_path = path_join(&[file_path.as_path(), Path::new(STORAGE_FORMAT_FILE)]);
+        let namespace_owner: Option<Arc<dyn Send + Sync>> =
+            Some(os::acquire_metadata_mutation_lease(&self.get_object_path(volume, path)?, namespace_owner).await);
         if opts.old_data_dir.is_some() && opts.undo_write {
             return self.undo_write(file_path.as_path(), &fi, &opts, namespace_owner).await;
         }
@@ -6714,6 +6709,8 @@ impl LocalDisk {
 
     async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &[FileInfo], opts: &DeleteOptions) -> Result<()> {
         let volume_dir = self.io_get_bucket_path(volume)?;
+        let object_path = self.get_object_path(volume, path)?;
+        let namespace_owner: Option<Arc<dyn Send + Sync>> = Some(os::acquire_metadata_mutation_lease(&object_path, None).await);
         let xlpath = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?;
         let object_dir = xlpath
             .parent()
@@ -6723,10 +6720,24 @@ impl LocalDisk {
             && opts.undo_write
         {
             if opts.undo_delete {
-                return restore_delete_rollback(object_dir, &xlpath, rollback_dir, &self.publication_root).await;
+                return restore_delete_rollback_with_namespace_owner(
+                    object_dir,
+                    &xlpath,
+                    rollback_dir,
+                    &self.publication_root,
+                    namespace_owner.clone(),
+                )
+                .await;
             }
 
-            return restore_metadata_backup(object_dir, &xlpath, rollback_dir, &self.publication_root).await;
+            return restore_metadata_backup_with_namespace_owner(
+                object_dir,
+                &xlpath,
+                rollback_dir,
+                &self.publication_root,
+                namespace_owner.clone(),
+            )
+            .await;
         }
 
         let (data, _) = match self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await {
@@ -6738,7 +6749,14 @@ impl LocalDisk {
                     return Err(DiskError::FileNotFound);
                 };
                 return self
-                    .write_missing_delete_marker(volume, path, delete_marker, object_dir, opts.old_data_dir, None)
+                    .write_missing_delete_marker(
+                        volume,
+                        path,
+                        delete_marker,
+                        object_dir,
+                        opts.old_data_dir,
+                        namespace_owner.clone(),
+                    )
                     .await;
             }
             Err(err) => return Err(err),
@@ -6754,7 +6772,8 @@ impl LocalDisk {
         let rollback_dir = opts.old_data_dir;
         let mut reserved_version_delete = false;
         if let Some(rollback_dir) = rollback_dir {
-            write_metadata_rollback_backup(object_dir, rollback_dir, &data).await?;
+            write_delete_rollback_file(object_dir, rollback_dir, STORAGE_FORMAT_FILE_BACKUP, &data, namespace_owner.clone())
+                .await?;
         }
 
         for fi in fis.iter() {
@@ -6768,13 +6787,16 @@ impl LocalDisk {
 
                     if reserved_version_delete && let Some(rollback_dir) = rollback_dir {
                         return Err(self
-                            .abort_reserved_version_delete(
+                            .abort_reserved_version_delete_with_failure(
                                 object_dir,
                                 rollback_dir,
                                 volume,
                                 path,
-                                "delete_versions_metadata_update",
-                                err,
+                                DeleteRollbackFailure {
+                                    stage: "delete_versions_metadata_update",
+                                    error: err,
+                                    namespace_owner: namespace_owner.clone(),
+                                },
                             )
                             .await);
                     }
@@ -6787,7 +6809,7 @@ impl LocalDisk {
                         DeleteRollbackFailure {
                             stage: "delete_versions_metadata_update",
                             error: err,
-                            namespace_owner: None,
+                            namespace_owner: namespace_owner.clone(),
                         },
                         &self.publication_root,
                     )
@@ -6804,13 +6826,16 @@ impl LocalDisk {
                     Err(err) => {
                         if reserved_version_delete && let Some(rollback_dir) = rollback_dir {
                             return Err(self
-                                .abort_reserved_version_delete(
+                                .abort_reserved_version_delete_with_failure(
                                     object_dir,
                                     rollback_dir,
                                     volume,
                                     path,
-                                    "delete_versions_data_path",
-                                    err,
+                                    DeleteRollbackFailure {
+                                        stage: "delete_versions_data_path",
+                                        error: err,
+                                        namespace_owner: namespace_owner.clone(),
+                                    },
                                 )
                                 .await);
                         }
@@ -6823,7 +6848,7 @@ impl LocalDisk {
                             DeleteRollbackFailure {
                                 stage: "delete_versions_data_path",
                                 error: err,
-                                namespace_owner: None,
+                                namespace_owner: namespace_owner.clone(),
                             },
                             &self.publication_root,
                         )
@@ -6836,13 +6861,16 @@ impl LocalDisk {
                         let err: DiskError = to_file_error(err).into();
                         if reserved_version_delete {
                             return Err(self
-                                .abort_reserved_version_delete(
+                                .abort_reserved_version_delete_with_failure(
                                     object_dir,
                                     rollback_dir,
                                     volume,
                                     path,
-                                    "delete_versions_rollback_dir",
-                                    err,
+                                    DeleteRollbackFailure {
+                                        stage: "delete_versions_rollback_dir",
+                                        error: err,
+                                        namespace_owner: namespace_owner.clone(),
+                                    },
                                 )
                                 .await);
                         }
@@ -6855,23 +6883,29 @@ impl LocalDisk {
                             DeleteRollbackFailure {
                                 stage: "delete_versions_rollback_dir",
                                 error: err,
-                                namespace_owner: None,
+                                namespace_owner: namespace_owner.clone(),
                             },
                             &self.publication_root,
                         )
                         .await);
                     }
-                    let reserved = match self.reserve_version_delete(volume, path, dir, rollback_dir).await {
+                    let reserved = match self
+                        .reserve_version_delete_with_namespace_owner(volume, path, dir, rollback_dir, namespace_owner.clone())
+                        .await
+                    {
                         Ok(reserved) => reserved,
                         Err(err) => {
                             return Err(self
-                                .abort_reserved_version_delete(
+                                .abort_reserved_version_delete_with_failure(
                                     object_dir,
                                     rollback_dir,
                                     volume,
                                     path,
-                                    "delete_versions_reserve_data",
-                                    err,
+                                    DeleteRollbackFailure {
+                                        stage: "delete_versions_reserve_data",
+                                        error: err,
+                                        namespace_owner: namespace_owner.clone(),
+                                    },
                                 )
                                 .await);
                         }
@@ -6879,11 +6913,12 @@ impl LocalDisk {
                     reserved_version_delete |= reserved;
                     let rollback_data_path = rollback_path.join(dir.to_string());
                     if !reserved
-                        && let Err(err) = rename_all_ignore_missing_source(
+                        && let Err(err) = os::rename_all_ignore_missing_source_with_owner(
                             &dir_path,
                             &rollback_data_path,
                             &rollback_path,
                             &self.publication_root,
+                            namespace_owner.clone(),
                         )
                         .await
                     {
@@ -6896,7 +6931,7 @@ impl LocalDisk {
                             DeleteRollbackFailure {
                                 stage: "delete_versions_stage_data",
                                 error: err,
-                                namespace_owner: None,
+                                namespace_owner: namespace_owner.clone(),
                             },
                             &self.publication_root,
                         )
@@ -6905,13 +6940,16 @@ impl LocalDisk {
                     if should_fail_after_delete_data_staged(path) {
                         if reserved_version_delete {
                             return Err(self
-                                .abort_reserved_version_delete(
+                                .abort_reserved_version_delete_with_failure(
                                     object_dir,
                                     rollback_dir,
                                     volume,
                                     path,
-                                    "delete_versions_test_after_stage",
-                                    DiskError::Unexpected,
+                                    DeleteRollbackFailure {
+                                        stage: "delete_versions_test_after_stage",
+                                        error: DiskError::Unexpected,
+                                        namespace_owner: namespace_owner.clone(),
+                                    },
                                 )
                                 .await);
                         }
@@ -6924,13 +6962,15 @@ impl LocalDisk {
                             DeleteRollbackFailure {
                                 stage: "delete_versions_test_after_stage",
                                 error: DiskError::Unexpected,
-                                namespace_owner: None,
+                                namespace_owner: namespace_owner.clone(),
                             },
                             &self.publication_root,
                         )
                         .await);
                     }
-                } else if let Err(err) = self.move_to_trash(&dir_path, true, false).await
+                } else if let Err(err) = self
+                    .move_to_trash_with_namespace_owner(&dir_path, true, false, namespace_owner.clone())
+                    .await
                     && !(err == DiskError::FileNotFound || err == DiskError::VolumeNotFound)
                 {
                     return Err(err);
@@ -6945,16 +6985,22 @@ impl LocalDisk {
 
         // Remove xl.meta when no versions remain
         if fm.versions.is_empty() {
-            if let Err(err) = self.delete_file(&volume_dir, &xlpath, true, false).await {
+            if let Err(err) = self
+                .delete_file_with_namespace_owner(&volume_dir, &xlpath, true, false, namespace_owner.clone())
+                .await
+            {
                 if reserved_version_delete && let Some(rollback_dir) = rollback_dir {
                     return Err(self
-                        .abort_reserved_version_delete(
+                        .abort_reserved_version_delete_with_failure(
                             object_dir,
                             rollback_dir,
                             volume,
                             path,
-                            "delete_versions_commit_delete",
-                            err,
+                            DeleteRollbackFailure {
+                                stage: "delete_versions_commit_delete",
+                                error: err,
+                                namespace_owner: namespace_owner.clone(),
+                            },
                         )
                         .await);
                 }
@@ -6967,7 +7013,7 @@ impl LocalDisk {
                     DeleteRollbackFailure {
                         stage: "delete_versions_commit_delete",
                         error: err,
-                        namespace_owner: None,
+                        namespace_owner: namespace_owner.clone(),
                     },
                     &self.publication_root,
                 )
@@ -6975,10 +7021,22 @@ impl LocalDisk {
             }
             if reserved_version_delete
                 && let Some(rollback_dir) = rollback_dir
-                && let Err(err) = self.commit_reserved_version_delete(volume, path, rollback_dir).await
+                && let Err(err) = self
+                    .commit_reserved_version_delete_with_namespace_owner(volume, path, rollback_dir, namespace_owner.clone())
+                    .await
             {
                 return Err(self
-                    .abort_reserved_version_delete(object_dir, rollback_dir, volume, path, "delete_versions_commit_intent", err)
+                    .abort_reserved_version_delete_with_failure(
+                        object_dir,
+                        rollback_dir,
+                        volume,
+                        path,
+                        DeleteRollbackFailure {
+                            stage: "delete_versions_commit_intent",
+                            error: err,
+                            namespace_owner: namespace_owner.clone(),
+                        },
+                    )
                     .await);
             }
             if should_fail_after_delete_commit(self.root.as_path(), path) {
@@ -6995,13 +7053,16 @@ impl LocalDisk {
                 let err: DiskError = err.into();
                 if reserved_version_delete && let Some(rollback_dir) = rollback_dir {
                     return Err(self
-                        .abort_reserved_version_delete(
+                        .abort_reserved_version_delete_with_failure(
                             object_dir,
                             rollback_dir,
                             volume,
                             path,
-                            "delete_versions_metadata_encode",
-                            err,
+                            DeleteRollbackFailure {
+                                stage: "delete_versions_metadata_encode",
+                                error: err,
+                                namespace_owner: namespace_owner.clone(),
+                            },
                         )
                         .await);
                 }
@@ -7014,7 +7075,7 @@ impl LocalDisk {
                     DeleteRollbackFailure {
                         stage: "delete_versions_metadata_encode",
                         error: err,
-                        namespace_owner: None,
+                        namespace_owner: namespace_owner.clone(),
                     },
                     &self.publication_root,
                 )
@@ -7023,12 +7084,28 @@ impl LocalDisk {
         };
 
         if let Err(err) = self
-            .write_all_meta(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str(), &buf, true)
+            .write_all_meta_with_namespace_owner(
+                volume,
+                format!("{path}/{STORAGE_FORMAT_FILE}").as_str(),
+                &buf,
+                true,
+                namespace_owner.clone(),
+            )
             .await
         {
             if reserved_version_delete && let Some(rollback_dir) = rollback_dir {
                 return Err(self
-                    .abort_reserved_version_delete(object_dir, rollback_dir, volume, path, "delete_versions_commit_write", err)
+                    .abort_reserved_version_delete_with_failure(
+                        object_dir,
+                        rollback_dir,
+                        volume,
+                        path,
+                        DeleteRollbackFailure {
+                            stage: "delete_versions_commit_write",
+                            error: err,
+                            namespace_owner: namespace_owner.clone(),
+                        },
+                    )
                     .await);
             }
             return Err(restore_delete_rollback_after_error(
@@ -7040,7 +7117,7 @@ impl LocalDisk {
                 DeleteRollbackFailure {
                     stage: "delete_versions_commit_write",
                     error: err,
-                    namespace_owner: None,
+                    namespace_owner: namespace_owner.clone(),
                 },
                 &self.publication_root,
             )
@@ -7049,10 +7126,22 @@ impl LocalDisk {
 
         if reserved_version_delete
             && let Some(rollback_dir) = rollback_dir
-            && let Err(err) = self.commit_reserved_version_delete(volume, path, rollback_dir).await
+            && let Err(err) = self
+                .commit_reserved_version_delete_with_namespace_owner(volume, path, rollback_dir, namespace_owner.clone())
+                .await
         {
             return Err(self
-                .abort_reserved_version_delete(object_dir, rollback_dir, volume, path, "delete_versions_commit_intent", err)
+                .abort_reserved_version_delete_with_failure(
+                    object_dir,
+                    rollback_dir,
+                    volume,
+                    path,
+                    DeleteRollbackFailure {
+                        stage: "delete_versions_commit_intent",
+                        error: err,
+                        namespace_owner: namespace_owner.clone(),
+                    },
+                )
                 .await);
         }
 
@@ -7063,6 +7152,107 @@ impl LocalDisk {
         Ok(())
     }
 
+    async fn reconcile_transition_state_metadata(
+        &self,
+        volume: &str,
+        object: &str,
+        version_id: Option<Uuid>,
+        condition: &super::TransitionStateReconcileCondition,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+        authority: Arc<crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority>,
+    ) -> Result<()> {
+        condition.target.validate()?;
+        if !authority.is_current()
+            || [
+                condition.expected_metadata_digest.as_str(),
+                condition.unchanged_metadata_digest.as_str(),
+            ]
+            .iter()
+            .any(|digest| {
+                digest.len() != 64
+                    || !digest
+                        .bytes()
+                        .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
+            })
+        {
+            return Err(DiskError::OutdatedXLMeta);
+        }
+        let metadata_path = format!("{object}/{STORAGE_FORMAT_FILE}");
+        let original = self.read_all(volume, &metadata_path).await?;
+        let original_digest = rustfs_utils::crypto::hex(<sha2::Sha256 as sha2::Digest>::digest(&original));
+        let mut metadata = FileMeta::load(&original)?;
+        let (_, selected) = metadata.find_version(version_id)?;
+        if selected.into_fileinfo(volume, object, true)?.transition_tier != condition.tier {
+            return Err(DiskError::OutdatedXLMeta);
+        }
+        let generation = metadata.transition_reconcile_generation(version_id)?;
+        if rustfs_utils::crypto::hex(<sha2::Sha256 as sha2::Digest>::digest(&generation)) != condition.unchanged_metadata_digest {
+            return Err(DiskError::OutdatedXLMeta);
+        }
+        let changed = metadata.reconcile_transition_state(version_id, &condition.target)?;
+        if !changed {
+            return Ok(());
+        }
+        if condition.verify_only || original_digest != condition.expected_metadata_digest {
+            return Err(DiskError::OutdatedXLMeta);
+        }
+        // fsync_dir_std is a no-op outside Unix, so those platforms cannot
+        // yet prove this repair's durable publication requirement.
+        if !cfg!(unix) || !effective_durability(volume).syncs_commit_metadata() {
+            return Err(DiskError::other(
+                "transition reconciliation requires Unix directory sync and enabled bucket metadata durability",
+            ));
+        }
+        // An outstanding rollback can still restore an older whole xl.meta.
+        // Its preparation and execution share this mutation domain; refuse
+        // repair until that transaction has settled and removed its backup.
+        let mut entries = fs::read_dir(self.io_get_object_path(volume, object)?)
+            .await
+            .map_err(to_file_error)?;
+        let mut remaining = 4096usize;
+        while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? {
+            remaining = remaining.checked_sub(1).ok_or(DiskError::OutdatedXLMeta)?;
+            if Uuid::parse_str(&entry.file_name().to_string_lossy()).is_err() {
+                continue;
+            }
+            for marker in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] {
+                if fs::try_exists(entry.path().join(marker)).await.map_err(to_file_error)? {
+                    return Err(DiskError::OutdatedXLMeta);
+                }
+            }
+        }
+        let replacement = metadata.marshal_msg()?;
+        let tmp_volume = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
+        let tmp_file = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, &Uuid::new_v4().to_string())?;
+        // Admission above requires metadata durability. Keep rename and its
+        // directory sync in the same owned executor even after cancellation.
+        self.write_all_internal(&tmp_file, InternalBuf::Ref(&replacement), SyncMode::FileOnly, &tmp_volume)
+            .await?;
+        if crash_inject::should_crash_at(CrashPoint::MetaWriteAfterTmpBeforeRename, &metadata_path) {
+            return Err(DiskError::Unexpected);
+        }
+        os::rename_reconciled_metadata(
+            tmp_file,
+            self.io_get_object_path(volume, &metadata_path)?,
+            self.io_get_bucket_path(volume)?,
+            self.publication_root.clone(),
+            namespace_owner.clone(),
+            authority,
+        )
+        .await?;
+        // Keep the same mutation lease through strong readback. Response loss
+        // leaves a monotonic subset for the coordinator's exact-copy retry.
+        let committed = self.read_all(volume, &metadata_path).await?;
+        let mut committed = FileMeta::load(&committed)?;
+        if committed.reconcile_transition_state(version_id, &condition.target)?
+            || committed.transition_reconcile_generation(version_id)? != generation
+        {
+            return Err(DiskError::OutdatedXLMeta);
+        }
+        Ok(())
+    }
+
+    #[cfg(test)]
     async fn write_all_meta(&self, volume: &str, path: &str, buf: &[u8], sync: bool) -> Result<()> {
         self.write_all_meta_with_namespace_owner(volume, path, buf, sync, None).await
     }
@@ -8321,6 +8511,7 @@ impl LocalDisk {
         Ok(Arc::new(QuotaMutationFenceClaim { state }))
     }
 
+    #[cfg(test)]
     async fn reserve_version_delete(&self, volume: &str, object: &str, data_dir: Uuid, rollback_dir: Uuid) -> Result<bool> {
         self.reserve_version_delete_with_namespace_owner(volume, object, data_dir, rollback_dir, None)
             .await
@@ -8373,6 +8564,7 @@ impl LocalDisk {
         Ok(true)
     }
 
+    #[cfg(test)]
     async fn commit_reserved_version_delete(&self, volume: &str, object: &str, rollback_dir: Uuid) -> Result<()> {
         self.commit_reserved_version_delete_with_namespace_owner(volume, object, rollback_dir, None)
             .await
@@ -8464,29 +8656,6 @@ impl LocalDisk {
         first_err.map_or(Ok(found), Err)
     }
 
-    async fn abort_reserved_version_delete(
-        &self,
-        object_dir: &Path,
-        rollback_dir: Uuid,
-        volume: &str,
-        object: &str,
-        stage: &'static str,
-        err: DiskError,
-    ) -> DiskError {
-        self.abort_reserved_version_delete_with_failure(
-            object_dir,
-            rollback_dir,
-            volume,
-            object,
-            DeleteRollbackFailure {
-                stage,
-                error: err,
-                namespace_owner: None,
-            },
-        )
-        .await
-    }
-
     async fn abort_reserved_version_delete_with_failure(
         &self,
         object_dir: &Path,
@@ -10031,6 +10200,23 @@ impl DiskAPI for LocalDisk {
 
     #[tracing::instrument(level = "trace", skip_all)]
     async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
+        let object_path = self.get_object_path(volume, path)?;
+        if let Some(condition) = &opts.transition_reconcile {
+            if !fi.metadata.is_empty() || opts.no_persistence || opts.replace_user_metadata {
+                return Err(DiskError::FileCorrupt);
+            }
+            let authority =
+                crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority::for_disk(
+                    condition,
+                )
+                .await?;
+            let owner: Option<Arc<dyn Send + Sync>> = Some(authority.clone());
+            let owner: Option<Arc<dyn Send + Sync>> = Some(os::acquire_metadata_mutation_lease(&object_path, owner).await);
+            return self
+                .reconcile_transition_state_metadata(volume, path, fi.version_id, condition, owner, authority)
+                .await;
+        }
+        let namespace_owner: Option<Arc<dyn Send + Sync>> = Some(os::acquire_metadata_mutation_lease(&object_path, None).await);
         if !fi.metadata.is_empty() {
             let file_path = self.io_get_object_path(volume, path)?;
 
@@ -10058,7 +10244,13 @@ impl DiskAPI for LocalDisk {
             let wbuf = xl_meta.marshal_msg()?;
 
             return self
-                .write_all_meta(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str(), &wbuf, !opts.no_persistence)
+                .write_all_meta_with_namespace_owner(
+                    volume,
+                    format!("{path}/{STORAGE_FORMAT_FILE}").as_str(),
+                    &wbuf,
+                    !opts.no_persistence,
+                    namespace_owner,
+                )
                 .await;
         }
 
@@ -10066,7 +10258,10 @@ impl DiskAPI for LocalDisk {
     }
 
     async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
-        self.write_metadata_with_namespace_owner(volume, path, fi, None).await
+        let object_path = self.get_object_path(volume, path)?;
+        let namespace_owner: Option<Arc<dyn Send + Sync>> = Some(os::acquire_metadata_mutation_lease(&object_path, None).await);
+        self.write_metadata_with_namespace_owner(volume, path, fi, namespace_owner)
+            .await
     }
 
     #[tracing::instrument(level = "trace", skip_all)]

crates/ecstore/src/disk/local/commit.rs

@@ -279,11 +279,14 @@ impl LocalDisk {
             Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?),
             None => None,
         };
+        // Quota admission -> metadata RMW -> namespace/volume publication.
+        let metadata_lease =
+            os::acquire_metadata_mutation_lease(&self.get_object_path(dst_volume, dst_path)?, state.namespace_owner.take()).await;
         let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
             &self.root,
             dst_volume,
             &destination_object_path,
-            state.namespace_owner.take(),
+            Some(metadata_lease),
         )
         .await;
         if let Some(claim) = quota_fence_claim {

crates/ecstore/src/disk/mod.rs

@@ -1277,6 +1277,25 @@ pub struct CheckPartsResp {
 pub struct UpdateMetadataOpts {
     pub no_persistence: bool,
     pub replace_user_metadata: bool,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub transition_reconcile: Option<Box<TransitionStateReconcileCondition>>,
+}
+
+/// An exact-copy precondition for the single-version tier repair protocol.
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct TransitionStateReconcileCondition {
+    pub expected_metadata_digest: String,
+    pub unchanged_metadata_digest: String,
+    pub target: rustfs_filemeta::TransitionStateReconcileTarget,
+    pub tier: String,
+    pub topology_generation: String,
+    pub verify_only: bool,
+    /// Local ownership is never accepted from the wire. A remote disk acquires
+    /// its own fleet and backend leases before entering the mutation domain.
+    #[serde(skip)]
+    pub(crate) authority:
+        Option<Arc<crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority>>,
 }
 
 pub struct DiskLocation {

crates/ecstore/src/disk/os.rs

@@ -452,13 +452,13 @@ pub(crate) mod windows_rename_test_hooks {
 
 /// Test-only hooks into the destination-parent walk of rename preparation.
 ///
-/// The prune race lives between two syscalls inside
+/// Pruning and Windows sharing races live between syscalls inside
 /// [`mkdir_all_below_existing_base_std`], so only an injection at that exact
 /// point reproduces it deterministically. Hooks are keyed by the absolute path
 /// of the component just opened and queued per path: a retrying preparation
 /// visits the same component again, so a test models a pruner that keeps
 /// walking upward by queueing one hook per visit.
-#[cfg(all(test, unix))]
+#[cfg(all(test, any(unix, windows)))]
 pub(crate) mod prepare_rename_test_hooks {
     use super::*;
 
@@ -1449,6 +1449,38 @@ fn disk_namespace_mutation_lock(path: &Path) -> Arc<NamespaceMutationLock> {
     lock
 }
 
+static DISK_METADATA_MUTATION_LOCKS: LazyLock<Mutex<NamespaceMutationLockRegistry>> =
+    LazyLock::new(|| Mutex::new(HashMap::new()));
+
+/// Serializes the complete xl.meta read/modify/commit transaction. This domain
+/// precedes namespace/volume publication locks, whose narrower syscall leases
+/// may retain it after cancellation of the async caller.
+pub(crate) struct MetadataMutationLease {
+    _guard: OwnedMutexGuard<()>,
+    _owner: Option<Arc<dyn Send + Sync>>,
+}
+
+pub(crate) async fn acquire_metadata_mutation_lease(
+    object: &Path,
+    owner: Option<Arc<dyn Send + Sync>>,
+) -> Arc<MetadataMutationLease> {
+    let lock = {
+        let mut locks = DISK_METADATA_MUTATION_LOCKS.lock();
+        locks.retain(|_, lock| lock.strong_count() > 0);
+        if let Some(lock) = locks.get(object).and_then(Weak::upgrade) {
+            lock
+        } else {
+            let lock = Arc::new(AsyncMutex::new(()));
+            locks.insert(object.to_path_buf(), Arc::downgrade(&lock));
+            lock
+        }
+    };
+    Arc::new(MetadataMutationLease {
+        _guard: lock.lock_owned().await,
+        _owner: owner,
+    })
+}
+
 /// Keeps a namespace transaction serialized even when its async waiter is
 /// cancelled while a blocking filesystem call is still running.
 pub(crate) struct NamespaceMutationLease {
@@ -2112,6 +2144,41 @@ pub(crate) async fn rename_all_with_lease(
     Ok(())
 }
 
+/// Publish a conditional repair and sync its directory in one owned executor.
+/// Cancellation cannot release its metadata/fleet/tier leases between rename
+/// and fsync. The last authority check runs after destination preparation.
+pub(crate) async fn rename_reconciled_metadata(
+    source: PathBuf,
+    destination: PathBuf,
+    base_dir: PathBuf,
+    publication_root: PublicationRoot,
+    owner: Option<Arc<dyn Send + Sync>>,
+    authority: Arc<crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority>,
+) -> Result<()> {
+    let lease = acquire_namespace_mutation_lease_with_owner(&destination, owner).await;
+    run_blocking_namespace_operation(lease, move || {
+        let preparation = prepare_rename_with_retry(&source, &destination, &base_dir, &publication_root)?;
+        #[cfg(all(any(test, feature = "test-util"), not(windows)))]
+        prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &destination);
+        if !authority.is_current() {
+            return Err(io::Error::new(io::ErrorKind::WouldBlock, "transition reconciliation authority expired"));
+        }
+        rename_prepared(&source, &destination, &preparation)?;
+        if let Some(parent) = destination.parent() {
+            fsync_dir_std(parent)?;
+        }
+        Ok(())
+    })
+    .await
+    .map_err(|err| {
+        if err.kind() == io::ErrorKind::WouldBlock {
+            DiskError::OutdatedXLMeta
+        } else {
+            to_file_error(err).into()
+        }
+    })
+}
+
 #[cfg(windows)]
 #[tracing::instrument(level = "debug", skip_all)]
 pub(crate) async fn rename_all_with_commit_guard(
@@ -4208,11 +4275,14 @@ pub(crate) fn mkdir_all_below_existing_base_std(
         let mut handles = Vec::with_capacity(capacity);
         handles.push(publication_root.directory.clone());
         let mut guard = ExistingBaseDirectoryGuard::new(handles);
-        for component in base_relative.components() {
+        let mut components = base_relative
+            .components()
+            .map(|component| (component, FILE_OPEN))
+            .chain(relative.components().map(|component| (component, FILE_OPEN_IF)))
+            .filter(|(component, _)| !matches!(component, Component::CurDir))
+            .peekable();
+        while let Some((component, disposition)) = components.next() {
             let Component::Normal(component) = component else {
-                if matches!(component, Component::CurDir) {
-                    continue;
-                }
                 return Err(io::Error::new(
                     io::ErrorKind::InvalidInput,
                     "rename base directory contains an invalid path component",
@@ -4222,40 +4292,22 @@ pub(crate) fn mkdir_all_below_existing_base_std(
                 .handles
                 .last()
                 .ok_or_else(|| io::Error::other("Windows publication root guard is empty"))?;
-            let child = open_windows_directory_component(parent, component, FILE_OPEN)?;
-            guard.handles.push(child);
-        }
-        for component in relative.components() {
-            let Component::Normal(component) = component else {
-                continue;
+            // The kernel opens the final parent for write during a relative
+            // rename. Share writes from its first open: a temporary read-only
+            // share would block another rename into the same trash directory.
+            // Ancestors stay strict and no handle shares delete access, keeping
+            // every retained directory identity pinned.
+            let share_access = if components.peek().is_none() {
+                FILE_SHARE_READ | FILE_SHARE_WRITE
+            } else {
+                FILE_SHARE_READ
             };
-            let parent = guard
-                .handles
-                .last()
-                .ok_or_else(|| io::Error::other("Windows base directory guard is empty"))?;
-            let child = open_windows_directory_component(parent, component, FILE_OPEN_IF)?;
+            let child = open_windows_relative_directory_component(parent, component, disposition, share_access)?;
             guard.handles.push(child);
-        }
-
-        // Windows resolves a handle-relative rename by opening the target for
-        // write. Keep every ancestor strict, but let that internal open share
-        // the final parent. Delete sharing remains omitted, so the retained
-        // directory entry cannot be renamed or removed during publication.
-        if guard.handles.len() > 1 {
-            let component = dir_path
-                .file_name()
-                .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination parent must have a name"))?;
-            let parent_index = guard.handles.len() - 2;
-            let parent = guard
-                .handles
-                .get(parent_index)
-                .ok_or_else(|| io::Error::other("Windows destination guard lost its parent handle"))?;
-            let rename_parent =
-                open_windows_relative_directory_component(parent, component, FILE_OPEN, FILE_SHARE_READ | FILE_SHARE_WRITE)?;
-            *guard
-                .handles
-                .last_mut()
-                .ok_or_else(|| io::Error::other("Windows destination guard is empty"))? = rename_parent;
+            #[cfg(test)]
+            if components.peek().is_none() {
+                prepare_rename_test_hooks::run_after_component_opened(dir_path);
+            }
         }
 
         Ok(guard)
@@ -5446,6 +5498,111 @@ mod tests {
         assert_eq!(std::fs::read(dst).expect("read committed metadata"), b"metadata");
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn rename_all_concurrent_trash_renames_remove_every_rollback_directory() {
+        let temp_dir = tempdir().expect("create temp dir");
+        let trash = temp_dir.path().join(".rustfs.sys/tmp/.trash");
+        std::fs::create_dir_all(&trash).expect("create trash directory");
+        let publication_root = PublicationRoot::new(temp_dir.path()).expect("open publication root");
+        let sources: Vec<_> = (0..16)
+            .map(|index| {
+                let source = temp_dir
+                    .path()
+                    .join(format!("bucket/{index}.mp4"))
+                    .join(uuid::Uuid::new_v4().to_string());
+                std::fs::create_dir_all(&source).expect("create rollback directory");
+                std::fs::write(source.join("xl.meta.bkp"), b"rollback metadata").expect("write metadata backup");
+                source
+            })
+            .collect();
+
+        let results = futures::future::join_all(sources.iter().enumerate().map(|(index, source)| {
+            super::rename_all_ignore_missing_source(source, trash.join(index.to_string()), &trash, &publication_root)
+        }))
+        .await;
+
+        for (index, (source, result)) in sources.iter().zip(results).enumerate() {
+            result.expect("concurrent rollback cleanup must reach the shared trash directory");
+            assert!(!source.exists(), "rollback cleanup must not leave a directory in the bucket");
+            assert_eq!(
+                std::fs::read(trash.join(index.to_string()).join("xl.meta.bkp")).expect("read moved metadata backup"),
+                b"rollback metadata",
+                "trash staging must retain the complete backup"
+            );
+        }
+    }
+
+    #[cfg(windows)]
+    #[test]
+    fn windows_trash_rename_succeeds_during_concurrent_parent_preparation() {
+        use std::os::windows::fs::OpenOptionsExt;
+        use std::sync::atomic::{AtomicBool, Ordering};
+        use windows_sys::Win32::{
+            Foundation::{ERROR_SHARING_VIOLATION, GENERIC_WRITE},
+            Storage::FileSystem::{
+                DELETE, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_DELETE, FILE_SHARE_READ,
+                FILE_SHARE_WRITE,
+            },
+        };
+
+        // Exercise the final parent both in the existing base walk and in the
+        // creatable suffix walk. Neither may briefly deny write sharing.
+        for nested_parent in [false, true] {
+            let temp_dir = tempdir().expect("create temp dir");
+            let tmp = temp_dir.path().join(".rustfs.sys/tmp");
+            let trash = tmp.join(".trash");
+            std::fs::create_dir_all(&trash).expect("create trash directory");
+            let base = if nested_parent { &tmp } else { &trash };
+            let publication_root = PublicationRoot::new(temp_dir.path()).expect("open publication root");
+            let sources = ["first", "second"].map(|name| {
+                let source = temp_dir
+                    .path()
+                    .join("bucket")
+                    .join(name)
+                    .join(uuid::Uuid::new_v4().to_string());
+                std::fs::create_dir_all(&source).expect("create rollback directory");
+                std::fs::write(source.join("xl.meta.bkp"), name.as_bytes()).expect("write metadata backup");
+                source
+            });
+            let destinations = [trash.join("first"), trash.join("second")];
+            let first_preparation = prepare_rename_with_retry(&sources[0], &destinations[0], base, &publication_root)
+                .expect("prepare the first trash rename");
+            let first_source = sources[0].clone();
+            let first_destination = destinations[0].clone();
+            let interleaved = Arc::new(AtomicBool::new(false));
+            let interleaved_hook = Arc::clone(&interleaved);
+            prepare_rename_test_hooks::queue_after_component_opened(&trash, move || {
+                interleaved_hook.store(true, Ordering::Release);
+                rename_prepared(&first_source, &first_destination, &first_preparation)
+                    .expect("another preparation's first parent handle must allow the pending trash rename");
+            });
+
+            let second_preparation = prepare_rename_with_retry(&sources[1], &destinations[1], base, &publication_root)
+                .expect("prepare the second trash rename");
+            assert!(interleaved.load(Ordering::Acquire), "the competing parent-open window must be exercised");
+            for (path, access) in [(&tmp, GENERIC_WRITE), (&trash, DELETE)] {
+                let err = std::fs::OpenOptions::new()
+                    .access_mode(access)
+                    .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
+                    .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
+                    .open(path)
+                    .expect_err("ancestor writes and final-parent deletion must remain excluded");
+                assert_eq!(
+                    err.raw_os_error(),
+                    Some(i32::try_from(ERROR_SHARING_VIOLATION).expect("Windows error code must fit i32"))
+                );
+            }
+            rename_prepared(&sources[1], &destinations[1], &second_preparation).expect("publish the second trash rename");
+            for (index, payload) in [b"first".as_slice(), b"second".as_slice()].into_iter().enumerate() {
+                assert!(!sources[index].exists(), "both rollback directories must leave the bucket");
+                assert_eq!(
+                    std::fs::read(destinations[index].join("xl.meta.bkp")).expect("read the staged backup"),
+                    payload
+                );
+            }
+        }
+    }
+
     #[cfg(windows)]
     #[tokio::test]
     async fn windows_rename_all_supports_same_parent_publication() {

crates/ecstore/src/runtime/sources.rs

@@ -14,7 +14,7 @@
 
 use std::{
     collections::{HashMap, HashSet},
-    sync::{Arc, OnceLock},
+    sync::{Arc, LazyLock, OnceLock, RwLock as StdRwLock},
     time::SystemTime,
 };
 
@@ -57,6 +57,13 @@ use uuid::Uuid;
 const TEST_RPC_SECRET: &str = "test-rpc-secret";
 
 pub(crate) type WorkloadSnapshotProviderRef = Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>;
+pub type ScannerDirtyUsageMutationObserver = Arc<dyn Fn(&str, &str, ScannerDirtyUsageMutationSource) + Send + Sync + 'static>;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum ScannerDirtyUsageMutationSource {
+    Replication,
+    TierExpiration,
+}
 
 #[derive(Clone, Default)]
 pub(crate) struct LockRegistry {
@@ -88,6 +95,8 @@ impl LockRegistry {
 }
 
 static WORKLOAD_ADMISSION_SNAPSHOT_PROVIDER: OnceLock<WorkloadSnapshotProviderRef> = OnceLock::new();
+static SCANNER_DIRTY_USAGE_MUTATION_OBSERVER: LazyLock<StdRwLock<Option<ScannerDirtyUsageMutationObserver>>> =
+    LazyLock::new(|| StdRwLock::new(None));
 
 pub(crate) fn set_workload_admission_snapshot_provider(
     provider: WorkloadSnapshotProviderRef,
@@ -99,6 +108,28 @@ pub(crate) fn workload_admission_snapshot_provider() -> Option<WorkloadSnapshotP
     WORKLOAD_ADMISSION_SNAPSHOT_PROVIDER.get().cloned()
 }
 
+pub fn set_scanner_dirty_usage_mutation_observer(
+    observer: Option<ScannerDirtyUsageMutationObserver>,
+) -> Option<ScannerDirtyUsageMutationObserver> {
+    let mut slot = SCANNER_DIRTY_USAGE_MUTATION_OBSERVER
+        .write()
+        .unwrap_or_else(|poisoned| poisoned.into_inner());
+    std::mem::replace(&mut *slot, observer)
+}
+
+pub(crate) fn notify_scanner_dirty_usage_mutation(bucket: &str, object: &str, source: ScannerDirtyUsageMutationSource) {
+    if bucket.is_empty() || object.is_empty() {
+        return;
+    }
+    let observer = SCANNER_DIRTY_USAGE_MUTATION_OBSERVER
+        .read()
+        .unwrap_or_else(|poisoned| poisoned.into_inner())
+        .clone();
+    if let Some(observer) = observer {
+        observer(bucket, object, source);
+    }
+}
+
 pub(crate) fn record_erasure_write_quorum_failure(stage: &'static str, dominant_error: &'static str) {
     global_internode_metrics().record_erasure_write_quorum_failure(stage, dominant_error);
 }
@@ -580,12 +611,16 @@ pub(crate) async fn init_tier_config_mgr(store: Arc<ECStore>) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::{
-        LockRegistry, clear_local_disk_id_map_for_test, local_disk_path_by_id, local_node_name, reconcile_local_disk_ids,
-        replace_local_disk_id, set_local_node_name,
+        LockRegistry, ScannerDirtyUsageMutationSource, clear_local_disk_id_map_for_test, local_disk_path_by_id, local_node_name,
+        notify_scanner_dirty_usage_mutation, reconcile_local_disk_ids, replace_local_disk_id, set_local_node_name,
+        set_scanner_dirty_usage_mutation_observer,
     };
     use crate::disk::endpoint::Endpoint;
     use rustfs_lock::{LocalClient, LockClient};
-    use std::{collections::HashMap, sync::Arc};
+    use std::{
+        collections::HashMap,
+        sync::{Arc, Mutex},
+    };
     use uuid::Uuid;
 
     fn url_endpoint(raw: &str) -> Endpoint {
@@ -620,6 +655,42 @@ mod tests {
         assert!(Arc::ptr_eq(&clients[1], &client_b));
     }
 
+    #[test]
+    #[serial_test::serial(scanner_dirty_usage_mutation_observer)]
+    fn scanner_dirty_usage_mutation_observer_filters_empty_identity_and_preserves_source() {
+        let observed = Arc::new(Mutex::new(Vec::new()));
+        let observed_clone = Arc::clone(&observed);
+        let previous = set_scanner_dirty_usage_mutation_observer(Some(Arc::new(move |bucket, object, source| {
+            observed_clone.lock().expect("observer lock should not be poisoned").push((
+                bucket.to_string(),
+                object.to_string(),
+                source,
+            ));
+        })));
+
+        notify_scanner_dirty_usage_mutation("photos", "2026/image.jpg", ScannerDirtyUsageMutationSource::Replication);
+        notify_scanner_dirty_usage_mutation("", "2026/empty-bucket.jpg", ScannerDirtyUsageMutationSource::TierExpiration);
+        notify_scanner_dirty_usage_mutation("photos", "", ScannerDirtyUsageMutationSource::TierExpiration);
+        notify_scanner_dirty_usage_mutation("archive", "expired.bin", ScannerDirtyUsageMutationSource::TierExpiration);
+        set_scanner_dirty_usage_mutation_observer(previous);
+
+        assert_eq!(
+            *observed.lock().expect("observer lock should not be poisoned"),
+            vec![
+                (
+                    "photos".to_string(),
+                    "2026/image.jpg".to_string(),
+                    ScannerDirtyUsageMutationSource::Replication
+                ),
+                (
+                    "archive".to_string(),
+                    "expired.bin".to_string(),
+                    ScannerDirtyUsageMutationSource::TierExpiration
+                ),
+            ]
+        );
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn local_node_name_round_trips_through_common_runtime_helper() {

crates/ecstore/src/services/notification_sys.rs

@@ -67,12 +67,9 @@ const DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
 // Keep this synchronized with the version served by node_service. Including
 // the local member in the minimum prevents an older coordinator from
 // self-authorizing a policy implemented only by newer remote peers.
-const LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
-/// Version 5 is reserved for a fleet whose every metadata writer preserves
-/// explicit transition version state and destination identity, and implements
-/// conditional per-generation `xl.meta` writes with strong readback. The node
-/// service must not advertise this version until the conditional writer from
-/// rustfs/backlog#684 is available.
+const LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION: u32 = 5;
+/// Version 5 preserves explicit transition state/destination bindings and
+/// supports exact-generation metadata repair with strong all-copy readback.
 const LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION: u32 = 5;
 
 fn resolve_admin_peer_probe_timeout_secs(configured: Option<u64>) -> u64 {
@@ -568,6 +565,10 @@ pub(crate) fn tier_delete_journal_topology_generation(proof: &TierDeleteJournalF
     stable_tier_delete_journal_topology_generation(&proof.token.topology_fingerprint)
 }
 
+pub(crate) fn cross_pool_fence_topology_generation(proof: &CrossPoolFenceFleetProofToken) -> String {
+    stable_tier_delete_journal_topology_generation(&proof.0.topology_fingerprint)
+}
+
 /// Acquire one non-cloneable authority that must span the complete reconcile
 /// effect window, including its final strong readback.
 pub async fn acquire_legacy_transition_state_reconcile_fleet_proof() -> Option<LegacyTransitionStateReconcileFleetProofToken> {
@@ -603,6 +604,10 @@ fn acquire_legacy_transition_state_reconcile_fleet_proof_from(
 }
 
 async fn observe_legacy_transition_state_reconcile_fleet(expected_topology: &str) -> Option<BTreeMap<String, Uuid>> {
+    #[cfg(all(test, feature = "test-util"))]
+    if let Ok(observation) = LEGACY_RECONCILE_TEST_OBSERVATION.try_with(Clone::clone) {
+        return Some(observation);
+    }
     let notification_sys = get_global_notification_sys()?;
     let (peer_epochs, minimum_version) = timeout(
         REMOTE_VERSION_STATE_PROBE_TIMEOUT,
@@ -615,6 +620,34 @@ async fn observe_legacy_transition_state_reconcile_fleet(expected_topology: &str
     reconcile_result.ok()
 }
 
+#[cfg(all(test, feature = "test-util"))]
+tokio::task_local! {
+    static LEGACY_RECONCILE_TEST_OBSERVATION: BTreeMap<String, Uuid>;
+}
+
+#[cfg(all(test, feature = "test-util"))]
+pub(crate) async fn with_legacy_transition_state_fleet_proof_for_test<F: std::future::Future>(future: F) -> F::Output {
+    struct Revoke;
+    impl Drop for Revoke {
+        fn drop(&mut self) {
+            revoke_fleet_capability_proof(legacy_transition_state_reconcile_fleet_proof_slot());
+        }
+    }
+    let topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get().expect("test store topology");
+    assert!(
+        publish_fleet_capability_probe_result(
+            legacy_transition_state_reconcile_fleet_proof_slot(),
+            topology,
+            Ok(BTreeMap::new()),
+            Instant::now(),
+        )
+        .is_none()
+    );
+    let _revoke = Revoke;
+    let _remote_version = install_current_remote_version_state_fleet_proof_for_test();
+    LEGACY_RECONCILE_TEST_OBSERVATION.scope(BTreeMap::new(), future).await
+}
+
 /// Revalidate the exact fleet generation captured by a reconcile token with a
 /// fresh synchronous observation. Callers must await this before each
 /// conditional metadata write and after the final strong readback.
@@ -633,6 +666,18 @@ pub async fn legacy_transition_state_reconcile_fleet_proof_matches(
     .await
 }
 
+/// Final local check in the disk publication executor. The corresponding
+/// counted permit remains owned until the filesystem operation has drained.
+pub(crate) fn legacy_transition_state_reconcile_fleet_proof_current(
+    proof: &LegacyTransitionStateReconcileFleetProofToken,
+) -> bool {
+    let Some(topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else { return false };
+    let state = legacy_transition_state_reconcile_fleet_proof_slot()
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    legacy_transition_state_reconcile_fleet_proof_matches_at(&state, proof, topology, Instant::now())
+}
+
 pub async fn acquire_ilm_recovery_export_fleet_proof() -> Option<IlmRecoveryExportFleetProofToken> {
     let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
     let proof = {
@@ -1120,6 +1165,14 @@ pub(crate) fn install_remote_version_state_fleet_proof_for_test(topology_fingerp
     RemoteVersionStateFleetProofGuard
 }
 
+#[cfg(all(test, feature = "test-util"))]
+pub(crate) fn install_current_remote_version_state_fleet_proof_for_test() -> RemoteVersionStateFleetProofGuard {
+    let topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY
+        .get()
+        .expect("the test store must bind its fleet topology before installing a writer proof");
+    install_remote_version_state_fleet_proof_for_test(topology)
+}
+
 #[cfg(all(test, feature = "test-util"))]
 pub(crate) struct TransitionTransactionCompactionFleetProofGuard;
 
@@ -3914,15 +3967,11 @@ mod tests {
         assert!(decommission_v3.is_err(), "v3 members do not understand the per-target decommission fence");
         assert!(reconcile_v3.is_err());
 
-        let (generic_v4, journal_v4, decommission_v4, reconcile_v4) =
-            cross_pool_fence_policy_results(peers.clone(), LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION);
+        let (generic_v4, journal_v4, decommission_v4, reconcile_v4) = cross_pool_fence_policy_results(peers.clone(), 4);
         assert!(generic_v4.is_ok());
         assert!(journal_v4.is_ok());
         assert!(decommission_v4.is_ok(), "an all-v4 fleet may create sticky per-target reservations");
-        assert!(
-            reconcile_v4.is_err(),
-            "the current local policy lacks the conditional xl.meta writer required by reconcile"
-        );
+        assert!(reconcile_v4.is_err(), "v4 does not support conditional transition metadata writes");
 
         let (generic_v5, journal_v5, decommission_v5, reconcile_v5) = cross_pool_fence_policy_results(peers, 5);
         assert!(generic_v5.is_ok());
@@ -4600,7 +4649,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn legacy_transition_state_reconcile_single_node_stays_closed_before_local_cas_support() {
+    async fn legacy_transition_state_reconcile_single_node_advertises_conditional_writer() {
         let notification_sys = NotificationSys {
             peer_clients: Vec::new(),
             all_peer_clients: vec![None],
@@ -4616,8 +4665,8 @@ mod tests {
         assert_eq!(minimum_version, LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION);
         let (_, _, _, reconcile_result) = cross_pool_fence_policy_results(peers, minimum_version);
         assert!(
-            reconcile_result.is_err(),
-            "the current node must not self-authorize reconcile before the conditional writer lands"
+            reconcile_result.is_ok(),
+            "the current node implements the conditional writer and preserves repaired bindings"
         );
     }
 

crates/ecstore/src/services/rebalance/control.rs

@@ -572,7 +572,7 @@ impl ECStore {
     where
         S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
     {
-        // Lock order: pool_meta_save_gate -> pool.bin -> rebalance.bin.
+        // Lock order: pool_meta_save_gate -> rebalance.bin -> pool.bin.
         let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
         pool_meta_guard.ensure_write_safe("rebalance worker activation")?;
         // Classify the durable rebalance record while holding both namespace

crates/ecstore/src/services/rebalance/entry.rs

@@ -50,6 +50,11 @@ fn ensure_rebalance_entry_active(cancel: &CancellationToken) -> Result<()> {
     Ok(())
 }
 
+#[cfg(test)]
+tokio::task_local! {
+    static REBALANCE_ENTRY_RUN_FENCE_BARRIER: (Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>);
+}
+
 #[derive(Debug)]
 struct RebalanceEntryTarget {
     bucket: String,
@@ -256,9 +261,15 @@ impl ECStore {
             .sort_by_key(|v| (v.mod_time.is_none(), std::cmp::Reverse(v.mod_time)));
 
         // Entry lock order is bucket incarnation -> activation_gate -> rebalance.bin -> movement gate.
+        // Target capacity admission can then acquire pool.bin under the run fence.
         // Stop waits for in-flight entries through cleanup, but not for entries admitted later.
         ensure_rebalance_entry_active(&cancel)?;
         let run_guard = self.rebalance_run_guard(rebalance_id.as_ref(), "rebalance entry").await?;
+        #[cfg(test)]
+        if let Ok((arrived, release)) = REBALANCE_ENTRY_RUN_FENCE_BARRIER.try_with(Clone::clone) {
+            arrived.notify_one();
+            release.notified().await;
+        }
         let lock_lost_signal = run_guard.lock_lost_signal();
         #[cfg(test)]
         let _run_signal_test_fence = lock_lost_signal
@@ -1237,6 +1248,130 @@ mod tests {
         assert_eq!(pool_stats.cleanup_warnings.count, 1, "deferred cleanup must not add a permanent warning");
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn real_rebalance_entry_progresses_while_peer_activation_waits_for_run_fence() {
+        const REBALANCE_ID: &str = "rebalance-peer-activation-lock-order";
+        let (_temp_dirs, store, peer) = crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(Some(
+            active_rebalance_meta(REBALANCE_ID),
+        ))
+        .await;
+        assert!(!Arc::ptr_eq(&store.ctx, &peer.ctx), "node-local movement gates must be independent");
+        {
+            let mut meta = peer.rebalance_meta.write().await;
+            let meta = meta.as_mut().expect("peer should know the durable run");
+            meta.activation_gate = Arc::default();
+            meta.cancel = None;
+        }
+        let bucket = crate::disk::RUSTFS_META_BUCKET;
+        let object = "rebalance-peer-activation-object";
+        let version_id = uuid::Uuid::new_v4();
+        let payload = b"entry must drain before peer activation takes the pool fence".repeat(1024);
+        let source_set = store.pools[0].get_disks_by_key(object);
+        let target_set = store.pools[1].get_disks_by_key(object);
+        let opts = ObjectOptions {
+            versioned: true,
+            version_id: Some(version_id.to_string()),
+            ..Default::default()
+        };
+        let mut writer = PutObjReader::from_vec(payload.clone());
+        let source_before = source_set
+            .put_object(bucket, object, &mut writer, &opts)
+            .await
+            .expect("source version should be written");
+        let entry = metacache_entry_from_source(&source_set, bucket, object).await;
+        let arrived = Arc::new(tokio::sync::Notify::new());
+        let release = Arc::new(tokio::sync::Notify::new());
+        // JoinSet aborts both scoped tasks if an assertion or timeout fails.
+        let mut tasks = tokio::task::JoinSet::new();
+        let entry_store = Arc::clone(&store);
+        tasks.spawn(
+            REBALANCE_ENTRY_RUN_FENCE_BARRIER.scope((Arc::clone(&arrived), Arc::clone(&release)), async move {
+                entry_store
+                    .rebalance_entry(
+                        RebalanceEntryTarget {
+                            bucket: bucket.to_string(),
+                            pool_index: 0,
+                        },
+                        entry,
+                        source_set,
+                        Arc::new(RebalanceBucketConfigs::default()),
+                        Arc::from(REBALANCE_ID),
+                        CancellationToken::new(),
+                    )
+                    .await
+            }),
+        );
+        tokio::time::timeout(StdDuration::from_secs(30), arrived.notified())
+            .await
+            .expect("real entry must acquire its persisted run read fence");
+
+        let attempted = Arc::new(tokio::sync::Notify::new());
+        let peer_pool = Arc::clone(&peer.pools[0]);
+        let (activation_done, activation_result) = tokio::sync::oneshot::channel();
+        tasks.spawn(
+            crate::core::pools::REBALANCE_ACTIVATION_LOCK_ATTEMPT.scope(Arc::clone(&attempted), async move {
+                let result = peer.fence_rebalance_worker_activation(peer_pool, REBALANCE_ID).await;
+                let result = result.map(|fence| match fence {
+                    super::super::control::RebalanceWorkerActivationFence::Ready(fence) => {
+                        fence.ensure_held().expect("peer activation must retain both fences");
+                    }
+                    super::super::control::RebalanceWorkerActivationFence::NotStartedTerminal => {
+                        panic!("the paused entry's run must still require activation");
+                    }
+                });
+                activation_done.send(result).expect("activation receiver should remain alive");
+                Ok(RebalanceEntryOutcome::Completed)
+            }),
+        );
+        tokio::time::timeout(StdDuration::from_secs(30), attempted.notified())
+            .await
+            .expect("peer activation must attempt the persisted rebalance write fence");
+        release.notify_one();
+
+        tokio::time::timeout(StdDuration::from_secs(30), async {
+            while let Some(result) = tasks.join_next().await {
+                assert!(matches!(
+                    result
+                        .expect("scoped task must not panic")
+                        .expect("entry must not fail or defer"),
+                    RebalanceEntryOutcome::Completed
+                ));
+            }
+        })
+        .await
+        .expect("entry and peer activation must both make progress");
+        activation_result
+            .await
+            .expect("peer activation result should be sent")
+            .expect("peer activation must not time out behind the entry it blocks");
+
+        let mut reader = target_set
+            .get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
+            .await
+            .expect("the exact target version must be readable");
+        let mut actual = Vec::new();
+        reader
+            .stream
+            .read_to_end(&mut actual)
+            .await
+            .expect("target body should drain completely");
+        assert_eq!(actual, payload);
+        assert_eq!(reader.object_info.version_id, source_before.version_id);
+        assert_eq!(reader.object_info.etag, source_before.etag);
+        assert_eq!(reader.object_info.mod_time, source_before.mod_time);
+        let source_error = store.pools[0]
+            .get_object_info(bucket, object, &opts)
+            .await
+            .expect_err("completed entry must clean up the source version");
+        assert!(crate::error::is_err_object_not_found(&source_error) || crate::error::is_err_version_not_found(&source_error));
+        let meta = store.rebalance_meta.read().await;
+        let stats = &meta.as_ref().expect("local run must remain installed").pool_stats[0];
+        assert_eq!(stats.num_objects, 1);
+        assert_eq!(stats.num_versions, 1);
+        assert_eq!(stats.cleanup_warnings.count, 0);
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn real_rebalance_run_fence_loss_before_target_commit_preserves_target_and_source() {

crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs

@@ -1907,6 +1907,124 @@ fn test_is_transient_rebalance_error_accepts_wrapped_disk_timeout() {
     assert!(is_transient_rebalance_error(&Error::Io(std::io::Error::other(DiskError::Timeout))));
 }
 
+#[test]
+fn test_rebalance_stage_wrapped_transient_errors_remain_retryable() {
+    let cases = [
+        Error::Lock(rustfs_lock::LockError::timeout(".rustfs.sys/pool.bin@latest", Duration::from_secs(5))),
+        Error::Lock(rustfs_lock::LockError::network(
+            "peer unavailable",
+            std::io::Error::from(std::io::ErrorKind::ConnectionReset),
+        )),
+        Error::SlowDown,
+        Error::ErasureReadQuorum,
+        Error::ErasureWriteQuorum,
+        Error::Io(std::io::Error::other(DiskError::Timeout)),
+        Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),
+    ];
+    for mut error in cases {
+        for depth in 0..=3 {
+            assert!(is_transient_rebalance_error(&error), "transient source lost at depth {depth}: {error:?}");
+            assert!(
+                should_defer_rebalance_entry_failure(&error),
+                "exhausted transient entries must be deferred"
+            );
+            assert!(should_retry_rebalance_listing(&error, 0, 3));
+            assert!(
+                !should_retry_rebalance_listing(&error, 2, 3),
+                "wrapping must not bypass the attempt limit"
+            );
+            error = data_movement::data_movement_stage_error_for_test(
+                "rebalance_object",
+                "put_object",
+                "bucket",
+                "baseline/00042.bin",
+                error,
+            );
+        }
+    }
+}
+
+#[test]
+fn test_rebalance_stage_wrapped_terminal_errors_remain_terminal() {
+    let cases = [
+        Error::FileAccessDenied,
+        Error::FileCorrupt,
+        Error::OperationCanceled,
+        Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()),
+        Error::Lock(rustfs_lock::LockError::already_locked("bucket/object", "owner")),
+        Error::other("permission denied"),
+    ];
+    for mut error in cases {
+        for depth in 0..=3 {
+            assert!(
+                !is_transient_rebalance_error(&error),
+                "terminal source must survive depth {depth}: {error:?}"
+            );
+            assert!(!should_defer_rebalance_entry_failure(&error));
+            // Object names are untrusted context, not evidence of a transient failure.
+            error = data_movement::data_movement_stage_error_for_test(
+                "rebalance_object",
+                "put_object",
+                "bucket",
+                "remote lock rpc timed out",
+                error,
+            );
+        }
+    }
+}
+
+#[tokio::test]
+async fn test_rebalance_stage_wrapped_lock_timeout_retries_real_migration_loop() {
+    for succeeds_on_retry in [true, false] {
+        let backend = MigrationBackendSpy::new(None, None);
+        let attempts = AtomicUsize::new(0);
+        let waits = AtomicUsize::new(0);
+        let mut transfer = |_, _, _| {
+            let attempt = attempts.fetch_add(1, Ordering::SeqCst);
+            async move {
+                if succeeds_on_retry && attempt > 0 {
+                    return Ok(());
+                }
+                Err(data_movement::data_movement_stage_error_for_test(
+                    "rebalance_object",
+                    "put_object",
+                    "bucket",
+                    "baseline/00042.bin",
+                    Error::Lock(rustfs_lock::LockError::timeout(".rustfs.sys/pool.bin@latest", Duration::from_secs(5))),
+                ))
+            }
+        };
+        let version = version_normal();
+        let result = migrate_entry_version_with_retry_wait(
+            &backend,
+            "bucket".to_string(),
+            0,
+            &version,
+            None,
+            3,
+            false,
+            &mut transfer,
+            |_: String, _: String, _: ObjectOptions| async { Ok::<_, Error>(ObjectInfo::default()) },
+            |_| {
+                waits.fetch_add(1, Ordering::SeqCst);
+                std::future::ready(())
+            },
+        )
+        .await;
+        assert_eq!(result.moved, succeeds_on_retry);
+        assert_eq!(result.failed, !succeeds_on_retry);
+        assert_eq!(attempts.load(Ordering::SeqCst), if succeeds_on_retry { 2 } else { 3 });
+        assert_eq!(backend.get_calls(), attempts.load(Ordering::SeqCst));
+        assert_eq!(waits.load(Ordering::SeqCst), attempts.load(Ordering::SeqCst) - 1);
+        if !succeeds_on_retry {
+            assert_eq!(result.stage, Some("write_target"));
+            assert!(should_defer_rebalance_entry_failure(
+                result.error.as_ref().expect("exhaustion must retain its source error")
+            ));
+        }
+    }
+}
+
 #[test]
 fn test_is_transient_rebalance_error_accepts_io_timeout_message() {
     assert!(is_transient_rebalance_error(&Error::Io(std::io::Error::other("timeout"))));

crates/ecstore/src/services/rebalance/worker.rs

@@ -244,6 +244,7 @@ pub(super) fn resolve_rebalance_bucket_result(
 }
 
 pub(super) fn is_transient_rebalance_error(err: &Error) -> bool {
+    let err = rebalance_error_source(err);
     match err {
         Error::SlowDown
         | Error::ErasureReadQuorum
@@ -256,6 +257,15 @@ pub(super) fn is_transient_rebalance_error(err: &Error) -> bool {
     }
 }
 
+fn rebalance_error_source(mut err: &Error) -> &Error {
+    // Stage context contains object names, so classify the preserved source,
+    // not timeout-like text supplied by an object name. Iterate nested stages.
+    while let Some(source) = crate::data_movement::data_movement_stage_source(err) {
+        err = source;
+    }
+    err
+}
+
 fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
     match err {
         rustfs_lock::LockError::Timeout { .. } | rustfs_lock::LockError::Network { .. } => true,
@@ -309,6 +319,7 @@ pub(super) fn rebalance_listing_retry_delay(attempt: usize) -> Duration {
 }
 
 fn is_rebalance_lock_or_rpc_timeout(err: &Error) -> bool {
+    let err = rebalance_error_source(err);
     match err {
         Error::Lock(rustfs_lock::LockError::Timeout { .. }) | Error::Lock(rustfs_lock::LockError::Network { .. }) => true,
         Error::Io(io_err) => is_rebalance_lock_or_rpc_timeout_message(&io_err.to_string()),
@@ -585,3 +596,48 @@ impl SetDisks {
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod error_source_tests {
+    use super::*;
+
+    #[test]
+    fn stage_wrapped_errors_select_the_source_backoff_policy() {
+        let cases = [
+            (
+                Error::Lock(rustfs_lock::LockError::timeout(".rustfs.sys/pool.bin@latest", Duration::from_secs(5))),
+                true,
+            ),
+            (
+                Error::Lock(rustfs_lock::LockError::network(
+                    "peer unavailable",
+                    std::io::Error::from(std::io::ErrorKind::ConnectionReset),
+                )),
+                true,
+            ),
+            (Error::other("remote lock rpc timed out"), true),
+            (Error::SlowDown, false),
+            (Error::Io(std::io::Error::other(DiskError::Timeout)), false),
+            (Error::FileAccessDenied, false),
+        ];
+        for (mut error, lock_backoff) in cases {
+            for depth in 0..=3 {
+                assert_eq!(
+                    is_rebalance_lock_or_rpc_timeout(&error),
+                    lock_backoff,
+                    "wrong backoff at depth {depth}: {error:?}"
+                );
+                if !lock_backoff {
+                    assert_eq!(rebalance_migration_retry_delay(1, &error), REBALANCE_MIGRATION_RETRY_BASE_DELAY * 2);
+                }
+                error = crate::data_movement::data_movement_stage_error_for_test(
+                    "rebalance_object",
+                    "put_object",
+                    "bucket",
+                    "remote lock rpc timed out",
+                    error,
+                );
+            }
+        }
+    }
+}

crates/ecstore/src/services/tier/test_util.rs

@@ -653,6 +653,26 @@ impl MockWarmBackend {
 
 #[async_trait]
 impl WarmBackend for MockWarmBackend {
+    async fn probe_legacy_metadata(
+        &self,
+        object: &str,
+        remote_version: Option<&str>,
+    ) -> Result<super::warm_backend::LegacyTransitionStateProbe, std::io::Error> {
+        use super::warm_backend::LegacyTransitionStateProbe as Probe;
+        let candidate = match remote_version {
+            Some(version) if !version.is_empty() => self.probe_transition_version(object, version).await?,
+            _ => self.probe_transition_candidate(object).await?,
+        };
+        Ok(match candidate {
+            TransitionCandidateProbe::Missing => Probe::Missing,
+            TransitionCandidateProbe::UnversionedPresent => Probe::UnversionedPresent,
+            TransitionCandidateProbe::VersionedPresent(version) if version == "null" => Probe::SuspendedNullPresent,
+            TransitionCandidateProbe::VersionedPresent(version) => Probe::VersionedPresent(version),
+            TransitionCandidateProbe::Ambiguous => Probe::Ambiguous,
+            TransitionCandidateProbe::Unsupported => Probe::Unsupported,
+        })
+    }
+
     fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> {
         if remote_version_id.is_empty() {
             return Ok(());
@@ -874,8 +894,9 @@ pub async fn register_mock_tier_backend(handle: &Arc<RwLock<TierConfigMgr>>, tie
             ..Default::default()
         },
     );
-    tier_config_mgr
-        .install_test_driver(tier_name, Box::new(backend))
+    drop(tier_config_mgr);
+    TierConfigMgr::install_test_driver_in(handle, tier_name, Box::new(backend))
+        .await
         .expect("mock tier driver should install");
 }
 

crates/ecstore/src/services/tier/tier.rs

@@ -2322,6 +2322,14 @@ struct SharedWarmBackendProxy(SharedWarmBackend);
 
 #[async_trait::async_trait]
 impl WarmBackend for SharedWarmBackendProxy {
+    async fn probe_legacy_metadata(
+        &self,
+        object: &str,
+        remote_version: Option<&str>,
+    ) -> io::Result<crate::services::tier::warm_backend::LegacyTransitionStateProbe> {
+        self.0.probe_legacy_metadata(object, remote_version).await
+    }
+
     async fn validate(&self) -> io::Result<()> {
         self.0.validate().await
     }
@@ -2490,6 +2498,27 @@ impl TierOperationLease {
         self.inner.driver.probe_transition_version(object, remote_version_id).await
     }
 
+    pub(crate) async fn probe_legacy_transition_state(
+        &self,
+        object: &str,
+        remote_version: Option<&str>,
+    ) -> io::Result<crate::services::tier::warm_backend::LegacyTransitionStateProbe> {
+        let Some(reconciler) = self
+            .inner
+            .reconciler
+            .get_or_try_init(|| async {
+                crate::services::tier::warm_backend::new_transition_candidate_reconciler(&self.inner.tier_config)
+                    .await
+                    .map(|reconciler| reconciler.map(Arc::from))
+            })
+            .await
+            .map_err(|err| io::Error::other(err.message))?
+        else {
+            return self.inner.driver.probe_legacy_metadata(object, remote_version).await;
+        };
+        reconciler.probe_legacy_transition_state(object, remote_version).await
+    }
+
     pub(crate) fn is_current_generation(&self) -> bool {
         lock_unpoisoned(&self.runtime)
             .generations
@@ -6013,6 +6042,19 @@ impl TierConfigMgr {
         Ok(())
     }
 
+    #[cfg(any(test, feature = "test-util"))]
+    pub(crate) async fn install_test_driver_in(
+        handle: &Arc<RwLock<Self>>,
+        tier_name: &str,
+        driver: WarmBackendImpl,
+    ) -> std::result::Result<(), AdminError> {
+        let mut manager = handle.write().await;
+        // Register the generation runtime before installing the mock so its
+        // explicit lack of a network reconciler survives the first lease.
+        tier_driver_runtime(handle, &manager);
+        manager.install_test_driver(tier_name, driver)
+    }
+
     #[cfg(any(test, feature = "test-util"))]
     pub(crate) fn install_test_driver(
         &mut self,
@@ -17498,6 +17540,66 @@ mod tests {
         assert!(current.tiers.contains_key("COLD-B"));
     }
 
+    async fn wait_for_reference_proof_barrier(
+        barrier: &TierDriverBuildBarrier,
+        update: &mut tokio::task::JoinHandle<std::result::Result<(), TierConfigUpdateError>>,
+    ) -> std::result::Result<(), String> {
+        tokio::select! {
+            biased;
+            result = &mut *update => Err(format!("tier update exited before the reference proof barrier: {result:?}")),
+            () = barrier.arrived.notified() => Ok(()),
+            () = tokio::time::sleep(Duration::from_secs(30)) => {
+                // Aborting the caller does not stop its owned mutation task.
+                // Let a late arrival pass the test-only barrier.
+                barrier.release.add_permits(1);
+                update.abort();
+                Err("timed out waiting for the reference proof barrier".to_string())
+            }
+        }
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn reference_proof_barrier_reports_update_failure_before_arrival() {
+        let manager = TierConfigMgr::new();
+        let store = Arc::new(CasConfigStore::default());
+        let mut persisted = empty_mgr();
+        persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
+        persisted
+            .save_tiering_config_if_current(store.clone(), None)
+            .await
+            .expect("early update failure fixture should persist");
+        let barrier = tier_reference_proof_test_barrier();
+        let scoped_barrier = barrier.clone();
+        let factory: TierDriverTestFactory =
+            Arc::new(|_| Err(AdminError::msg("injected driver initialization failure before reference proof")));
+        let mut update = tokio::spawn(async move {
+            TIER_REFERENCE_PROOF_TEST_BARRIER
+                .scope(
+                    scoped_barrier,
+                    TIER_DRIVER_TEST_FACTORY.scope(
+                        factory,
+                        TIER_MUTATION_TEST_PEERS.scope(
+                            Vec::new(),
+                            TierConfigMgr::update_candidate_with_config_lock(
+                                &manager,
+                                store,
+                                TierCandidateMutation::Remove("COLD-A".to_string(), true),
+                            ),
+                        ),
+                    ),
+                )
+                .await
+        });
+
+        let err = tokio::time::timeout(Duration::from_secs(5), wait_for_reference_proof_barrier(&barrier, &mut update))
+            .await
+            .expect("an early update failure should be observed without waiting for the barrier deadline")
+            .expect_err("a failed update cannot reach the reference proof barrier");
+        assert!(err.contains("Mutation"), "{err}");
+        assert!(err.contains("injected driver initialization failure before reference proof"), "{err}");
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn reference_proof_rejects_a_changed_prepared_fence_revision_before_publish() {
@@ -17518,7 +17620,7 @@ mod tests {
         let scoped_barrier = barrier.clone();
         let update_manager = manager.clone();
         let update_store = store.clone();
-        let update = tokio::spawn(async move {
+        let mut update = tokio::spawn(async move {
             TIER_REFERENCE_PROOF_TEST_BARRIER
                 .scope(
                     scoped_barrier,
@@ -17533,7 +17635,9 @@ mod tests {
                 )
                 .await
         });
-        barrier.arrived.notified().await;
+        wait_for_reference_proof_barrier(&barrier, &mut update)
+            .await
+            .expect("tier update should reach the reference proof barrier");
 
         let unrelated = prepared_remove_intent("COLD-B", uuid::Uuid::from_u128(0x2237));
         TierConfigMgr::apply_prepared_mutation_intent_block(&manager, &unrelated)
@@ -17541,8 +17645,9 @@ mod tests {
             .expect("an unrelated prepared fence should advance the runtime revision");
         barrier.release.add_permits(1);
 
-        let err = update
+        let err = tokio::time::timeout(Duration::from_secs(30), update)
             .await
+            .expect("tier update should finish after the reference proof barrier releases")
             .expect("tier update task should join")
             .expect_err("a reference proof cannot authorize publication across a fence revision change");
         let TierConfigUpdateError::Publish(err) = err else {

crates/ecstore/src/services/tier/warm_backend.rs

@@ -89,6 +89,18 @@ pub enum TransitionCandidateProbe {
     Unsupported,
 }
 
+/// Live evidence for repairing legacy metadata. Ordinary candidate GETs do not
+/// establish the bucket's versioning model and cannot supply this authority.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub enum LegacyTransitionStateProbe {
+    Missing,
+    UnversionedPresent,
+    SuspendedNullPresent,
+    VersionedPresent(String),
+    Ambiguous,
+    Unsupported,
+}
+
 #[derive(Clone, Copy)]
 pub(crate) struct TransitionCandidateIdentity {
     pub transaction_id: uuid::Uuid,
@@ -97,6 +109,14 @@ pub(crate) struct TransitionCandidateIdentity {
 
 #[async_trait::async_trait]
 pub(crate) trait TransitionCandidateReconciler {
+    async fn probe_legacy_transition_state(
+        &self,
+        _object: &str,
+        _remote_version: Option<&str>,
+    ) -> Result<LegacyTransitionStateProbe, std::io::Error> {
+        Ok(LegacyTransitionStateProbe::Unsupported)
+    }
+
     async fn probe_transition_candidate_for(
         &self,
         object: &str,
@@ -106,6 +126,14 @@ pub(crate) trait TransitionCandidateReconciler {
 
 #[async_trait::async_trait]
 pub trait WarmBackend {
+    async fn probe_legacy_metadata(
+        &self,
+        _object: &str,
+        _remote_version: Option<&str>,
+    ) -> Result<LegacyTransitionStateProbe, std::io::Error> {
+        Ok(LegacyTransitionStateProbe::Unsupported)
+    }
+
     async fn validate(&self) -> Result<(), std::io::Error> {
         Ok(())
     }
@@ -448,6 +476,18 @@ impl MeteredWarmBackend {
 
 #[async_trait::async_trait]
 impl WarmBackend for MeteredWarmBackend {
+    async fn probe_legacy_metadata(
+        &self,
+        object: &str,
+        remote_version: Option<&str>,
+    ) -> Result<LegacyTransitionStateProbe, std::io::Error> {
+        let result = self.inner.probe_legacy_metadata(object, remote_version).await;
+        if matches!(result, Ok(LegacyTransitionStateProbe::Unsupported)) {
+            return result;
+        }
+        Self::record(TierRequestOperation::Probe, result)
+    }
+
     /// Delegated without a counter: only one backend issues a remote request
     /// here, and every other one takes the trait default, so a `validate`
     /// counter would mostly record requests that never happened.
@@ -524,6 +564,18 @@ struct MeteredTransitionCandidateReconciler {
 
 #[async_trait::async_trait]
 impl TransitionCandidateReconciler for MeteredTransitionCandidateReconciler {
+    async fn probe_legacy_transition_state(
+        &self,
+        object: &str,
+        remote_version: Option<&str>,
+    ) -> Result<LegacyTransitionStateProbe, std::io::Error> {
+        let result = self.inner.probe_legacy_transition_state(object, remote_version).await;
+        if matches!(result, Ok(LegacyTransitionStateProbe::Unsupported)) {
+            return result;
+        }
+        MeteredWarmBackend::record(TierRequestOperation::Probe, result)
+    }
+
     async fn probe_transition_candidate_for(
         &self,
         object: &str,

crates/ecstore/src/services/tier/warm_backend_minio.rs

@@ -101,6 +101,19 @@ impl WarmBackend for WarmBackendMinIO {
 
 #[async_trait::async_trait]
 impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendMinIO {
+    async fn probe_legacy_transition_state(
+        &self,
+        object: &str,
+        remote_version: Option<&str>,
+    ) -> Result<super::warm_backend::LegacyTransitionStateProbe, std::io::Error> {
+        crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_legacy_transition_state(
+            &self.0,
+            object,
+            remote_version,
+        )
+        .await
+    }
+
     async fn probe_transition_candidate_for(
         &self,
         object: &str,

crates/ecstore/src/services/tier/warm_backend_rustfs.rs

@@ -146,6 +146,19 @@ impl WarmBackend for WarmBackendRustFS {
 
 #[async_trait::async_trait]
 impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendRustFS {
+    async fn probe_legacy_transition_state(
+        &self,
+        object: &str,
+        remote_version: Option<&str>,
+    ) -> Result<super::warm_backend::LegacyTransitionStateProbe, std::io::Error> {
+        crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_legacy_transition_state(
+            &self.0,
+            object,
+            remote_version,
+        )
+        .await
+    }
+
     async fn probe_transition_candidate_for(
         &self,
         object: &str,

crates/ecstore/src/services/tier/warm_backend_s3.rs

@@ -376,7 +376,6 @@ struct TransitionCandidateVersions {
 }
 
 impl TransitionCandidateVersions {
-    #[cfg(test)]
     fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) {
         for version in versions.versions.iter().filter(|version| version.key == remote_object) {
             if self.version_id.is_some() {
@@ -512,6 +511,20 @@ mod tests {
     }
 
     async fn candidate_probe_fixture() -> Option<(WarmBackendS3, tokio::task::JoinHandle<Vec<String>>)> {
+        scripted_probe_fixture([
+                "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: opaque-version\r\nConnection: close\r\n\r\nx",
+                "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nConnection: close\r\n\r\nx",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\n<Error><Code>NoSuchObject</Code><Message>missing</Message></Error>",
+                "HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\n<Error><Code>AccessDenied</Code><Message>denied</Message></Error>",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
+                "HTTP/1.1 416 Range Not Satisfiable\r\nContent-Type: application/xml\r\nContent-Length: 72\r\nConnection: close\r\n\r\n<Error><Code>InvalidRange</Code><Message>empty version</Message></Error>",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 67\r\nConnection: close\r\n\r\n<Error><Code>NoSuchVersion</Code><Message>missing</Message></Error>",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
+            ].into_iter().map(str::to_owned).collect()).await
+    }
+
+    async fn scripted_probe_fixture(responses: Vec<String>) -> Option<(WarmBackendS3, tokio::task::JoinHandle<Vec<String>>)> {
         let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
             Ok(listener) => listener,
             Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
@@ -522,17 +535,6 @@ mod tests {
             .expect("listener local address should be available")
             .to_string();
         let fixture = tokio::spawn(async move {
-            let responses = [
-                "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: opaque-version\r\nConnection: close\r\n\r\nx",
-                "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nConnection: close\r\n\r\nx",
-                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
-                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\n<Error><Code>NoSuchObject</Code><Message>missing</Message></Error>",
-                "HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\n<Error><Code>AccessDenied</Code><Message>denied</Message></Error>",
-                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
-                "HTTP/1.1 416 Range Not Satisfiable\r\nContent-Type: application/xml\r\nContent-Length: 72\r\nConnection: close\r\n\r\n<Error><Code>InvalidRange</Code><Message>empty version</Message></Error>",
-                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 67\r\nConnection: close\r\n\r\n<Error><Code>NoSuchVersion</Code><Message>missing</Message></Error>",
-                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
-            ];
             let mut requests = Vec::new();
             for response in responses {
                 let (mut stream, _) = listener.accept().await.expect("fixture should accept candidate GET");
@@ -673,6 +675,117 @@ mod tests {
         assert!(requests[8].to_ascii_lowercase().contains("?versionid=historical-version"));
     }
 
+    fn legacy_probe_xml_response(body: &str) -> String {
+        format!(
+            "HTTP/1.1 200 OK\r\nContent-Type: application/xml\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+            body.len()
+        )
+    }
+
+    fn legacy_probe_versioning_response(status: &str) -> String {
+        let state = if status.is_empty() {
+            String::new()
+        } else {
+            format!("<Status>{status}</Status>")
+        };
+        legacy_probe_xml_response(&format!(
+            "<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">{state}</VersioningConfiguration>"
+        ))
+    }
+
+    fn legacy_probe_versions_response(versions: &[&str]) -> String {
+        let versions = versions.iter().map(|version| format!(
+            "<Version><Key>archive/object</Key><VersionId>{version}</VersionId><IsLatest>true</IsLatest><LastModified>2026-09-01T00:00:00Z</LastModified><ETag>\"legacy-etag\"</ETag><Size>7</Size><StorageClass>STANDARD</StorageClass></Version>"
+        )).collect::<String>();
+        legacy_probe_xml_response(&format!(
+            "<ListVersionsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Name>bucket</Name><Prefix>archive/object</Prefix><KeyMarker/><VersionIdMarker/><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>{versions}</ListVersionsResult>"
+        ))
+    }
+
+    #[tokio::test]
+    async fn legacy_transition_state_probe_verifies_disabled_suspended_and_enabled_responses() {
+        use super::super::warm_backend::LegacyTransitionStateProbe as Probe;
+        for (initial, confirmed, version, expected) in [
+            ("", "", "null", Probe::UnversionedPresent),
+            ("Suspended", "Suspended", "null", Probe::SuspendedNullPresent),
+            ("Enabled", "Enabled", "version-a", Probe::VersionedPresent("version-a".to_string())),
+            ("Enabled", "Enabled", "null", Probe::SuspendedNullPresent),
+            ("", "", "unexpected-version", Probe::Ambiguous),
+            ("Suspended", "Enabled", "null", Probe::Ambiguous),
+        ] {
+            let responses = vec![
+                legacy_probe_versioning_response(initial),
+                legacy_probe_versions_response(&[version]),
+                legacy_probe_versioning_response(confirmed),
+            ];
+            let (backend, fixture) = scripted_probe_fixture(responses)
+                .await
+                .expect("legacy probe loopback fixture");
+            let result =
+                tokio::time::timeout(Duration::from_secs(10), backend.probe_legacy_transition_state("archive/object", None))
+                    .await
+                    .expect("legacy probe must finish")
+                    .expect("legacy probe should decode provider XML");
+            assert_eq!(result, expected, "initial={initial} confirmed={confirmed} version={version}");
+            let requests = fixture.await.expect("legacy probe fixture should finish");
+            assert_eq!(requests.len(), 3);
+            assert!(requests.iter().all(|request| request.starts_with("GET ")));
+            assert!(requests[0].lines().next().expect("request line").contains("versioning"));
+            assert!(requests[1].lines().next().expect("request line").contains("versions"));
+            assert!(requests[2].lines().next().expect("request line").contains("versioning"));
+        }
+    }
+
+    #[tokio::test]
+    async fn legacy_transition_state_probe_preserves_historical_exact_version() {
+        use super::super::warm_backend::LegacyTransitionStateProbe as Probe;
+        let responses = vec![
+            legacy_probe_versioning_response("Enabled"),
+            "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: historical-version\r\nConnection: close\r\n\r\nx".to_string(),
+            legacy_probe_versioning_response("Enabled"),
+        ];
+        let (backend, fixture) = scripted_probe_fixture(responses).await.expect("exact legacy probe fixture");
+        assert_eq!(
+            backend
+                .probe_legacy_transition_state("archive/object", Some("historical-version"))
+                .await
+                .expect("exact version proof"),
+            Probe::VersionedPresent("historical-version".to_string())
+        );
+        let requests = fixture.await.expect("exact probe fixture should finish");
+        assert!(
+            requests[1]
+                .lines()
+                .next()
+                .expect("request line")
+                .contains("versionId=historical-version")
+        );
+        assert!(requests[1].to_ascii_lowercase().contains("range: bytes=0-0"));
+        assert!(requests.iter().all(|request| request.starts_with("GET ")));
+    }
+
+    #[tokio::test]
+    async fn legacy_transition_state_probe_retains_multiple_candidates() {
+        use super::super::warm_backend::LegacyTransitionStateProbe as Probe;
+        let responses = vec![
+            legacy_probe_versioning_response("Enabled"),
+            legacy_probe_versions_response(&["version-a", "version-b"]),
+        ];
+        let (backend, fixture) = scripted_probe_fixture(responses)
+            .await
+            .expect("ambiguous legacy probe fixture");
+        assert_eq!(
+            backend
+                .probe_legacy_transition_state("archive/object", None)
+                .await
+                .expect("ambiguous proof"),
+            Probe::Ambiguous
+        );
+        let requests = fixture.await.expect("ambiguous fixture should finish");
+        assert_eq!(requests.len(), 2);
+        assert!(requests.iter().all(|request| request.starts_with("GET ")));
+    }
+
     fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult {
         ListVersionsResult {
             versions: versions
@@ -921,6 +1034,56 @@ impl WarmBackend for WarmBackendS3 {
 
 #[async_trait::async_trait]
 impl TransitionCandidateReconciler for WarmBackendS3 {
+    async fn probe_legacy_transition_state(
+        &self,
+        object: &str,
+        remote_version: Option<&str>,
+    ) -> Result<super::warm_backend::LegacyTransitionStateProbe, std::io::Error> {
+        use super::warm_backend::LegacyTransitionStateProbe as Probe;
+
+        let initial_versioning = self.remote_bucket_versioning().await?;
+        let candidate = if let Some(version) = remote_version.filter(|version| !version.is_empty()) {
+            validate_remote_version_id(version)?;
+            match self.probe_transition_version(object, version).await? {
+                TransitionCandidateProbe::VersionedPresent(actual) if actual == version => Some(actual),
+                TransitionCandidateProbe::Missing => return Ok(Probe::Missing),
+                _ => return Ok(Probe::Ambiguous),
+            }
+        } else {
+            let remote_object = self.get_dest(object);
+            let mut opts = ListObjectsOptions::default();
+            opts.set("prefix", &remote_object);
+            opts.set("max-keys", "1000");
+            let mut key_marker = String::new();
+            let mut version_marker = String::new();
+            let mut candidates = TransitionCandidateVersions::default();
+            let mut complete = false;
+            // This is one synchronous record inspection, not an unbounded
+            // remote history scan. The caller also bounds the whole probe.
+            for _ in 0..128 {
+                let page = self
+                    .client
+                    .list_object_versions_query(&self.bucket, &opts, &key_marker, &version_marker, "")
+                    .await?;
+                candidates.extend(&remote_object, &page);
+                if candidates.ambiguous {
+                    return Ok(Probe::Ambiguous);
+                }
+                if !page.is_truncated {
+                    complete = true;
+                    break;
+                }
+                advance_version_markers(&mut key_marker, &mut version_marker, &page)?;
+            }
+            if !complete {
+                return Ok(Probe::Ambiguous);
+            }
+            candidates.version_id
+        };
+        let confirmed_versioning = self.remote_bucket_versioning().await?;
+        classify_legacy_transition_state(candidate.as_deref(), initial_versioning, confirmed_versioning)
+    }
+
     async fn probe_transition_candidate_for(
         &self,
         object: &str,
@@ -931,3 +1094,32 @@ impl TransitionCandidateReconciler for WarmBackendS3 {
             .await
     }
 }
+
+fn classify_legacy_transition_state(
+    candidate: Option<&str>,
+    initial: RemoteBucketVersioning,
+    confirmed: RemoteBucketVersioning,
+) -> Result<super::warm_backend::LegacyTransitionStateProbe, std::io::Error> {
+    use super::warm_backend::LegacyTransitionStateProbe as Probe;
+    if initial != confirmed {
+        return Ok(Probe::Ambiguous);
+    }
+    let Some(version) = candidate else {
+        return Ok(Probe::Missing);
+    };
+    if !version.is_empty() {
+        validate_remote_version_id(version)?;
+        if uuid::Uuid::parse_str(version).is_ok_and(|id| id.is_nil()) {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::InvalidData,
+                "legacy tier probe returned a nil version identifier",
+            ));
+        }
+    }
+    Ok(match (confirmed, version) {
+        (RemoteBucketVersioning::Disabled, "" | "null") => Probe::UnversionedPresent,
+        (RemoteBucketVersioning::Disabled, _) | (_, "") => Probe::Ambiguous,
+        (_, "null") => Probe::SuspendedNullPresent,
+        (_, version) => Probe::VersionedPresent(version.to_string()),
+    })
+}

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -51,12 +51,12 @@ use super::super::{
     can_try_inline_data_shards_direct, capacity_scope_from_disks, codec_streaming_rollout_applies, coding,
     collect_inline_data_shard_fileinfos_by_index_or_reason, current_dirty_generation, debug, disk,
     file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, info, inline_erasure_shard_file_offset,
-    inline_erasure_shard_size, is_err_object_not_found, is_err_version_not_found, is_get_metadata_data_read_early_stop_enabled,
-    is_get_metadata_early_stop_bounded_fanout_enabled, is_get_metadata_early_stop_enabled,
-    is_get_metadata_non_inline_data_read_early_stop_enabled, is_object_dangling, is_version_early_stop_enabled,
-    issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure, merge_file_meta_versions,
-    object_fits_single_block, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs, reduce_write_quorum_errs,
-    send_heal_request_with_admission, should_prevent_write, to_object_err, try_read_inline_data_shards_direct, warn,
+    inline_erasure_shard_size, is_get_metadata_data_read_early_stop_enabled, is_get_metadata_early_stop_bounded_fanout_enabled,
+    is_get_metadata_early_stop_enabled, is_get_metadata_non_inline_data_read_early_stop_enabled, is_object_dangling,
+    is_version_early_stop_enabled, issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure,
+    merge_file_meta_versions, object_fits_single_block, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs,
+    reduce_write_quorum_errs, send_heal_request_with_admission, should_prevent_write, to_object_err,
+    try_read_inline_data_shards_direct, warn,
 };
 #[cfg(test)]
 pub(in crate::set_disk) use super::metadata_quorum::MetadataEarlyStopDecision;
@@ -3086,21 +3086,61 @@ impl SetDisks {
         let read_quorum = disks.len().div_ceil(2).max(1);
         let (raw_fileinfos, errs) = Self::read_all_raw_file_info(&disks, bucket, disk_object.as_str(), false).await;
 
-        if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
-            let object_err = to_object_err(err.into(), vec![bucket, object]);
-            if is_err_object_not_found(&object_err) || is_err_version_not_found(&object_err) {
-                return Ok(None);
-            }
-            return Err(object_err);
+        if let Some(err) = errs
+            .iter()
+            .flatten()
+            .find(|err| !matches!(err, DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound))
+        {
+            return Err(to_object_err(err.clone().into(), vec![bucket, object]));
+        }
+        // A minority live owner must not disappear behind majority absence.
+        // Only explicit absence on every readable disk proves no ownership.
+        if raw_fileinfos.iter().all(Option::is_none) {
+            return Ok(None);
         }
 
         let mut shallow_versions = Vec::with_capacity(raw_fileinfos.len());
+        type TransitionCopy = (FileInfo, Option<crate::services::tier::tier::TierDestinationId>);
+        let mut transition_copies: std::collections::HashMap<Option<Uuid>, Vec<TransitionCopy>> =
+            std::collections::HashMap::new();
+        let decode_error = |err| Error::other(format!("exact object versions decode failed for {bucket}/{object}: {err}"));
         for raw_fileinfo in raw_fileinfos.into_iter().flatten() {
             let meta = FileMeta::load(&raw_fileinfo.buf)
                 .map_err(|err| Error::other(format!("exact object metadata decode failed for {bucket}/{object}: {err}")))?;
+            let versions = meta.get_all_file_info_versions(bucket, object, true).map_err(decode_error)?;
+            for version in versions.versions.into_iter().chain(versions.free_versions) {
+                if version.transition_status != rustfs_filemeta::TRANSITION_COMPLETE {
+                    continue;
+                }
+                let destination =
+                    crate::services::tier::tier::tier_destination_id_from_metadata(&version.metadata).map_err(Error::other)?;
+                transition_copies
+                    .entry(version.version_id.filter(|id| !id.is_nil()))
+                    .or_default()
+                    .push((version, destination));
+            }
             shallow_versions.push(meta.versions);
         }
 
+        // Exact cleanup/recovery reads must not select a repaired majority
+        // while another physical copy still carries legacy absence. Missing
+        // copies permit deletion retries; an unreadable disk proves nothing.
+        for copies in transition_copies
+            .values()
+            .filter(|copies| copies.iter().any(|(_, destination)| destination.is_some()))
+        {
+            let (first, destination) = &copies[0];
+            if copies.iter().any(|(copy, identity)| {
+                identity != destination
+                    || copy.transition_version_state != first.transition_version_state
+                    || copy.transition_version != first.transition_version
+                    || copy.transition_tier != first.transition_tier
+                    || copy.transitioned_objname != first.transitioned_objname
+            }) {
+                return Err(Error::other("exact transition metadata has not converged across physical copies"));
+            }
+        }
+
         if shallow_versions.len() < read_quorum {
             return Err(to_object_err(StorageError::ErasureReadQuorum, vec![bucket, object]));
         }
@@ -3117,7 +3157,7 @@ impl SetDisks {
             ..Default::default()
         }
         .get_all_file_info_versions(bucket, object, true)
-        .map_err(|err| Error::other(format!("exact object versions decode failed for {bucket}/{object}: {err}")))?;
+        .map_err(decode_error)?;
 
         for file_info in file_info_versions
             .versions
@@ -5815,17 +5855,17 @@ impl SetDisks {
                 }
 
                 if let Some(disk) = disks[i].as_ref() {
-                    let path = path_join_buf(&[prefix, STORAGE_FORMAT_FILE]);
+                    // A failed version-only copy owns only its new version.
+                    // Removing the whole xl.meta would also erase existing
+                    // versions and any concurrently reconciled tier binding.
+                    let mut rollback = FileInfo {
+                        version_id: files[i].version_id,
+                        ..Default::default()
+                    };
+                    rollback.set_skip_tier_free_version();
                     revert_futures.push(async move {
                         if let Err(err) = disk
-                            .delete(
-                                bucket,
-                                &path,
-                                DeleteOptions {
-                                    recursive: true,
-                                    ..Default::default()
-                                },
-                            )
+                            .delete_version(bucket, prefix, rollback, false, DeleteOptions::default())
                             .await
                         {
                             warn!("write meta revert err {:?}", err);
@@ -12092,7 +12132,9 @@ mod tests {
         let bucket = "write-unique-bucket";
         let object = "object";
         let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
-        let files = vec![metadata_test_fileinfo(object), metadata_test_fileinfo(object)];
+        let mut fi = metadata_test_fileinfo(object);
+        fi.mod_time = Some(OffsetDateTime::now_utc());
+        let files = vec![fi.clone(), fi];
 
         let result = SetDisks::write_unique_file_info(&[Some(disk.clone()), None], bucket, bucket, object, &files, 2).await;
 
@@ -12106,6 +12148,53 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn write_unique_file_info_rollback_preserves_existing_reconciled_version() {
+        let bucket = "write-unique-existing";
+        let object = "object";
+        let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
+        let mut original = metadata_test_fileinfo(object);
+        original.version_id = Some(Uuid::from_u128(1));
+        original.data_dir = Some(Uuid::from_u128(3));
+        original.mod_time = Some(OffsetDateTime::now_utc());
+        original.transition_status = rustfs_filemeta::TRANSITION_COMPLETE.to_string();
+        original.transition_tier = "WARM".to_string();
+        original.transitioned_objname = "remote-original".to_string();
+        original.transition_version_state = rustfs_filemeta::TransitionVersionState::KnownDisabled;
+        rustfs_utils::http::insert_str(
+            &mut original.metadata,
+            rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+            "ab".repeat(32),
+        );
+        disk.write_metadata(bucket, bucket, object, original.clone())
+            .await
+            .expect("existing reconciled source");
+        let raw = disk
+            .read_all(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}"))
+            .await
+            .expect("original metadata");
+        let before = FileMeta::load(&raw).unwrap().find_version(original.version_id).unwrap().1;
+        let mut added = original.clone();
+        added.version_id = Some(Uuid::from_u128(2));
+        let result =
+            SetDisks::write_unique_file_info(&[Some(disk.clone()), None], bucket, bucket, object, &[added.clone(), added], 2)
+                .await;
+        assert!(result.is_err());
+        let raw = disk
+            .read_all(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}"))
+            .await
+            .expect("preserved xl.meta");
+        let after = FileMeta::load(&raw).expect("preserved metadata");
+        assert_eq!(
+            after
+                .find_version(original.version_id)
+                .expect("original source survives rollback")
+                .1,
+            before
+        );
+        assert!(after.find_version(Some(Uuid::from_u128(2))).is_err());
+    }
+
     #[tokio::test]
     async fn update_object_meta_handles_empty_metadata_and_missing_quorum() {
         let set = io_primitives_test_set(vec![None, None], 1).await;

crates/ecstore/src/set_disk/metadata.rs

@@ -326,14 +326,15 @@ impl SetDisks {
         let parity_blocks = Self::common_parity(&parities, default_parity_count as i32);
 
         if parity_blocks < 0 {
-            // No parity value reached read quorum. Distinguish two cases:
-            // enough disks answered with valid-looking metadata that simply
-            // cannot be reconciled (corrupt/foreign entries — retrying cannot
-            // help, and heal should see Corrupt, rustfs#5801) versus too few
-            // healthy answers (a genuine quorum condition where retry may
-            // succeed once disks recover).
+            // A consistent layout can require more replies than the initial
+            // half-set probe. Reaching that probe alone is not corruption;
+            // only invalid or conflicting healthy replies establish that.
             let healthy_replies = errs.iter().filter(|err| err.is_none()).count();
-            if healthy_replies >= expected_rquorum {
+            let consistent_parity = parities
+                .iter()
+                .find(|&&parity| parity >= 0)
+                .filter(|&&parity| parities.iter().filter(|&&candidate| candidate == parity).count() == healthy_replies);
+            if healthy_replies >= expected_rquorum && consistent_parity.is_none() {
                 error!(
                     "object_quorum_from_meta: irreconcilable parity across {healthy_replies} healthy replies (corrupt metadata), errs={errs:?}"
                 );
@@ -1652,6 +1653,40 @@ mod tests {
         assert_eq!(err, DiskError::FileCorrupt);
     }
 
+    #[test]
+    fn consistent_parity_below_its_data_shard_quorum_is_not_corruption() {
+        for (drive_count, parity) in [(6, 2), (8, 2), (12, 4)] {
+            let data = drive_count - parity;
+            let mut metas = (1..=drive_count)
+                .map(|index| {
+                    let mut info = FileInfo::new("bucket/object", data, parity);
+                    info.size = 1024;
+                    info.erasure.index = index;
+                    info
+                })
+                .collect::<Vec<_>>();
+            let mut errs = vec![Some(DiskError::DiskNotFound); drive_count];
+            errs[..data].fill(None);
+            assert_eq!(
+                SetDisks::object_quorum_from_meta(&metas, &errs, parity).expect("exact data quorum should resolve"),
+                (data as i32, data as i32)
+            );
+
+            errs[data - 1] = Some(DiskError::DiskNotFound);
+            assert_eq!(
+                SetDisks::object_quorum_from_meta(&metas, &errs, parity).expect_err("one fewer shard cannot resolve"),
+                DiskError::ErasureReadQuorum,
+                "layout {drive_count}/{parity} has consistent metadata but insufficient shards"
+            );
+
+            metas[0].erasure.parity_blocks = usize::MAX;
+            assert_eq!(
+                SetDisks::object_quorum_from_meta(&metas, &errs, parity).expect_err("corrupt healthy replies must be rejected"),
+                DiskError::FileCorrupt
+            );
+        }
+    }
+
     /// Too few healthy replies remains a genuine quorum condition where a
     /// retry may succeed once disks recover.
     #[test]

crates/ecstore/src/set_disk/mod.rs

@@ -865,6 +865,7 @@ pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, ren
 mod ctx;
 mod metadata;
 mod ops;
+pub(crate) use ops::bucket::BucketInfoQuorum;
 
 #[cfg(test)]
 pub(crate) use ops::hermetic_set_disks_isolated;
@@ -3888,6 +3889,33 @@ pub struct SetDisks {
     >,
 }
 
+/// Read every physical copy before selecting a version quorum. A minority
+/// legacy record is still evidence and must not disappear behind a majority
+/// not-found result. Only an explicit file/volume absence produces `None`;
+/// an unreadable disk cannot prove that no conflicting copy exists.
+pub(crate) async fn read_legacy_transition_state_metadata_copies(
+    set: &SetDisks,
+    bucket: &str,
+    object: &str,
+) -> std::result::Result<Vec<Option<Vec<u8>>>, DiskError> {
+    let disk_object = rustfs_utils::path::encode_dir_object(object);
+    let disks = set.get_disks_internal().await;
+    if disks.is_empty() {
+        return Err(DiskError::DiskNotFound);
+    }
+
+    // Include inline bytes in the generation: a conditional repair preserves
+    // the entire xl.meta, including payloads belonging to other versions.
+    let (copies, errs) = SetDisks::read_all_raw_file_info(&disks, bucket, disk_object.as_str(), true).await;
+    for err in errs.into_iter().flatten() {
+        if !matches!(err, DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound) {
+            return Err(err);
+        }
+    }
+
+    Ok(copies.into_iter().map(|copy| copy.map(|copy| copy.buf)).collect())
+}
+
 // DistributedLock sends the raw ObjectKey to its clients; LockRegistry clones
 // each endpoint's canonical Arc, so an exact Arc set identifies the lock domain.
 pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
@@ -4236,7 +4264,7 @@ impl SetDisks {
         self.get_object_metadata_cache_generations[generation.index].load(Ordering::Acquire) == generation.value
     }
 
-    async fn invalidate_get_object_metadata_cache(&self, bucket: &str, object: &str) {
+    pub(crate) async fn invalidate_get_object_metadata_cache(&self, bucket: &str, object: &str) {
         let hash = self.get_object_metadata_cache_hash(bucket, object);
         let hash_bytes = hash.to_le_bytes();
         let index = usize::from(u16::from_le_bytes([hash_bytes[0], hash_bytes[1]]) % GET_OBJECT_METADATA_CACHE_FENCE_SHARDS);

crates/ecstore/src/set_disk/ops/bucket.rs

@@ -21,12 +21,72 @@
 
 use super::super::{
     BUCKET_OP_IGNORED_ERRS, BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, DiskError, Error, HashMap,
-    MakeBucketOptions, Result, SetDisks, is_reserved_or_invalid_bucket, join_all, reduce_write_quorum_errs,
+    MakeBucketOptions, Result, SetDisks, is_reserved_or_invalid_bucket, join_all, reduce_read_quorum_errs,
+    reduce_write_quorum_errs,
 };
 use crate::api::bucket::metadata_sys;
 use crate::disk::DiskAPI;
 
+#[derive(Clone, Copy)]
+pub(crate) enum BucketInfoQuorum {
+    Read,
+    Write,
+}
+
 impl SetDisks {
+    pub(crate) async fn stat_bucket_with_quorum(&self, bucket: &str, quorum: BucketInfoQuorum) -> Result<BucketInfo> {
+        let disks = self.disk_inventory().await;
+        let disk_count = disks.len();
+        let mut futures = Vec::with_capacity(disk_count);
+        for disk in disks {
+            let bucket = bucket.to_string();
+            futures.push(async move {
+                match disk {
+                    Some(disk) => disk.stat_volume(&bucket).await,
+                    None => Err(DiskError::DiskNotFound),
+                }
+            });
+        }
+
+        let results = join_all(futures).await;
+        let mut infos = Vec::with_capacity(results.len());
+        let mut errs = Vec::with_capacity(results.len());
+        for result in results {
+            match result {
+                Ok(info) => {
+                    infos.push(Some(info));
+                    errs.push(None);
+                }
+                Err(err) => {
+                    infos.push(None);
+                    errs.push(Some(err));
+                }
+            }
+        }
+
+        let error = match quorum {
+            // Bucket mutations use a majority regardless of object storage
+            // class. A namespace read must intersect that majority; object
+            // readers still enforce the persisted layout's data-shard quorum.
+            BucketInfoQuorum::Read => reduce_read_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, disk_count.div_ceil(2).max(1)),
+            BucketInfoQuorum::Write => reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, disk_count / 2 + 1),
+        };
+        if let Some(err) = error {
+            return Err(err.into());
+        }
+
+        infos
+            .into_iter()
+            .flatten()
+            .next()
+            .map(|info| BucketInfo {
+                name: info.name,
+                created: info.created,
+                ..Default::default()
+            })
+            .ok_or(Error::VolumeNotFound)
+    }
+
     pub(crate) async fn list_bucket_for_scanner(&self, _opts: &BucketOptions) -> Result<(Vec<BucketInfo>, bool)> {
         let disks = self.disk_inventory().await;
         let write_quorum = (disks.len() / 2) + 1;
@@ -131,59 +191,12 @@ impl BucketOperations for SetDisks {
 
     #[tracing::instrument(skip(self))]
     async fn get_bucket_info(&self, bucket: &str, _opts: &BucketOptions) -> Result<BucketInfo> {
-        let disks = self.disk_inventory().await;
-        let write_quorum = (disks.len() / 2) + 1;
-
-        let mut futures = Vec::with_capacity(disks.len());
-        for disk in disks {
-            let bucket = bucket.to_string();
-            futures.push(async move {
-                match disk {
-                    Some(disk) => disk.stat_volume(&bucket).await,
-                    None => Err(DiskError::DiskNotFound),
-                }
-            });
-        }
-
-        let results = join_all(futures).await;
-        let mut infos = Vec::with_capacity(results.len());
-        let mut errs = Vec::with_capacity(results.len());
-        for result in results {
-            match result {
-                Ok(info) => {
-                    infos.push(Some(info));
-                    errs.push(None);
-                }
-                Err(err) => {
-                    infos.push(None);
-                    errs.push(Some(err));
-                }
-            }
-        }
-
-        if let Some(err) = reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, write_quorum) {
-            return Err(err.into());
-        }
-
-        let mut versioning = false;
-        let mut object_locking = false;
+        let mut info = self.stat_bucket_with_quorum(bucket, BucketInfoQuorum::Write).await?;
         if let Ok(sys) = metadata_sys::get(bucket).await {
-            versioning = sys.versioning();
-            object_locking = sys.object_locking();
+            info.versioning = sys.versioning();
+            info.object_locking = sys.object_locking();
         }
-
-        infos
-            .into_iter()
-            .flatten()
-            .next()
-            .map(|info| BucketInfo {
-                name: info.name,
-                created: info.created,
-                versioning,
-                object_locking,
-                ..Default::default()
-            })
-            .ok_or(Error::VolumeNotFound)
+        Ok(info)
     }
 
     #[tracing::instrument(skip(self))]

crates/ecstore/src/store/bucket.rs

@@ -19,7 +19,7 @@ use crate::bucket::{
 };
 use crate::error::is_err_bucket_not_found;
 use crate::runtime::sources as runtime_sources;
-use crate::set_disk::get_lock_acquire_timeout;
+use crate::set_disk::{BucketInfoQuorum, get_lock_acquire_timeout};
 use crate::storage_api_contracts::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, SRBucketDeleteOp};
 use crate::storage_api_contracts::namespace::NamespaceLocking as _;
 use futures::stream::{self, StreamExt};
@@ -770,19 +770,57 @@ impl ECStore {
         Ok(())
     }
 
+    /// Prove a live bucket generation before repairing missing expansion volumes.
+    /// Unlike request validation, repair only needs one erasure set to confirm
+    /// existence; an incomplete expansion set is precisely what repair fixes.
+    /// Callers must hold the bucket namespace lock through the subsequent heal.
+    pub(crate) async fn bucket_exists_for_heal(&self, bucket: &str) -> Result<bool> {
+        let results = futures::future::join_all(
+            self.bucket_sets()
+                .map(|(_, _, set)| async move { set.get_bucket_info(bucket, &BucketOptions::default()).await }),
+        )
+        .await;
+        let mut first_error = None;
+        for result in results {
+            match result {
+                Ok(_) => return Ok(true),
+                Err(err) if is_err_strict_volume_not_found(&err) => {}
+                Err(err) if first_error.is_none() => first_error = Some(err),
+                Err(_) => {}
+            }
+        }
+        match first_error {
+            Some(err) => Err(err),
+            None => Ok(false),
+        }
+    }
+
     #[instrument(skip(self))]
     pub(crate) async fn get_bucket_info_from_sets(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
+        self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Write)
+            .await
+    }
+
+    async fn get_bucket_info_from_sets_with_quorum(
+        &self,
+        bucket: &str,
+        opts: &BucketOptions,
+        quorum: BucketInfoQuorum,
+    ) -> Result<BucketInfo> {
         // One host may participate in several pools after expansion. Resolve the
         // namespace against each erasure set so disks from different pools can
         // never be combined into one bucket quorum.
         // Bucket validation is request-path IO. Keep the previous peer fanout's
         // latency shape by probing every set concurrently; scanner listings use
         // a separate bounded path below because they run continuously.
-        let mut scoped_results =
-            futures::future::join_all(self.bucket_sets().map(|(pool_index, set_index, set)| async move {
-                (pool_index, set_index, set.get_bucket_info(bucket, opts).await)
-            }))
-            .await;
+        let mut scoped_results = futures::future::join_all(self.bucket_sets().map(|(pool_index, set_index, set)| async move {
+            let result = match quorum {
+                BucketInfoQuorum::Read => set.stat_bucket_with_quorum(bucket, quorum).await,
+                BucketInfoQuorum::Write => set.get_bucket_info(bucket, opts).await,
+            };
+            (pool_index, set_index, result)
+        }))
+        .await;
         scoped_results.sort_unstable_by_key(|(pool_index, set_index, _)| (*pool_index, *set_index));
 
         let mut first_info = None;
@@ -806,7 +844,11 @@ impl ECStore {
 
     #[instrument(skip(self))]
     pub(super) async fn handle_get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
-        let mut info = self.get_bucket_info_from_sets(bucket, opts).await?;
+        let mut info = match self.get_bucket_info_from_sets(bucket, opts).await {
+            Ok(info) => info,
+            Err(Error::ErasureWriteQuorum) => return self.get_bucket_info_at_read_quorum(bucket, opts).await,
+            Err(err) => return Err(err),
+        };
 
         if let Ok(sys) = metadata_sys::get_in(&self.ctx, bucket).await {
             if should_override_created_from_metadata(sys.created) {
@@ -819,6 +861,35 @@ impl ECStore {
         Ok(info)
     }
 
+    async fn get_bucket_info_at_read_quorum(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
+        // Lock order: bucket lifecycle -> internal metadata object read locks.
+        // Keep create/delete from changing the namespace while a read quorum
+        // confirms both physical presence and persisted bucket metadata.
+        let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
+        await_bucket_namespace_operation(Some(&guard), bucket, "bucket read quorum validation", async {
+            let mut info = self
+                .get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Read)
+                .await?;
+            let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
+            if !persisted {
+                // A minority of directories left by failed creation is not an
+                // authoritative bucket. Never turn fabricated defaults into
+                // permission to serve degraded reads.
+                return Err(Error::ErasureReadQuorum);
+            }
+            if metadata.name != bucket {
+                return Err(Error::FileCorrupt);
+            }
+            if should_override_created_from_metadata(metadata.created) {
+                info.created = Some(metadata.created);
+            }
+            info.versioning = metadata.versioning();
+            info.object_locking = metadata.object_locking();
+            Ok(info)
+        })
+        .await
+    }
+
     #[instrument(skip(self))]
     pub(super) async fn handle_list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
         // TODO(backlog): support cached bucket listing via opts.cached
@@ -1049,7 +1120,7 @@ mod tests {
         run_physical_bucket_deletion, scan_metadata_less_residue, scan_metadata_less_residue_with_budget,
         should_override_created_from_metadata, validate_table_bucket_delete_allowed,
     };
-    use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
+    use crate::bucket::metadata::{BucketMetadata, table_bucket_catalog_metadata_prefix};
     use crate::bucket::metadata_sys;
     use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
     use crate::disk::{BUCKET_META_PREFIX, DiskAPI, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE};
@@ -1076,6 +1147,7 @@ mod tests {
     use std::sync::atomic::{AtomicBool, Ordering};
     use std::time::{Duration, SystemTime};
     use time::OffsetDateTime;
+    use tokio::io::AsyncReadExt;
     use tokio::sync::{Notify, OnceCell};
     use tokio_util::sync::CancellationToken;
     use uuid::Uuid;
@@ -1359,11 +1431,18 @@ mod tests {
     }
 
     async fn setup_multi_pool_bucket_test_env() -> (tempfile::TempDir, Arc<ECStore>) {
+        setup_bucket_quorum_test_env(&[4, 4], None).await
+    }
+
+    async fn setup_bucket_quorum_test_env(
+        drives_per_pool: &[usize],
+        standard_parity: Option<usize>,
+    ) -> (tempfile::TempDir, Arc<ECStore>) {
         let temp_dir = tempfile::tempdir().expect("multi-pool bucket test directory should be created");
         let mut pools = Vec::new();
-        for pool_index in 0..2 {
+        for (pool_index, &drive_count) in drives_per_pool.iter().enumerate() {
             let mut endpoints = Vec::new();
-            for disk_index in 0..4 {
+            for disk_index in 0..drive_count {
                 let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
                 tokio::fs::create_dir_all(&disk_path)
                     .await
@@ -1378,7 +1457,7 @@ mod tests {
             pools.push(PoolEndpoints {
                 legacy: false,
                 set_count: 1,
-                drives_per_set: 4,
+                drives_per_set: drive_count,
                 endpoints: Endpoints::from(endpoints),
                 cmd_line: format!("bucket-test-pool-{pool_index}"),
                 platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
@@ -1399,9 +1478,12 @@ mod tests {
         )
         .await
         .expect("multi-pool ECStore should initialize");
-        let storage_class =
-            crate::config::storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[4, 4])
-                .expect("multi-pool storage class should match both four-disk pools");
+        let mut storage_class_kvs = rustfs_config::server_config::KVS::new();
+        if let Some(parity) = standard_parity {
+            storage_class_kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), format!("EC:{parity}"));
+        }
+        let storage_class = crate::config::storageclass::lookup_config_for_pools_without_env(&storage_class_kvs, drives_per_pool)
+            .expect("storage class should match every test erasure set");
         for pool in &ecstore.pools {
             for set in &pool.disk_set {
                 set.set_test_storage_class_config(storage_class.clone());
@@ -2050,6 +2132,98 @@ mod tests {
             .expect("metadata initialization should recreate the bucket volume in the new pool");
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn bucket_metadata_init_repairs_half_created_expansion_pool() {
+        // Check both pool orders: an incomplete set must not hide a later
+        // complete set, and a complete set must not weaken request validation.
+        for complete_pool in 0..2 {
+            let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
+            let bucket = format!("partial-expansion-{}", Uuid::new_v4().simple());
+            for pool_index in 0..2 {
+                let present_disks = if pool_index == complete_pool { 4 } else { 2 };
+                for disk_index in 0..present_disks {
+                    tokio::fs::create_dir(
+                        temp_dir
+                            .path()
+                            .join(format!("pool{pool_index}-disk{disk_index}"))
+                            .join(&bucket),
+                    )
+                    .await
+                    .expect("fixture bucket volume should be created");
+                }
+            }
+            assert_eq!(
+                ecstore
+                    .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
+                    .await
+                    .expect_err("request validation must reject a half-created expansion set"),
+                StorageError::ErasureWriteQuorum
+            );
+
+            metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
+
+            for pool_index in 0..2 {
+                for disk_index in 0..4 {
+                    assert!(
+                        temp_dir
+                            .path()
+                            .join(format!("pool{pool_index}-disk{disk_index}"))
+                            .join(&bucket)
+                            .is_dir(),
+                        "metadata initialization must heal every missing expansion volume"
+                    );
+                }
+            }
+            ecstore
+                .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
+                .await
+                .expect("strict request validation should succeed after volume repair");
+        }
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn bucket_metadata_init_does_not_combine_partial_set_evidence() {
+        let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
+        let bucket = format!("no-quorum-expansion-{}", Uuid::new_v4().simple());
+        for pool_index in 0..2 {
+            for disk_index in 0..2 {
+                tokio::fs::create_dir(
+                    temp_dir
+                        .path()
+                        .join(format!("pool{pool_index}-disk{disk_index}"))
+                        .join(&bucket),
+                )
+                .await
+                .expect("fixture bucket volume should be created");
+            }
+        }
+        assert_eq!(
+            ecstore
+                .bucket_exists_for_heal(&bucket)
+                .await
+                .expect_err("repair must require a complete quorum within one set"),
+            StorageError::ErasureWriteQuorum
+        );
+
+        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
+
+        for pool_index in 0..2 {
+            for disk_index in 0..4 {
+                assert_eq!(
+                    temp_dir
+                        .path()
+                        .join(format!("pool{pool_index}-disk{disk_index}"))
+                        .join(&bucket)
+                        .is_dir(),
+                    disk_index < 2,
+                    "unproven bucket generations must not recreate missing volumes"
+                );
+            }
+        }
+    }
+
     #[tokio::test]
     #[serial]
     async fn bucket_metadata_init_does_not_recreate_stale_bucket_name() {
@@ -2067,6 +2241,218 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn bucket_info_read_quorum_tracks_erasure_layout() {
+        for (drive_count, parity) in [(2, 1), (3, 1), (4, 2), (5, 2), (6, 3), (8, 4), (6, 2), (12, 6)] {
+            let (_temp_dir, store) = setup_bucket_quorum_test_env(&[drive_count], Some(parity)).await;
+            metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+            let bucket = format!("read-quorum-{drive_count}-{parity}");
+            let object = "uncached-object";
+            let body = b"erasure read quorum must follow the persisted layout".repeat(32_768);
+            store
+                .make_bucket(&bucket, &MakeBucketOptions::default())
+                .await
+                .expect("healthy namespace should accept bucket creation");
+            store
+                .put_object(&bucket, object, &mut PutObjReader::from_vec(body.clone()), &ObjectOptions::default())
+                .await
+                .expect("healthy erasure set should accept the seed object");
+            let set = &store.pools[0].disk_set[0];
+            let lock = set
+                .new_ns_lock(&bucket, object)
+                .await
+                .expect("seed namespace lock should resolve");
+            drop(
+                lock.get_write_lock(Duration::from_secs(30))
+                    .await
+                    .expect("seed physical fanout must finish before taking disks offline"),
+            );
+            if (drive_count, parity) == (6, 3) {
+                let mut kvs = rustfs_config::server_config::KVS::new();
+                kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), "EC:2".to_string());
+                set.set_test_storage_class_config(
+                    crate::config::storageclass::lookup_config_for_pools_without_env(&kvs, &[drive_count])
+                        .expect("a later storage-class change must not raise old objects' read quorum"),
+                );
+            }
+
+            let offline_indexes = (0..parity).collect::<Vec<_>>();
+            let offline = take_set_disks_offline(&store, set, &offline_indexes).await;
+            let info = store
+                .get_bucket_info(&bucket, &BucketOptions::default())
+                .await
+                .expect("bucket validation must admit the object's exact read quorum");
+            assert_eq!(info.name, bucket);
+
+            let mut reader = store
+                .get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
+                .await
+                .expect("the persisted layout should remain readable at its exact data-shard quorum");
+            let mut restored = Vec::new();
+            reader
+                .stream
+                .read_to_end(&mut restored)
+                .await
+                .expect("quorum read should reconstruct the body");
+            assert_eq!(restored, body, "layout {drive_count}/{parity} must retain exact object contents");
+            drop(reader);
+
+            if drive_count - parity == drive_count / 2 {
+                let error = store
+                    .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
+                    .await
+                    .expect_err("bucket mutations must retain their majority namespace check");
+                assert_eq!(error, StorageError::ErasureWriteQuorum);
+            }
+
+            let below_quorum = take_set_disks_offline(&store, set, &[parity]).await;
+            let read = store
+                .get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
+                .await;
+            match read {
+                Ok(mut reader) => assert!(
+                    reader.stream.read_to_end(&mut Vec::new()).await.is_err(),
+                    "layout {drive_count}/{parity} must reject fewer than its data-shard quorum"
+                ),
+                Err(error) => assert!(
+                    matches!(error, StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _)),
+                    "a missing shard must report read quorum loss, got {error}"
+                ),
+            }
+            restore_set_disks(&store, set, below_quorum).await;
+            restore_set_disks(&store, set, offline).await;
+        }
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn bucket_info_read_quorum_is_scoped_to_each_erasure_set() {
+        let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4, 6], None).await;
+        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = "read-quorum-mixed-pools";
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("healthy pools should accept bucket creation");
+
+        let first_set = &store.pools[0].disk_set[0];
+        let second_set = &store.pools[1].disk_set[0];
+        let first_offline = take_set_disks_offline(&store, first_set, &[0, 1]).await;
+        let second_offline = take_set_disks_offline(&store, second_set, &[0, 1, 2]).await;
+        store
+            .get_bucket_info(bucket, &BucketOptions::default())
+            .await
+            .expect("each set independently satisfies its namespace read quorum");
+
+        for (set, extra_disk) in [(first_set, 2), (second_set, 3)] {
+            let extra_offline = take_set_disks_offline(&store, set, &[extra_disk]).await;
+            assert_eq!(
+                store
+                    .get_bucket_info(bucket, &BucketOptions::default())
+                    .await
+                    .expect_err("another pool must not subsidize a set below its read quorum"),
+                StorageError::ErasureReadQuorum
+            );
+            restore_set_disks(&store, set, extra_offline).await;
+        }
+        restore_set_disks(&store, first_set, first_offline).await;
+        restore_set_disks(&store, second_set, second_offline).await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn bucket_info_read_quorum_requires_authoritative_metadata() {
+        for state in ["missing", "corrupt", "foreign", "incarnation"] {
+            let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await;
+            metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+            let bucket = format!("read-quorum-{state}-metadata");
+            let mut metadata = if state == "missing" {
+                store
+                    .make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
+                    .await
+                    .expect("simulate directories left before bucket metadata is published");
+                BucketMetadata::new(&bucket)
+            } else {
+                store
+                    .make_bucket(&bucket, &MakeBucketOptions::default())
+                    .await
+                    .expect("healthy bucket should publish metadata");
+                metadata_sys::get_in(&store.ctx, &bucket)
+                    .await
+                    .expect("seed metadata should be cached")
+                    .as_ref()
+                    .clone()
+            };
+            let path = metadata.save_file_path();
+            match state {
+                "corrupt" => crate::config::com::save_config(store.clone(), &path, b"corrupt".to_vec())
+                    .await
+                    .expect("persist corrupt metadata while the cached copy remains valid"),
+                "foreign" => {
+                    metadata.name = "different-bucket".to_string();
+                    let mut encoded = vec![1, 0, 1, 0];
+                    encoded.extend(metadata.marshal_msg().expect("foreign metadata should encode"));
+                    crate::config::com::save_config(store.clone(), &path, encoded)
+                        .await
+                        .expect("persist metadata for a different bucket at the requested path");
+                }
+                "incarnation" => crate::bucket::metadata::save_bucket_incarnation(store.clone(), &bucket, Uuid::new_v4())
+                    .await
+                    .expect("persist a different bucket generation"),
+                _ => {}
+            }
+
+            let set = &store.pools[0].disk_set[0];
+            let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
+            let error = store
+                .get_bucket_info(&bucket, &BucketOptions::default())
+                .await
+                .expect_err("read admission must not trust residual directories or cached metadata");
+            match state {
+                "missing" => assert_eq!(error, StorageError::ErasureReadQuorum),
+                "foreign" => assert_eq!(error, StorageError::FileCorrupt),
+                "incarnation" => assert!(error.to_string().contains("sidecar does not match bucket metadata")),
+                "corrupt" => assert!(error.to_string().contains("format invalid"), "unexpected corruption error: {error}"),
+                _ => unreachable!(),
+            }
+            restore_set_disks(&store, set, offline).await;
+        }
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn bucket_info_read_quorum_accepts_persisted_legacy_metadata() {
+        let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await;
+        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = "interop";
+        store
+            .make_bucket_on_sets(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("legacy bucket directories should exist");
+        let hex = include_str!("../../tests/fixtures/minio/bucket_metadata.blob.hex")
+            .split_whitespace()
+            .collect::<String>();
+        let body = (0..hex.len())
+            .step_by(2)
+            .map(|index| u8::from_str_radix(&hex[index..index + 2], 16).expect("pinned MinIO metadata fixture"))
+            .collect();
+        crate::config::com::save_config(store.clone(), &BucketMetadata::new(bucket).save_file_path(), body)
+            .await
+            .expect("legacy metadata should be persisted without an incarnation sidecar");
+
+        let set = &store.pools[0].disk_set[0];
+        let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
+        let info = store
+            .get_bucket_info(bucket, &BucketOptions::default())
+            .await
+            .expect("persisted MinIO metadata should authorize reads at the namespace read quorum");
+        assert_eq!(info.name, bucket);
+        assert!(info.versioning);
+        assert!(info.object_locking);
+        restore_set_disks(&store, set, offline).await;
+    }
+
     #[tokio::test]
     #[serial]
     async fn bucket_namespace_reads_report_missing_when_every_set_is_absent() {
@@ -2090,6 +2476,7 @@ mod tests {
     #[serial]
     async fn bucket_namespace_reads_fail_closed_when_any_set_loses_quorum() {
         let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
+        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
         let bucket = format!("degraded-expansion-{}", Uuid::new_v4().simple());
         ecstore.pools[0].disk_set[0]
             .make_bucket(&bucket, &MakeBucketOptions::default())
@@ -2097,6 +2484,7 @@ mod tests {
             .expect("bucket should be created in the original pool only");
         ecstore.pools[1].disk_set[0].disks.write().await[0] = None;
         ecstore.pools[1].disk_set[0].disks.write().await[1] = None;
+        ecstore.pools[1].disk_set[0].disks.write().await[2] = None;
 
         let list_err = ecstore
             .list_bucket(&BucketOptions::default())
@@ -2108,7 +2496,7 @@ mod tests {
             .get_bucket_info(&bucket, &BucketOptions::default())
             .await
             .expect_err("bucket validation must fail when an expansion pool is unavailable");
-        assert_eq!(info_err, StorageError::ErasureWriteQuorum);
+        assert_eq!(info_err, StorageError::ErasureReadQuorum);
     }
 
     #[tokio::test]

crates/ecstore/src/store/init.rs

@@ -95,7 +95,6 @@ fn preflight_startup_rpc_secret_with(
     }
 }
 
-const LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY: Duration = Duration::from_secs(60 * 3);
 const LOCAL_DECOMMISSION_RESUME_RETRY_DELAY: Duration = Duration::from_secs(30);
 const LOCAL_DECOMMISSION_WATCHDOG_INTERVAL: Duration = Duration::from_secs(30);
 const LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY: Duration = Duration::from_secs(60 * 5);
@@ -280,20 +279,26 @@ where
     }
 }
 
+async fn reconcile_local_decommission_after_init(store: &Arc<ECStore>, rx: CancellationToken) -> Result<()> {
+    store
+        .ensure_pool_meta_side_effects_safe("decommission worker recovery blocked while pool metadata requires recovery")
+        .await?;
+    if store.has_active_local_decommission_worker().await {
+        return Ok(());
+    }
+    store.refresh_pool_status_meta().await?;
+    let resume_required = pool_meta_has_active_decommission(&*store.pool_meta.read().await);
+    if resume_required {
+        crate::core::pools::acquire_pool_activation_fleet_proof(&store.ctx).await?;
+    }
+    store.spawn_missing_local_decommission_routines_with_token(rx).await
+}
+
 async fn supervise_local_decommission_after_init(store: Arc<ECStore>, rx: CancellationToken) {
     run_local_decommission_watchdog(rx.clone(), || {
         let store = store.clone();
         let worker_rx = rx.clone();
-        async move {
-            store
-                .ensure_pool_meta_side_effects_safe("decommission worker recovery blocked while pool metadata requires recovery")
-                .await?;
-            if store.has_active_local_decommission_worker().await {
-                return Ok(());
-            }
-            store.refresh_pool_status_meta().await?;
-            store.spawn_missing_local_decommission_routines_with_token(worker_rx).await
-        }
+        async move { reconcile_local_decommission_after_init(&store, worker_rx).await }
     })
     .await;
 }
@@ -784,14 +789,9 @@ impl ECStore {
             );
         }
         if has_local_decommission_leadership {
-            let store = self.clone();
-            let decommission_rx = rx.clone();
-            tokio::spawn(async move {
-                if !wait_for_local_decommission_resume_delay(&decommission_rx, LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY).await {
-                    return;
-                }
-                supervise_local_decommission_after_init(store, decommission_rx).await;
-            });
+            // The watchdog checks recovery safety and retries transient failures.
+            // Resume persisted work without an unconditional cold-start delay.
+            tokio::spawn(supervise_local_decommission_after_init(self.clone(), rx.clone()));
         }
 
         let recovery_store = self.clone();
@@ -2104,6 +2104,44 @@ mod tests {
         );
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn test_local_decommission_watchdog_cancelled_start_does_not_reconcile() {
+        let rx = CancellationToken::new();
+        rx.cancel();
+        run_local_decommission_watchdog(rx, || async {
+            panic!("cancelled startup must not schedule persisted work");
+        })
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn test_local_decommission_recovery_waits_for_live_fleet_proof_before_reserving_worker() {
+        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
+        mark_test_pool_decommissioning(&store, 0).await;
+        assert!(store.ctx.is_dist_erasure().await);
+        let worker_rx = CancellationToken::new();
+
+        {
+            let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
+            let err = super::reconcile_local_decommission_after_init(&store, worker_rx.clone())
+                .await
+                .expect_err("cold distributed recovery must wait for live fleet proof");
+            assert!(
+                crate::core::pools::is_pool_activation_fleet_proof_error(&err),
+                "recovery must reach the live fleet proof gate: {err:?}"
+            );
+            assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
+            assert!(pool_meta_has_active_decommission(&*store.pool_meta.read().await));
+        }
+
+        super::reconcile_local_decommission_after_init(&store, worker_rx.clone())
+            .await
+            .expect("restored fleet proof should admit the persisted worker");
+        assert!(store.has_active_local_decommission_worker().await);
+        worker_rx.cancel();
+    }
+
     #[tokio::test(start_paused = true)]
     async fn test_local_decommission_watchdog_retries_general_failures_until_cancelled() {
         let rx = CancellationToken::new();
@@ -2115,11 +2153,13 @@ mod tests {
                 let attempts = attempts.clone();
                 let rx = rx.clone();
                 async move {
-                    if attempts.fetch_add(1, Ordering::SeqCst) == 0 {
-                        Err(StorageError::SlowDown)
-                    } else {
-                        rx.cancel();
-                        Ok(())
+                    match attempts.fetch_add(1, Ordering::SeqCst) {
+                        0 => Err(StorageError::other("pool activation requires a live fleet capability proof")),
+                        1 => Err(StorageError::SlowDown),
+                        _ => {
+                            rx.cancel();
+                            Ok(())
+                        }
                     }
                 }
             }
@@ -2128,8 +2168,11 @@ mod tests {
         tokio::task::yield_now().await;
         assert_eq!(attempts.load(Ordering::SeqCst), 1);
         tokio::time::advance(LOCAL_DECOMMISSION_RESUME_RETRY_DELAY).await;
-        task.await.expect("watchdog task should exit after cancellation");
+        tokio::task::yield_now().await;
         assert_eq!(attempts.load(Ordering::SeqCst), 2);
+        tokio::time::advance(local_decommission_watchdog_retry_delay(2)).await;
+        task.await.expect("watchdog task should exit after cancellation");
+        assert_eq!(attempts.load(Ordering::SeqCst), 3);
     }
 
     #[tokio::test(start_paused = true)]
@@ -12760,6 +12803,422 @@ mod tests {
         body
     }
 
+    #[cfg(feature = "test-util")]
+    #[test]
+    #[serial_test::serial(storage_class_env)]
+    fn legacy_transition_state_inspection_and_apply_keep_all_disk_copies_unchanged() {
+        run_large_stack_async_test("legacy-state-reconcile-inspection", || {
+            legacy_transition_state_inspection_and_apply_case(false)
+        });
+    }
+
+    #[cfg(feature = "test-util")]
+    #[cfg(not(windows))]
+    #[test]
+    #[serial_test::serial(storage_class_env)]
+    fn legacy_transition_state_backfill_retries_partial_commits_and_preserves_other_bytes() {
+        run_large_stack_async_test("legacy-state-reconcile-backfill", || {
+            legacy_transition_state_inspection_and_apply_case(true)
+        });
+    }
+
+    #[cfg(feature = "test-util")]
+    async fn legacy_transition_state_inspection_and_apply_case(write_enabled: bool) {
+        #[cfg(windows)]
+        assert!(!write_enabled, "Windows supports inspection but cannot prove repair directory durability");
+        use crate::bucket::lifecycle::legacy_transition_state_reconcile::{
+            LegacyTransitionStateReconcileOutcome as Outcome, LegacyTransitionStateReconcileRequest,
+            LegacyTransitionStateReconcileSelector,
+        };
+        for (remote_version, expected_state) in [
+            ("", rustfs_filemeta::TransitionVersionState::KnownDisabled),
+            ("null", rustfs_filemeta::TransitionVersionState::SuspendedNull),
+            ("opaque-version", rustfs_filemeta::TransitionVersionState::Exact),
+        ] {
+            let temp_dir = tempfile::tempdir().expect("legacy reconcile store directory");
+            let (ctx, store, _shutdown) =
+                without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-state-reconcile-inspect", &[4]))
+                    .await;
+            crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+            let tier_name = "LEGACY-RECONCILE";
+            let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
+            backend.set_put_remote_version(Some(remote_version.to_string())).await;
+            let bucket = "legacy-state-reconcile-bucket";
+            let object = "archive.bin";
+            store
+                .make_bucket(bucket, &MakeBucketOptions::default())
+                .await
+                .expect("create legacy fixture bucket");
+            let mut reader = PutObjReader::from_vec(b"legacy reconcile body".repeat(1024));
+            let source = store
+                .put_object(bucket, object, &mut reader, &ObjectOptions::default())
+                .await
+                .expect("write source");
+            {
+                // Create the fixture under the existing remote-version writer
+                // gate. This does not authorize legacy metadata reconciliation.
+                let _proof = crate::services::notification_sys::install_current_remote_version_state_fleet_proof_for_test();
+                temp_env::async_with_vars(
+                    [
+                        (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, Some("true")),
+                        (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, Some("true")),
+                    ],
+                    store.transition_object(
+                        bucket,
+                        object,
+                        &ObjectOptions {
+                            transition: TransitionOptions {
+                                status: TRANSITION_PENDING.to_string(),
+                                tier: tier_name.to_string(),
+                                etag: source.etag.clone().expect("source ETag"),
+                                ..Default::default()
+                            },
+                            mod_time: source.mod_time,
+                            ..Default::default()
+                        },
+                    ),
+                )
+                .await
+                .expect("transition source");
+            }
+            assert!(
+                crate::services::notification_sys::acquire_legacy_transition_state_reconcile_fleet_proof()
+                    .await
+                    .is_none(),
+                "fixture setup must not grant the missing reconciliation write capability"
+            );
+            let selector = LegacyTransitionStateReconcileSelector {
+                bucket: bucket.to_string(),
+                object: object.to_string(),
+                version_id: "null".to_string(),
+            };
+            if expected_state == rustfs_filemeta::TransitionVersionState::Exact {
+                backend
+                    .set_transition_candidate_probe_override(Some(
+                        crate::services::tier::warm_backend::TransitionCandidateProbe::Ambiguous,
+                    ))
+                    .await;
+            }
+            let converged = store
+                .inspect_legacy_transition_state(selector.clone())
+                .await
+                .expect("inspect an already explicit transition");
+            assert_eq!(converged.outcome, Outcome::Migrated, "{converged:?}");
+            assert!(!converged.changed);
+            backend.set_transition_candidate_probe_override(None).await;
+            rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, remote_version.is_empty()).await;
+            let paths = (0..4)
+                .map(|disk| {
+                    temp_dir
+                        .path()
+                        .join(format!("pool0/set0/disk{disk}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"))
+                })
+                .collect::<Vec<_>>();
+            let mut original = Vec::new();
+            for path in &paths {
+                original.push(tokio::fs::read(path).await.expect("original xl.meta"));
+            }
+            backend.clear_op_log().await;
+            let inspection = store.inspect_legacy_transition_state(selector.clone());
+            assert!(
+                std::mem::size_of_val(&inspection) <= 4 * 1024,
+                "admin inspection future must remain stack-bounded"
+            );
+            let inspected = inspection.await.expect("inspect legacy state");
+            assert_eq!(inspected.outcome, Outcome::ReadyToMigrate, "{inspected:?}");
+            assert!(!inspected.readiness.post_ready, "current fleet cannot authorize conditional writes");
+            let target = inspected.target.expect("live probe should establish one model");
+            assert_eq!(target.state, expected_state);
+            let request = LegacyTransitionStateReconcileRequest {
+                confirm: true,
+                selector,
+                source: inspected.source.expect("immutable source"),
+                original_sets: inspected.original_sets,
+                target,
+                reconciliation_digest: inspected.reconciliation_digest.expect("expected tuple digest"),
+            };
+            #[cfg(not(windows))]
+            if write_enabled {
+                crate::services::notification_sys::with_legacy_transition_state_fleet_proof_for_test(async {
+                    crate::disk::local::bucket_durability::set(bucket, Some(crate::disk::local::DurabilityMode::None));
+                    let unsynced = store.reconcile_legacy_transition_state(request.clone()).await;
+                    crate::disk::local::bucket_durability::set(bucket, None);
+                    let unsynced = unsynced.expect("repair without metadata durability");
+                    assert_eq!(unsynced.outcome, Outcome::BackendUnavailable);
+                    assert!(!unsynced.changed);
+                    let rollback = paths[0].parent().expect("object directory").join(Uuid::new_v4().to_string());
+                    tokio::fs::create_dir(&rollback).await.expect("pending rollback directory");
+                    tokio::fs::write(rollback.join(crate::disk::STORAGE_FORMAT_FILE_BACKUP), &original[0])
+                        .await
+                        .expect("pending old metadata backup");
+                    let unsettled = store.reconcile_legacy_transition_state(request.clone()).await;
+                    tokio::fs::remove_dir_all(&rollback).await.expect("settle fixture rollback");
+                    let unsettled = unsettled.expect("repair must wait for rollback");
+                    assert_eq!(unsettled.outcome, Outcome::BackendUnavailable);
+                    assert!(!unsettled.changed);
+                    for (path, bytes) in paths.iter().zip(&original) {
+                        assert_eq!(tokio::fs::read(path).await.expect("blocked repair leaves original bytes"), *bytes);
+                    }
+                    // The first disk commits; the second stops after staging.
+                    // This models an interrupted cross-disk effect without rollback.
+                    let disks = store.all_set_disks()[0].disk_inventory().await;
+                    let first_disk = disks[0].as_ref().expect("first physical disk");
+                    let publication_path = first_disk
+                        .get_object_path_for_io_if_local(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}"))
+                        .expect("local disk")
+                        .expect("publication path");
+                    let crash_key = format!("{object}/{STORAGE_FORMAT_FILE}");
+                    let _hook = crate::disk::os::prepared_publication_test_hooks::install_at(
+                        crate::disk::os::prepared_publication_test_hooks::Stage::Rename,
+                        &publication_path,
+                        move || {
+                            crate::crash_inject::arm(crate::crash_inject::CrashPoint::MetaWriteAfterTmpBeforeRename, &crash_key);
+                        },
+                    );
+                    let partial = store
+                        .reconcile_legacy_transition_state(request.clone())
+                        .await
+                        .expect("partial repair response");
+                    assert_eq!(partial.outcome, Outcome::BackendUnavailable, "{partial:?}");
+                    assert!(partial.changed, "first copy was committed: {partial:?}");
+                    assert!(partial.changes_indeterminate);
+                    assert_ne!(tokio::fs::read(&paths[0]).await.expect("first committed copy"), original[0]);
+                    for (path, bytes) in paths[1..].iter().zip(&original[1..]) {
+                        assert_eq!(tokio::fs::read(path).await.expect("uncommitted copy"), *bytes);
+                    }
+                    assert!(
+                        store.all_set_disks()[0]
+                            .load_file_info_versions_exact(bucket, object)
+                            .await
+                            .is_err(),
+                        "cleanup cannot select a partial repair subset"
+                    );
+                    let repaired = store
+                        .reconcile_legacy_transition_state(request.clone())
+                        .await
+                        .expect("retry original snapshot");
+                    assert_eq!(repaired.outcome, Outcome::Migrated, "{repaired:?}");
+                    assert!(repaired.changed);
+                    assert!(!repaired.changes_indeterminate);
+                    let mut committed = Vec::new();
+                    for (path, original) in paths.iter().zip(&original) {
+                        let raw = tokio::fs::read(path).await.expect("repaired copy");
+                        let metadata = FileMeta::load(&raw).expect("decode repaired copy");
+                        let previous = FileMeta::load(original).expect("decode original copy");
+                        assert_eq!(
+                            metadata.transition_reconcile_generation(None).unwrap(),
+                            previous.transition_reconcile_generation(None).unwrap()
+                        );
+                        let (_, version) = metadata.find_version(None).expect("selected version");
+                        let info = version.into_fileinfo(bucket, object, true).expect("repaired FileInfo");
+                        assert_eq!(info.transition_version_state, expected_state);
+                        assert_eq!(info.transition_version, request.target.remote_version);
+                        committed.push(raw);
+                    }
+                    assert!(
+                        store.all_set_disks()[0]
+                            .load_file_info_versions_exact(bucket, object)
+                            .await
+                            .expect("converged cleanup snapshot")
+                            .is_some()
+                    );
+                    let replay = store
+                        .reconcile_legacy_transition_state(request.clone())
+                        .await
+                        .expect("idempotent original request replay");
+                    assert_eq!(replay.outcome, Outcome::Migrated, "{replay:?}");
+                    assert!(!replay.changed);
+                    for (path, expected) in paths.iter().zip(&committed) {
+                        assert_eq!(
+                            tokio::fs::read(path).await.expect("replayed copy"),
+                            *expected,
+                            "idempotence preserves raw encoding"
+                        );
+                    }
+                    for (path, bytes) in paths.iter().zip(&original) {
+                        tokio::fs::write(path, bytes)
+                            .await
+                            .expect("reset independent cancellation fixture");
+                    }
+                    let (entered_tx, entered) = tokio::sync::oneshot::channel();
+                    let (release, released) = std::sync::mpsc::channel::<()>();
+                    let _pause = crate::disk::os::prepared_publication_test_hooks::install_at(
+                        crate::disk::os::prepared_publication_test_hooks::Stage::Rename,
+                        &publication_path,
+                        move || {
+                            let _ = entered_tx.send(());
+                            let _ = released.recv();
+                        },
+                    );
+                    let mut repair = Box::pin(store.reconcile_legacy_transition_state(request.clone()));
+                    tokio::select! {
+                        result = &mut repair => panic!("repair completed before publication pause: {result:?}"),
+                        result = entered => result.expect("publication executor entered"),
+                    }
+                    let update_options = crate::disk::UpdateMetadataOpts::default();
+                    let mut update = Box::pin(first_disk.update_metadata(
+                        bucket,
+                        object,
+                        FileInfo {
+                            metadata: HashMap::from([("x-amz-meta-concurrent".to_string(), "kept".to_string())]),
+                            ..Default::default()
+                        },
+                        &update_options,
+                    ));
+                    assert!(
+                        tokio::time::timeout(std::time::Duration::from_millis(25), update.as_mut())
+                            .await
+                            .is_err(),
+                        "another metadata RMW must wait for publication"
+                    );
+                    drop(repair);
+                    assert!(
+                        tokio::time::timeout(std::time::Duration::from_millis(25), update.as_mut())
+                            .await
+                            .is_err(),
+                        "cancelling the coordinator must not release an in-flight disk mutation"
+                    );
+                    release.send(()).expect("resume owned publication");
+                    update.await.expect("serialized metadata update");
+                    let raw = tokio::fs::read(&paths[0])
+                        .await
+                        .expect("cancelled repair and later metadata update");
+                    let (_, version) = FileMeta::load(&raw)
+                        .expect("metadata after cancellation")
+                        .find_version(None)
+                        .expect("selected version");
+                    let info = version
+                        .into_fileinfo(bucket, object, true)
+                        .expect("metadata after serialized update");
+                    assert_eq!(info.transition_version_state, expected_state);
+                    assert_eq!(info.metadata.get("x-amz-meta-concurrent").map(String::as_str), Some("kept"));
+                    let stale = store
+                        .reconcile_legacy_transition_state(request.clone())
+                        .await
+                        .expect("stale original request");
+                    assert_eq!(
+                        stale.outcome,
+                        Outcome::Corrupt,
+                        "unrelated metadata change invalidates the original generation: {stale:?}"
+                    );
+                    assert!(!stale.changed);
+                    assert_eq!(tokio::fs::read(&paths[0]).await.expect("stale write leaves bytes unchanged"), raw);
+                    assert_eq!(backend.remove_count().await, 0);
+                    // A pinned version probe uses GET to verify that exact
+                    // candidate; every backend operation still targets it.
+                    let operations = backend.op_log().await;
+                    assert!(
+                        operations.iter().all(|operation| match operation {
+                            MockWarmOp::Probe { object } | MockWarmOp::Get { object } => object == &request.source.remote_object,
+                            _ => false,
+                        }),
+                        "unexpected backend effects: {operations:?}"
+                    );
+                })
+                .await;
+                continue;
+            }
+            let mut tampered = request.clone();
+            tampered.source.remote_object.push_str("-other");
+            let probes_before = backend.op_log().await.len();
+            let rejected = store
+                .reconcile_legacy_transition_state(tampered)
+                .await
+                .expect("reject tampered tuple");
+            assert_eq!(rejected.outcome, Outcome::Corrupt);
+            assert_eq!(backend.op_log().await.len(), probes_before, "invalid digest must not probe the backend");
+            let applied = store
+                .reconcile_legacy_transition_state(request)
+                .await
+                .expect("apply must report unavailable write authority");
+            assert_eq!(applied.outcome, Outcome::BackendUnavailable, "{applied:?}");
+            assert_eq!(applied.reason_code, "write_fence_unavailable");
+            assert!(!applied.changed);
+            for (path, expected) in paths.iter().zip(&original) {
+                assert_eq!(tokio::fs::read(path).await.expect("xl.meta after inspection"), *expected);
+            }
+            assert_eq!(backend.remove_count().await, 0);
+            assert!(
+                backend
+                    .op_log()
+                    .await
+                    .iter()
+                    .all(|operation| matches!(operation, MockWarmOp::Probe { .. }))
+            );
+
+            backend.set_unreachable(true).await;
+            let unavailable = store
+                .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
+                    bucket: bucket.to_string(),
+                    object: object.to_string(),
+                    version_id: "null".to_string(),
+                })
+                .await
+                .expect("unreachable tier is a diagnostic outcome");
+            assert_eq!(unavailable.outcome, Outcome::BackendUnavailable);
+            assert!(!unavailable.changed);
+            backend.set_unreachable(false).await;
+            for candidate in ["", "00000000-0000-0000-0000-000000000000", "bad\nversion"] {
+                backend
+                    .set_transition_candidate_probe_override(Some(
+                        crate::services::tier::warm_backend::TransitionCandidateProbe::VersionedPresent(candidate.to_string()),
+                    ))
+                    .await;
+                let invalid_proof = store
+                    .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
+                        bucket: bucket.to_string(),
+                        object: object.to_string(),
+                        version_id: "null".to_string(),
+                    })
+                    .await
+                    .expect("invalid backend proof is a diagnostic outcome");
+                assert_eq!(invalid_proof.outcome, Outcome::BackendUnavailable, "{invalid_proof:?}");
+                assert!(invalid_proof.target.is_none());
+            }
+            backend.set_transition_candidate_probe_override(None).await;
+
+            backend.clear_op_log().await;
+            for path in &paths[1..] {
+                tokio::fs::remove_file(path)
+                    .await
+                    .expect("hide majority metadata copies in fixture");
+            }
+            let minority = store
+                .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
+                    bucket: bucket.to_string(),
+                    object: object.to_string(),
+                    version_id: "null".to_string(),
+                })
+                .await
+                .expect("inspect minority legacy record");
+            assert_eq!(
+                minority.outcome,
+                Outcome::BackendUnavailable,
+                "a minority owner must remain visible: {minority:?}"
+            );
+            assert!(
+                backend.op_log().await.is_empty(),
+                "unproven metadata quorum cannot initiate a remote probe"
+            );
+            for (path, bytes) in paths.iter().zip(&original) {
+                tokio::fs::write(path, bytes).await.expect("restore fixture copies");
+            }
+            tokio::fs::write(&paths[0], b"corrupt-xl-meta")
+                .await
+                .expect("inject corrupt copy");
+            let corrupt = store
+                .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
+                    bucket: bucket.to_string(),
+                    object: object.to_string(),
+                    version_id: "null".to_string(),
+                })
+                .await
+                .expect("inspect corrupt legacy record");
+            assert_eq!(corrupt.outcome, Outcome::Corrupt, "{corrupt:?}");
+            assert!(backend.op_log().await.is_empty(), "corruption must fail before backend I/O");
+        }
+    }
+
     #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]

crates/ecstore/src/store/list_objects.rs

@@ -164,6 +164,16 @@ pub fn max_keys_plus_one(max_keys: i32, add_one: bool) -> i32 {
     max_keys
 }
 
+fn list_versions_scan_limit(max_keys: i32, has_version_marker: bool) -> i32 {
+    if max_keys <= 0 {
+        return 0;
+    }
+
+    // The marker object's versions may all be filtered out after gathering.
+    // Reserve its raw entry in addition to the next-page lookahead entry.
+    max_keys_plus_one(max_keys, true) + i32::from(has_version_marker)
+}
+
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
 enum GatherResultsState {
     LimitReached,
@@ -2139,15 +2149,19 @@ fn build_list_versions_next_marker(
         // here; advertise it as the literal `null` marker so a resumed listing
         // parses it back to `VersionMarker::Null` instead of a nil UUID that
         // `find_version_index` can never match (issue #6745).
-        (
-            Some(append_list_cache_id_to_marker(last.name.clone(), cache_id)),
+        let version_marker = if last.is_dir && last.mod_time.is_none() {
+            // A CommonPrefix has no version to resume; a version marker would
+            // make the next page include this same prefix again.
+            None
+        } else {
             Some(
                 last.version_id
                     .filter(|v| !v.is_nil())
                     .map(|v| v.to_string())
                     .unwrap_or_else(|| "null".to_string()),
-            ),
-        )
+            )
+        };
+        (Some(append_list_cache_id_to_marker(last.name.clone(), cache_id)), version_marker)
     } else if let Some(last_prefix) = prefixes.last() {
         (Some(append_list_cache_id_to_marker(last_prefix.clone(), cache_id)), None)
     } else {
@@ -2866,6 +2880,20 @@ fn listing_entries_supplement_target(
         return None;
     }
 
+    if let Some(directory) = entries.0.iter().flatten().find(|entry| entry.is_dir()) {
+        let directory_copies = entries
+            .0
+            .iter()
+            .flatten()
+            .filter(|entry| entry.is_dir() && entry.name == directory.name)
+            .count();
+        // A committed child may have some of its directory copies only on
+        // fallback disks, just like object metadata in a partial primary sample.
+        if directory_copies < resolver.dir_quorum {
+            return Some(directory.name.clone());
+        }
+    }
+
     for (idx, entry) in entries.0.iter().enumerate() {
         let Some(entry) = entry.as_ref().filter(|entry| entry.is_object()) else {
             continue;
@@ -4018,8 +4046,7 @@ impl ECStore {
             None
         };
 
-        let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) };
-        // Always request max_keys + 1 to detect if there are more results
+        let effective_max_keys = list_versions_scan_limit(max_keys, has_version_marker);
         let mut opts = ListPathOptions {
             bucket: bucket.to_owned(),
             prefix: prefix.to_owned(),
@@ -5325,7 +5352,7 @@ impl Sets {
             None
         };
 
-        let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) };
+        let effective_max_keys = list_versions_scan_limit(max_keys, has_version_marker);
         let mut opts = ListPathOptions {
             bucket: bucket.to_owned(),
             prefix: prefix.to_owned(),
@@ -6034,7 +6061,7 @@ impl SetDisks {
 
         let has_version_marker = version_marker.is_some();
         let version_marker = version_marker.map(parse_version_marker).transpose()?;
-        let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) };
+        let effective_max_keys = list_versions_scan_limit(max_keys, has_version_marker);
         let mut opts = ListPathOptions {
             bucket: bucket.to_owned(),
             prefix: prefix.to_owned(),
@@ -6248,7 +6275,7 @@ impl SetDisks {
             None
         };
 
-        let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) };
+        let effective_max_keys = list_versions_scan_limit(max_keys, has_version_marker);
         let mut opts = ListPathOptions {
             bucket: bucket.to_owned(),
             prefix: prefix.to_owned(),
@@ -7441,6 +7468,153 @@ mod test {
         assert!(cancel.is_cancelled());
     }
 
+    #[test]
+    fn list_versions_pagination_scan_limit_boundaries() {
+        for has_version_marker in [false, true] {
+            assert_eq!(super::list_versions_scan_limit(-1, has_version_marker), 0);
+            assert_eq!(super::list_versions_scan_limit(0, has_version_marker), 0);
+            let marker_slot = i32::from(has_version_marker);
+            assert_eq!(super::list_versions_scan_limit(1, has_version_marker), 2 + marker_slot);
+            assert_eq!(super::list_versions_scan_limit(MAX_OBJECT_LIST, has_version_marker), 1001 + marker_slot);
+            assert_eq!(super::list_versions_scan_limit(i32::MAX, has_version_marker), 1001 + marker_slot);
+        }
+    }
+
+    #[tokio::test]
+    async fn list_versions_pagination_does_not_require_an_empty_final_page() {
+        use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
+        use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
+
+        let (dirs, store) = isolated_store_over_temp_disks().await;
+        let bucket = "version-pagination-bucket";
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("pagination bucket should be created");
+        let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+
+        for kind in ["objects", "deletes", "null", "mixed", "delimiter"] {
+            let count = if kind == "mixed" { 5 } else { 10 };
+            let mut expected = Vec::new();
+            for index in 0..count {
+                let name = if kind == "delimiter" && index % 2 == 1 {
+                    format!("{kind}/testobject-{index:02}/child")
+                } else {
+                    format!("{kind}/testobject-{index:02}")
+                };
+                let entry = match kind {
+                    "deletes" => test_delete_marker_meta_entry(&name, mod_time),
+                    "null" => test_object_meta_entry(&name),
+                    "mixed" => test_object_with_delete_marker_meta_entry(&name, mod_time, mod_time + time::Duration::SECOND),
+                    _ => test_object_meta_entry_with_erasure_versions(&name, &[(mod_time, "etag", 2, 2)]),
+                };
+                for dir in &dirs {
+                    let object_dir = dir.path().join(bucket).join(&name);
+                    tokio::fs::create_dir_all(&object_dir)
+                        .await
+                        .expect("pagination object directory should be created");
+                    tokio::fs::write(object_dir.join(STORAGE_FORMAT_FILE), &entry.metadata)
+                        .await
+                        .expect("pagination metadata should be written");
+                }
+                if kind == "delimiter" && index % 2 == 1 {
+                    expected.push((name.trim_end_matches("child").to_owned(), None, false));
+                } else {
+                    let versions = entry.file_info_versions(bucket).expect("fixture versions should decode");
+                    expected.extend(
+                        versions
+                            .versions
+                            .iter()
+                            .map(|version| (name.clone(), version.version_id, version.deleted)),
+                    );
+                }
+            }
+            let prefix = format!("{kind}/");
+            let delimiter = (kind == "delimiter").then(|| "/".to_owned());
+            // Exercise each public/internal entry point with the reported page size.
+            // The store entry point also covers exact and one-over limit boundaries.
+            for (layer, max_keys) in [(0, 0), (0, 1), (0, 5), (0, 9), (0, 10), (0, 11), (1, 5), (2, 5), (3, 5)] {
+                if layer == 3 && delimiter.is_some() {
+                    continue;
+                }
+                let mut marker = None;
+                let mut version_marker = None;
+                let expected_pages = if max_keys == 0 {
+                    1
+                } else {
+                    10usize.div_ceil(usize::try_from(max_keys).expect("positive page size"))
+                };
+                let mut actual = Vec::new();
+                for page in 0..expected_pages {
+                    let result = match layer {
+                        0 => {
+                            store
+                                .clone()
+                                .inner_list_object_versions(bucket, &prefix, marker, version_marker, delimiter.clone(), max_keys)
+                                .await
+                        }
+                        1 => {
+                            store.pools[0]
+                                .clone()
+                                .inner_list_object_versions(bucket, &prefix, marker, version_marker, delimiter.clone(), max_keys)
+                                .await
+                        }
+                        2 => {
+                            store.pools[0].disk_set[0]
+                                .clone()
+                                .inner_list_object_versions(bucket, &prefix, marker, version_marker, delimiter.clone(), max_keys)
+                                .await
+                        }
+                        _ => {
+                            store.pools[0].disk_set[0]
+                                .clone()
+                                .inner_list_object_versions_for_recursive_delete(
+                                    bucket,
+                                    &prefix,
+                                    marker,
+                                    version_marker,
+                                    max_keys,
+                                )
+                                .await
+                        }
+                    }
+                    .expect("version page should list successfully");
+                    let page_size = usize::try_from(max_keys).expect("nonnegative page size");
+                    assert_eq!(result.objects.len() + result.prefixes.len(), (10 - page * page_size).min(page_size));
+                    let has_more = page + 1 < expected_pages;
+                    assert_eq!(result.is_truncated, has_more, "{kind}, layer {layer}, max_keys {max_keys}, page {page}");
+                    assert_eq!(
+                        result.next_marker.is_some(),
+                        has_more,
+                        "key marker must exist only when another page exists"
+                    );
+                    if !has_more {
+                        assert!(
+                            result.next_version_idmarker.is_none(),
+                            "the final page must not advertise a version marker"
+                        );
+                    }
+                    actual.extend(
+                        result
+                            .objects
+                            .into_iter()
+                            .map(|object| (object.name, object.version_id, object.delete_marker)),
+                    );
+                    actual.extend(result.prefixes.into_iter().map(|prefix| (prefix, None, false)));
+                    marker = result.next_marker;
+                    version_marker = result.next_version_idmarker;
+                }
+                // Objects and CommonPrefixes are serialized separately; compare their
+                // identities without relying on their relative position in the response.
+                actual.sort();
+                let mut expected = if max_keys == 0 { Vec::new() } else { expected.clone() };
+                expected.sort();
+                assert_eq!(actual, expected, "{kind}, layer {layer}, max_keys {max_keys}");
+            }
+        }
+    }
+
     #[test]
     fn version_marker_is_applied_only_when_key_marker_entry_is_present() {
         let version_marker = Some(VersionMarker::Null);
@@ -9448,6 +9622,71 @@ mod test {
         assert!(supplemented.is_latest_delete_marker());
     }
 
+    #[tokio::test]
+    async fn latest_listing_supplement_checks_fallback_disks_for_common_prefix_quorum() {
+        let mut fallback_disks = Vec::new();
+        let mut fallback_tempdirs = Vec::new();
+        for index in 0..4 {
+            let tempdir = tempfile::tempdir().expect("fallback tempdir should be created");
+            let endpoint = Endpoint::try_from(tempdir.path().to_str().expect("fallback path should be utf8"))
+                .expect("fallback endpoint should parse");
+            let disk = new_disk(
+                &endpoint,
+                &DiskOption {
+                    cleanup: false,
+                    health_check: false,
+                },
+            )
+            .await
+            .expect("fallback disk should be created");
+            disk.make_volume("bucket").await.expect("fallback bucket should be created");
+            for copies in [3, 4] {
+                if index < copies {
+                    let object = format!("quux-{copies}/thud");
+                    let entry = test_object_meta_entry(&object);
+                    disk.write_all("bucket", &format!("{object}/{STORAGE_FORMAT_FILE}"), bytes::Bytes::from(entry.metadata))
+                        .await
+                        .expect("fallback child metadata should be written");
+                }
+            }
+            fallback_disks.push(disk);
+            fallback_tempdirs.push(tempdir);
+        }
+        let supplement = ListingSupplement::new(
+            ListingSupplementOptions {
+                bucket: "bucket".to_owned(),
+                path: String::new(),
+                recursive: false,
+                incl_deleted: false,
+                skip_hidden_prefix_check: false,
+                filter_prefix: None,
+                forward_to: None,
+                per_disk_limit: 100,
+                skip_total_timeout: true,
+                walkdir_timeout: None,
+                walkdir_stall_timeout: None,
+            },
+            Arc::new(fallback_disks),
+            FallbackClaimTracker::default(),
+        );
+        // A 16-drive EC:4 set asks 12 primary disks. A committed write may
+        // exist on eight primary disks and all four remaining fallback disks.
+        let resolver = list_metadata_resolution_params("bucket".to_owned(), 4, 12, false, 0);
+        for fallback_copies in [3, 4] {
+            let prefix = format!("quux-{fallback_copies}/");
+            let mut primary = vec![Some(test_dir_meta_entry(&prefix)); 8];
+            primary.extend([None, None, None, None]);
+            let entry =
+                resolve_listing_entries_with_supplement(MetaCacheEntries(primary), resolver.clone(), true, supplement.clone())
+                    .await;
+            assert_eq!(
+                entry.map(|entry| entry.name),
+                (fallback_copies == 4).then_some(prefix),
+                "the common prefix needs all twelve copies, including fallback disks"
+            );
+        }
+    }
+
     #[test]
     fn latest_listing_supplement_keeps_a_subquorum_delete_marker_hidden() {
         let object_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");

crates/ecstore/src/store/object.rs

@@ -3079,12 +3079,7 @@ impl ECStore {
         let store = Arc::clone(self);
         let write = async move {
             let object = "buckets/.scanner-pause-backlog.json";
-            let mut opts = ObjectOptions {
-                max_parity: true,
-                http_preconditions: Some(preconditions),
-                write_completion: crate::object_api::WriteCompletion::TailDrained,
-                ..Default::default()
-            };
+            let mut opts = ObjectOptions::default();
             // Match migration: fixed object namespace -> durable pool metadata ->
             // actual replica namespace. The replica need not be the hash-routed set.
             let object_guard = if store.single_pool() {
@@ -3110,9 +3105,14 @@ impl ECStore {
             } else {
                 None
             };
-            let result = set
-                .put_object(RUSTFS_META_BUCKET, object, &mut PutObjReader::from_vec(data), &opts)
-                .await;
+            let result = crate::data_movement::scanner_backlog::persist_native_scanner_pause_backlog_replica(
+                set,
+                data,
+                preconditions,
+                opts,
+                "publish",
+            )
+            .await;
             drop(capacity_guard);
             drop(object_guard);
             result
@@ -3747,29 +3747,37 @@ impl ECStore {
         opts: &ObjectOptions,
         no_lock: bool,
     ) -> Result<usize> {
+        let capacity_owner = DecommissionCapacityOwner::from_options(opts);
         match self
             .get_pool_info_existing_with_opts(bucket, object, &data_movement_pool_lookup_opts(opts, no_lock))
             .await
         {
-            Ok((pinfo, _)) => Ok(pinfo.index),
+            Ok((pinfo, _)) => {
+                if let Some(owner) = capacity_owner {
+                    if self.is_decommission_capacity_target_reserved(owner, pinfo.index).await? {
+                        return Ok(pinfo.index);
+                    }
+                } else {
+                    return Ok(pinfo.index);
+                }
+            }
             Err(err) => {
                 if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
                     return Err(err);
                 }
-
-                if let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
-                    let expected_data_bytes = opts
-                        .capacity_expected_data_bytes()
-                        .or_else(|| usize::try_from(size).ok())
-                        .unwrap_or_default();
-                    return self
-                        .select_decommission_capacity_target_pool(owner, expected_data_bytes)
-                        .await;
-                }
-
-                self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull)
             }
         }
+        if let Some(owner) = capacity_owner {
+            let expected_data_bytes = opts
+                .capacity_expected_data_bytes()
+                .or_else(|| usize::try_from(size).ok())
+                .unwrap_or_default();
+            return self
+                .select_decommission_capacity_target_pool(owner, expected_data_bytes)
+                .await;
+        }
+
+        self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull)
     }
 
     async fn find_data_movement_target_info(
@@ -4251,8 +4259,10 @@ impl ECStore {
     }
 
     /// Return metadata for DELETE preflight, including an explicitly addressed
-    /// delete marker. Read APIs must keep using `get_object_info`; authorization
-    /// and Object Lock enforcement still belong to the caller and locked delete.
+    /// delete marker. GET/HEAD may also use this metadata-only lookup to enrich
+    /// an already failed read with marker headers, never to serve marker data.
+    /// Normal reads must keep using `get_object_info`; authorization and Object
+    /// Lock enforcement still belong to the caller and locked delete.
     #[instrument(level = "trace", skip_all)]
     pub async fn get_object_info_for_delete(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
         self.get_object_info_snapshot(bucket, object, opts, true).await

crates/filemeta/src/filemeta.rs

@@ -50,6 +50,9 @@ use tracing::{error, warn};
 use uuid::Uuid;
 use xxhash_rust::xxh64;
 
+mod transition_reconcile;
+pub use transition_reconcile::TransitionStateReconcileTarget;
+
 // XL header specifies the format
 pub static XL_FILE_HEADER: [u8; 4] = *b"XL2 ";
 // pub static XL_FILE_VERSION_CURRENT: [u8; 4] = [0; 4];
@@ -391,6 +394,16 @@ impl FileMeta {
 
                     if ver_vid == fi_vid {
                         let mut ver = FileMetaVersion::try_from(version.meta.as_slice())?;
+                        let previous = ver
+                            .object
+                            .as_ref()
+                            .is_some_and(|object| {
+                                rustfs_utils::http::contains_key_bytes(
+                                    &object.meta_sys,
+                                    rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+                                )
+                            })
+                            .then(|| ver.clone());
 
                         if let Some(ref mut obj) = ver.object {
                             if replace_user_metadata {
@@ -447,6 +460,9 @@ impl FileMeta {
                             }
                         }
 
+                        if let Some(previous) = previous {
+                            transition_reconcile::preserve_reconciled_transition(&previous, &mut ver)?;
+                        }
                         // Update
                         version.header = ver.header();
                         version.meta = ver.marshal_msg()?;
@@ -492,7 +508,7 @@ impl FileMeta {
         Ok(())
     }
 
-    pub fn add_version_filemata(&mut self, version: FileMetaVersion) -> Result<()> {
+    pub fn add_version_filemata(&mut self, mut version: FileMetaVersion) -> Result<()> {
         if !version.valid() {
             return Err(Error::other("file meta version invalid"));
         }
@@ -512,6 +528,7 @@ impl FileMeta {
             if existing.free_version() != version.free_version() {
                 return Err(Error::other("cannot replace a free version with a non-free version"));
             }
+            transition_reconcile::preserve_reconciled_transition(&existing, &mut version)?;
             return self.set_idx(fidx, version);
         }
 

crates/filemeta/src/filemeta/transition_reconcile.rs

@@ -0,0 +1,384 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+use super::{FileMeta, FileMetaVersion};
+use crate::{Error, Result, TRANSITION_COMPLETE, TransitionVersionState};
+use rustfs_utils::http::metadata_compat::{
+    SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, contains_key_bytes,
+    get_consistent_bytes, insert_bytes, remove_bytes,
+};
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+const RECONCILE_SUFFIXES: [&str; 3] = [
+    SUFFIX_TRANSITIONED_VERSION_STATE,
+    SUFFIX_TRANSITIONED_VERSION_ID,
+    SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+];
+
+/// The only fields a legacy transition repair is allowed to persist.
+#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct TransitionStateReconcileTarget {
+    pub state: TransitionVersionState,
+    pub remote_version: Option<String>,
+    pub destination_id: String,
+}
+
+impl TransitionStateReconcileTarget {
+    pub fn validate(&self) -> Result<()> {
+        let valid_version = match self.state {
+            TransitionVersionState::KnownDisabled => self.remote_version.is_none(),
+            TransitionVersionState::SuspendedNull => self.remote_version.as_deref() == Some("null"),
+            TransitionVersionState::Exact => self.remote_version.as_deref().is_some_and(|value| {
+                !value.is_empty()
+                    && value.len() <= 1024
+                    && value != "null"
+                    && !value.chars().any(char::is_control)
+                    && !Uuid::parse_str(value).is_ok_and(|id| id.is_nil())
+            }),
+            TransitionVersionState::Unknown => false,
+        };
+        if !valid_version
+            || self.destination_id.len() != 64
+            || !self
+                .destination_id
+                .bytes()
+                .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
+        {
+            return Err(Error::FileCorrupt);
+        }
+        Ok(())
+    }
+}
+
+impl FileMeta {
+    /// Canonical identity of every version and inline byte, excluding only the
+    /// three repairable suffixes on the selected version. It survives a repair
+    /// and encoding-order changes, while detecting unrelated metadata changes.
+    pub fn transition_reconcile_generation(&self, version_id: Option<Uuid>) -> Result<Vec<u8>> {
+        if self
+            .versions
+            .iter()
+            .filter(|version| version.header.version_id.unwrap_or_default() == version_id.unwrap_or_default())
+            .count()
+            != 1
+        {
+            return Err(Error::FileCorrupt);
+        }
+        let (selected, _) = self.find_version(version_id)?;
+        let mut versions = Vec::with_capacity(self.versions.len());
+        for index in 0..self.versions.len() {
+            let mut version = self.get_idx(index)?;
+            if index == selected {
+                let object = version.object.as_mut().ok_or(Error::FileCorrupt)?;
+                for suffix in RECONCILE_SUFFIXES {
+                    remove_bytes(&mut object.meta_sys, suffix);
+                }
+            }
+            versions.push(version);
+        }
+        versions.sort_by_key(|version| version.get_version_id().unwrap_or_default());
+        let mut value = serde_json::to_value((&versions, &self.data)).map_err(|_| Error::FileCorrupt)?;
+        value.sort_all_objects();
+        serde_json::to_vec(&value).map_err(|_| Error::FileCorrupt)
+    }
+
+    /// Returns false for an already converged record. Callers must serialize
+    /// the read/check/commit and compare the observed metadata generation.
+    pub fn reconcile_transition_state(
+        &mut self,
+        version_id: Option<Uuid>,
+        target: &TransitionStateReconcileTarget,
+    ) -> Result<bool> {
+        target.validate()?;
+        let (index, mut version) = self.find_version(version_id)?;
+        let info = version.into_fileinfo("", "", true)?;
+        info.validate_for_metadata_read()?;
+        if info.transition_status != TRANSITION_COMPLETE
+            || info.transition_tier.is_empty()
+            || info.transitioned_objname.is_empty()
+        {
+            return Err(Error::FileCorrupt);
+        }
+        let object = version.object.as_mut().ok_or(Error::FileCorrupt)?;
+        let destination = get_consistent_bytes(&object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID);
+        if contains_key_bytes(&object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
+            && destination != Some(target.destination_id.as_bytes())
+        {
+            return Err(Error::FileCorrupt);
+        }
+        if contains_key_bytes(&object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE) {
+            if info.transition_version_state == target.state
+                && info.transition_version == target.remote_version
+                && destination == Some(target.destination_id.as_bytes())
+            {
+                return Ok(false);
+            }
+            return Err(Error::FileCorrupt);
+        }
+        if info.transition_version_state != TransitionVersionState::Unknown
+            || info
+                .transition_version
+                .as_deref()
+                .filter(|value| !value.is_empty())
+                .is_some_and(|value| Some(value) != target.remote_version.as_deref())
+        {
+            return Err(Error::FileCorrupt);
+        }
+        insert_bytes(
+            &mut object.meta_sys,
+            SUFFIX_TRANSITIONED_VERSION_STATE,
+            target.state.as_str().as_bytes().to_vec(),
+        );
+        remove_bytes(&mut object.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID);
+        if let Some(version) = &target.remote_version {
+            insert_bytes(&mut object.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, version.as_bytes().to_vec());
+        }
+        insert_bytes(
+            &mut object.meta_sys,
+            SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+            target.destination_id.as_bytes().to_vec(),
+        );
+        version.into_fileinfo("", "", true)?.validate_for_metadata_read()?;
+        self.set_idx(index, version)?;
+        Ok(true)
+    }
+}
+
+/// A stale healer or metadata writer may carry the original absent fields.
+/// Preserve a proven binding for the same immutable transition, or reject an
+/// attempted change of meaning. A new payload/version or a delete is separate.
+pub(super) fn preserve_reconciled_transition(previous: &FileMetaVersion, next: &mut FileMetaVersion) -> Result<()> {
+    let (Some(previous_object), Some(next_object)) = (&previous.object, &mut next.object) else {
+        return Ok(());
+    };
+    if !contains_key_bytes(&previous_object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
+        || !contains_key_bytes(&previous_object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE)
+        || previous_object.data_dir != next_object.data_dir
+    {
+        return Ok(());
+    }
+    let previous_info = previous.into_fileinfo("", "", true)?;
+    if previous_info.transition_version_state == TransitionVersionState::Unknown
+        || previous_info.transition_status != TRANSITION_COMPLETE
+    {
+        return Ok(());
+    }
+    let next_info = next.into_fileinfo("", "", true)?;
+    if previous_info.transition_tier != next_info.transition_tier
+        || previous_info.transitioned_objname != next_info.transitioned_objname
+        || previous_info.transition_status != next_info.transition_status
+        || previous_info.size != next_info.size
+        || previous_info.metadata.get("etag") != next_info.metadata.get("etag")
+    {
+        return Err(Error::FileCorrupt);
+    }
+    let destination =
+        get_consistent_bytes(&previous_object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID).ok_or(Error::FileCorrupt)?;
+    previous_info.validate_for_metadata_read()?;
+    TransitionStateReconcileTarget {
+        state: previous_info.transition_version_state,
+        remote_version: previous_info.transition_version.clone(),
+        destination_id: std::str::from_utf8(destination).map_err(|_| Error::FileCorrupt)?.to_string(),
+    }
+    .validate()?;
+    let next_object = next.object.as_mut().ok_or(Error::FileCorrupt)?;
+    if next_info
+        .transition_version
+        .as_ref()
+        .is_some_and(|version| Some(version) != previous_info.transition_version.as_ref())
+    {
+        return Err(Error::FileCorrupt);
+    }
+    if contains_key_bytes(&next_object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE) {
+        if previous_info.transition_version_state != next_info.transition_version_state
+            || previous_info.transition_version != next_info.transition_version
+        {
+            return Err(Error::FileCorrupt);
+        }
+    } else if next_info.transition_version_state != TransitionVersionState::Unknown {
+        return Err(Error::FileCorrupt);
+    }
+    if contains_key_bytes(&next_object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
+        && get_consistent_bytes(&next_object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID) != Some(destination)
+    {
+        return Err(Error::FileCorrupt);
+    }
+    for suffix in RECONCILE_SUFFIXES {
+        remove_bytes(&mut next_object.meta_sys, suffix);
+        if let Some(value) = get_consistent_bytes(&previous_object.meta_sys, suffix) {
+            insert_bytes(&mut next_object.meta_sys, suffix, value.to_vec());
+        }
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{ErasureInfo, FileInfo, ObjectPartInfo};
+
+    fn legacy() -> (FileMeta, FileInfo) {
+        let info = FileInfo {
+            version_id: Some(Uuid::from_u128(1)),
+            data_dir: Some(Uuid::from_u128(2)),
+            mod_time: Some(time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixture time")),
+            size: 7,
+            parts: vec![ObjectPartInfo {
+                number: 1,
+                size: 7,
+                actual_size: 7,
+                ..Default::default()
+            }],
+            erasure: ErasureInfo {
+                algorithm: "ReedSolomon".to_string(),
+                data_blocks: 2,
+                parity_blocks: 2,
+                block_size: 1024 * 1024,
+                index: 1,
+                distribution: vec![1, 2, 3, 4],
+                ..Default::default()
+            },
+            transition_status: TRANSITION_COMPLETE.to_string(),
+            transition_tier: "WARM".to_string(),
+            transitioned_objname: "remote-object".to_string(),
+            metadata: std::collections::HashMap::from([("etag".to_string(), "source-etag".to_string())]),
+            data: Some(bytes::Bytes::from_static(b"payload")),
+            ..Default::default()
+        };
+        let mut metadata = FileMeta::new();
+        metadata.add_version(info.clone()).expect("legacy fixture");
+        (metadata, info)
+    }
+
+    fn target(state: TransitionVersionState) -> TransitionStateReconcileTarget {
+        TransitionStateReconcileTarget {
+            state,
+            remote_version: match state {
+                TransitionVersionState::Exact => Some("opaque-version".to_string()),
+                TransitionVersionState::SuspendedNull => Some("null".to_string()),
+                _ => None,
+            },
+            destination_id: "ab".repeat(32),
+        }
+    }
+
+    #[test]
+    fn transition_reconcile_preserves_payload_and_generation_and_is_idempotent() {
+        for state in [
+            TransitionVersionState::KnownDisabled,
+            TransitionVersionState::SuspendedNull,
+            TransitionVersionState::Exact,
+        ] {
+            let (mut metadata, info) = legacy();
+            let mut other = info.clone();
+            other.version_id = Some(Uuid::from_u128(3));
+            other.data_dir = Some(Uuid::from_u128(4));
+            other.transition_status.clear();
+            other.transition_tier.clear();
+            other.transitioned_objname.clear();
+            other.data = Some(bytes::Bytes::from_static(b"other!!"));
+            metadata.add_version(other.clone()).expect("unrelated inline version");
+            let other_before = metadata.find_version(other.version_id).expect("unrelated version").1;
+            let original_data = metadata.data.clone();
+            let generation = metadata
+                .transition_reconcile_generation(info.version_id)
+                .expect("initial generation");
+            let target = target(state);
+            assert!(metadata.reconcile_transition_state(info.version_id, &target).expect("repair"));
+            let bytes = metadata.marshal_msg().expect("encode repair");
+            let mut reloaded = FileMeta::load(&bytes).expect("reload repair");
+            assert_eq!(reloaded.data, original_data);
+            assert_eq!(
+                reloaded
+                    .find_version(other.version_id)
+                    .expect("preserved unrelated version")
+                    .1,
+                other_before
+            );
+            assert_eq!(
+                reloaded
+                    .transition_reconcile_generation(info.version_id)
+                    .expect("repaired generation"),
+                generation
+            );
+            assert!(!reloaded.reconcile_transition_state(info.version_id, &target).expect("retry"));
+            let (_, version) = reloaded.find_version(info.version_id).expect("selected version");
+            let repaired = version.into_fileinfo("", "", true).expect("decode explicit state");
+            assert_eq!(repaired.transition_version_state, state);
+            assert_eq!(repaired.transition_version, target.remote_version);
+            assert_eq!(repaired.parts, info.parts);
+            for prefix in [
+                rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
+                rustfs_utils::http::MINIO_INTERNAL_PREFIX,
+            ] {
+                assert_eq!(
+                    version
+                        .object
+                        .as_ref()
+                        .expect("object")
+                        .meta_sys
+                        .get(&format!("{prefix}{SUFFIX_TRANSITIONED_VERSION_STATE}")),
+                    Some(&state.as_str().as_bytes().to_vec())
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn transition_reconcile_generation_detects_unrelated_metadata_and_inline_changes() {
+        let (mut metadata, info) = legacy();
+        let original = metadata.transition_reconcile_generation(info.version_id).expect("generation");
+        let mut updated = info.clone();
+        updated.metadata.insert("user-tag".to_string(), "changed".to_string());
+        metadata.update_object_version(updated).expect("update unrelated field");
+        assert_ne!(metadata.transition_reconcile_generation(info.version_id).expect("generation"), original);
+        let (mut metadata, mut info) = legacy();
+        info.data = Some(bytes::Bytes::from_static(b"changed"));
+        metadata.add_version(info.clone()).expect("change inline bytes");
+        assert_ne!(metadata.transition_reconcile_generation(info.version_id).expect("generation"), original);
+    }
+
+    #[test]
+    fn transition_reconcile_binding_survives_stale_heal_and_metadata_writes() {
+        let (mut metadata, mut stale) = legacy();
+        let target = target(TransitionVersionState::Exact);
+        metadata
+            .reconcile_transition_state(stale.version_id, &target)
+            .expect("repair");
+        metadata
+            .add_version(stale.clone())
+            .expect("stale heal must preserve the binding");
+        stale.metadata.insert("user-tag".to_string(), "updated".to_string());
+        metadata
+            .update_object_version(stale.clone())
+            .expect("ordinary metadata update");
+        assert!(
+            !metadata
+                .reconcile_transition_state(stale.version_id, &target)
+                .expect("binding remains exact")
+        );
+    }
+
+    #[test]
+    fn transition_reconcile_rejects_explicit_unknown_and_conflicting_binding() {
+        let (mut metadata, mut info) = legacy();
+        rustfs_utils::http::insert_str(&mut info.metadata, SUFFIX_TRANSITIONED_VERSION_STATE, "unknown".to_string());
+        metadata.add_version(info.clone()).expect("explicit unknown fixture");
+        assert!(
+            metadata
+                .reconcile_transition_state(info.version_id, &target(TransitionVersionState::Exact))
+                .is_err()
+        );
+        let (mut metadata, mut stale) = legacy();
+        metadata
+            .reconcile_transition_state(stale.version_id, &target(TransitionVersionState::Exact))
+            .expect("repair");
+        stale.transition_version_state = TransitionVersionState::KnownDisabled;
+        assert!(
+            metadata.add_version(stale).is_err(),
+            "an explicit state cannot be replaced with a different model"
+        );
+    }
+}

crates/heal/Cargo.toml

@@ -31,6 +31,7 @@ workspace = true
 
 [features]
 default = []
+test-util = []
 hotpath = [
     "hotpath/hotpath",
     "hotpath/tokio",
@@ -104,6 +105,7 @@ walkdir = { workspace = true }
 http = { workspace = true }
 temp-env = { workspace = true, features = ["async_closure"] }
 tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
+chrono = { workspace = true }
 
 [lib]
 doctest = false

crates/heal/src/heal/channel.rs

@@ -847,7 +847,7 @@ mod tests {
 
     fn create_test_heal_manager() -> Arc<HealManager> {
         let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-        Arc::new(HealManager::new(storage, None))
+        Arc::new(HealManager::new_without_root_recovery_for_test(storage, None))
     }
 
     #[test]
@@ -1481,7 +1481,7 @@ mod tests {
     #[tokio::test]
     async fn test_process_start_request_returns_admission_result() {
         let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-        let manager = Arc::new(HealManager::new(
+        let manager = Arc::new(HealManager::new_without_root_recovery_for_test(
             storage,
             Some(HealConfig {
                 queue_size: 1,
@@ -1826,7 +1826,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_process_query_request_reports_displaced_terminal_detail() {
-        let heal_manager = Arc::new(HealManager::new(
+        let heal_manager = Arc::new(HealManager::new_without_root_recovery_for_test(
             Arc::new(MockStorage),
             Some(HealConfig {
                 queue_size: 1,

crates/heal/src/heal/manager.rs

@@ -536,14 +536,14 @@ fn completed_status_is_retrying(status: &HealTaskStatus) -> bool {
     matches!(status, HealTaskStatus::Retrying { .. })
 }
 
-fn retry_budget_for_result(task: &HealTask, result: &Result<()>) -> Option<(Duration, String)> {
+fn retry_budget_for_result(task: &HealTask, result: &Result<()>, retryable_batch_failure: bool) -> Option<(Duration, String)> {
     let Err(err) = result else {
         return None;
     };
     if task.retry_attempts >= MAX_RECOVERABLE_HEAL_RETRIES {
         return None;
     }
-    if task.has_batch_failure() {
+    if task.has_batch_failure() && !retryable_batch_failure {
         return None;
     }
 
@@ -559,12 +559,13 @@ fn retry_budget_for_result(task: &HealTask, result: &Result<()>) -> Option<(Dura
 
 #[cfg(test)]
 fn retry_request_for_result(task: &HealTask, result: &Result<()>) -> Option<(HealRequest, Duration, String)> {
-    let (delay, error) = retry_budget_for_result(task, result)?;
+    let (delay, error) = retry_budget_for_result(task, result, false)?;
     Some((task.retry_request(), delay, error))
 }
 
 async fn retry_request_for_result_with_budget(task: &HealTask, result: &Result<()>) -> Option<(HealRequest, Duration, String)> {
-    let (delay, error) = retry_budget_for_result(task, result)?;
+    let retryable_batch_failure = task.batch_failure_is_retryable().await;
+    let (delay, error) = retry_budget_for_result(task, result, retryable_batch_failure)?;
     let request = match task.retry_request_with_remaining_timeout().await {
         Ok(request) => request,
         Err(err) => {
@@ -841,6 +842,10 @@ pub struct HealManager {
     replacement_recovery_anchors: Arc<std::sync::Mutex<HashMap<String, String>>>,
     /// Set IDs whose durable replacement metadata is corrupt or conflicting.
     replacement_recovery_blocked_sets: Arc<std::sync::Mutex<HashSet<String>>>,
+    /// Durable handoff of interrupted administrator root traversals.
+    root_recovery: Arc<root_recovery::RootHealRecovery>,
+    /// Keep forceStart's cancellation side effects inside the shutdown fence.
+    force_start_shutdown: Mutex<()>,
     /// Storage layer interface
     storage: Arc<dyn HealStorageAPI>,
     /// Cancel token
@@ -876,6 +881,7 @@ struct HealQueueContext<'a> {
     retrying_heals: &'a Arc<Mutex<HashMap<String, RetryingHeal>>>,
     mrf_repair_notice_targets: &'a Arc<StdMutex<HashMap<String, Vec<MrfRepairNoticeTarget>>>>,
     replacement_recovery_anchors: &'a Arc<std::sync::Mutex<HashMap<String, String>>>,
+    root_recovery: &'a Arc<root_recovery::RootHealRecovery>,
     config: &'a Arc<RwLock<HealConfig>>,
     statistics: &'a Arc<RwLock<HealStatistics>>,
     storage: &'a Arc<dyn HealStorageAPI>,
@@ -939,6 +945,10 @@ impl HealManager {
         matches!(request.source, HealRequestSource::Admin | HealRequestSource::Internal)
     }
 
+    fn queued_request_can_be_displaced(request: &HealRequest) -> bool {
+        !root_recovery::is_admin_heal_recovery(&request.heal_type, request.source)
+    }
+
     fn request_bypasses_mainline_throttle(request: &HealRequest) -> bool {
         request.force_start
             || matches!(request.source, HealRequestSource::Admin | HealRequestSource::Internal)
@@ -1075,11 +1085,15 @@ impl HealManager {
         let per_object_request = request.heal_type.is_per_object();
 
         if queue_len >= queue_capacity && !request.force_start {
-            if Self::can_displace_queued_work(&request) && queue.can_displace_lower_priority(request.priority) {
+            if Self::can_displace_queued_work(&request)
+                && queue.can_displace_lower_priority_where(request.priority, Self::queued_request_can_be_displaced)
+            {
                 let request_id = request.id.clone();
                 let priority = request.priority;
                 let source = request.source;
-                if let Some(displaced) = queue.push_displacing_lower_priority(request) {
+                if let Some(displaced) =
+                    queue.push_displacing_lower_priority_where(request, Self::queued_request_can_be_displaced)
+                {
                     publish_heal_queue_length(queue);
                     Self::record_admission_metric(source, HealAdmissionResult::Accepted, context);
                     demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
@@ -1353,6 +1367,51 @@ impl HealManager {
         });
     }
 
+    async fn start_root_recovery_terminal_gc(&self) {
+        let cancel = self.cancel_token.clone();
+        let root_recovery = self.root_recovery.clone();
+        tokio::spawn(async move {
+            let mut ticker = interval(RESUME_GC_INTERVAL);
+            loop {
+                tokio::select! {
+                    _ = cancel.cancelled() => break,
+                    _ = ticker.tick() => {
+                        match root_recovery.gc_terminal_receipts_once(SystemTime::now()).await {
+                            Ok(report) => {
+                                if report.pending_removed > 0 || report.terminals_removed > 0 || report.budget_exhausted {
+                                    debug!(
+                                        target: "rustfs::heal::manager",
+                                        event = EVENT_HEAL_RESUME_GC,
+                                        component = LOG_COMPONENT_HEAL,
+                                        subsystem = LOG_SUBSYSTEM_MANAGER,
+                                        state = "root_terminal_gc",
+                                        scanned = report.scanned,
+                                        retained = report.retained,
+                                        pending_removed = report.pending_removed,
+                                        terminals_removed = report.terminals_removed,
+                                        budget_exhausted = report.budget_exhausted,
+                                        "Root heal terminal receipt GC inspected durable state"
+                                    );
+                                }
+                            }
+                            Err(error) => {
+                                warn!(
+                                    target: "rustfs::heal::manager",
+                                    event = EVENT_HEAL_RESUME_GC,
+                                    component = LOG_COMPONENT_HEAL,
+                                    subsystem = LOG_SUBSYSTEM_MANAGER,
+                                    state = "root_terminal_gc_failed",
+                                    error = %error,
+                                    "Root heal terminal receipt GC failed"
+                                );
+                            }
+                        }
+                    }
+                }
+            }
+        });
+    }
+
     /// Create new HealManager
     pub fn new(storage: Arc<dyn HealStorageAPI>, config: Option<HealConfig>) -> Self {
         Self::new_with_workload_provider(storage, config, None)
@@ -1363,6 +1422,15 @@ impl HealManager {
         storage: Arc<dyn HealStorageAPI>,
         config: Option<HealConfig>,
         workload_provider: Option<WorkloadSnapshotProviderRef>,
+    ) -> Self {
+        Self::new_with_root_recovery(storage, config, workload_provider, Arc::new(root_recovery::RootHealRecovery::default()))
+    }
+
+    fn new_with_root_recovery(
+        storage: Arc<dyn HealStorageAPI>,
+        config: Option<HealConfig>,
+        workload_provider: Option<WorkloadSnapshotProviderRef>,
+        root_recovery: Arc<root_recovery::RootHealRecovery>,
     ) -> Self {
         let config = config.unwrap_or_default();
         Self {
@@ -1377,6 +1445,8 @@ impl HealManager {
             mrf_repair_notice_targets: Arc::new(StdMutex::new(HashMap::new())),
             replacement_recovery_anchors: Arc::new(std::sync::Mutex::new(HashMap::new())),
             replacement_recovery_blocked_sets: Arc::new(std::sync::Mutex::new(HashSet::new())),
+            root_recovery,
+            force_start_shutdown: Mutex::new(()),
             storage,
             cancel_token: CancellationToken::new(),
             statistics: Arc::new(RwLock::new(HealStatistics::new())),
@@ -1386,6 +1456,27 @@ impl HealManager {
         }
     }
 
+    #[cfg(any(test, feature = "test-util"))]
+    #[doc(hidden)]
+    pub fn new_without_root_recovery_for_test(storage: Arc<dyn HealStorageAPI>, config: Option<HealConfig>) -> Self {
+        Self::new_with_root_recovery(storage, config, None, Arc::new(root_recovery::RootHealRecovery::disabled_for_tests()))
+    }
+
+    #[cfg(any(test, feature = "test-util"))]
+    #[doc(hidden)]
+    pub fn new_with_workload_provider_without_root_recovery_for_test(
+        storage: Arc<dyn HealStorageAPI>,
+        config: Option<HealConfig>,
+        workload_provider: Option<WorkloadSnapshotProviderRef>,
+    ) -> Self {
+        Self::new_with_root_recovery(
+            storage,
+            config,
+            workload_provider,
+            Arc::new(root_recovery::RootHealRecovery::disabled_for_tests()),
+        )
+    }
+
     /// Start HealManager
     pub async fn start(&self) -> Result<()> {
         let mut state = self.state.write().await;
@@ -1412,6 +1503,23 @@ impl HealManager {
             "Heal manager starting"
         );
 
+        // Restore graceful-shutdown root responsibilities before automatic
+        // repair can admit overlapping work.
+        if let Err(error) = self.replay_root_heals().await {
+            // A missing owner or invalid root record must not block existing
+            // replacement recovery. Keep its file for a later restart after
+            // the owner is readable or the record has been repaired.
+            warn!(
+                target: "rustfs::heal::manager",
+                event = EVENT_HEAL_MANAGER_STATE,
+                component = LOG_COMPONENT_HEAL,
+                subsystem = LOG_SUBSYSTEM_MANAGER,
+                state = "root_recovery_deferred",
+                error = %error,
+                "Root heal restart recovery deferred"
+            );
+        }
+
         // start scheduler
         self.start_scheduler().await?;
 
@@ -1421,6 +1529,7 @@ impl HealManager {
 
         // Inspect resume artifacts in a bounded, fail-closed background task.
         self.start_resume_gc().await;
+        self.start_root_recovery_terminal_gc().await;
 
         // start auto disk scanner to heal unformatted disks
         if self.config.read().await.enable_auto_heal {
@@ -1449,6 +1558,7 @@ impl HealManager {
 
     /// Stop HealManager
     pub async fn stop(&self) -> Result<()> {
+        let _force_start_guard = self.force_start_shutdown.lock().await;
         info!(
             target: "rustfs::heal::manager",
             event = EVENT_HEAL_MANAGER_STATE,
@@ -1458,11 +1568,39 @@ impl HealManager {
             "Heal manager stopping"
         );
 
-        // cancel all tasks
-        self.cancel_token.cancel();
-
-        // wait for all tasks to complete
+        // Keep scheduler, cancellation, and retry ownership stable until every
+        // unfinished admin control-plane heal has a durable successor. A failed
+        // write must leave the manager running and the shutdown marker unclean.
         let mut active_heals = self.active_heals.lock().await;
+        let queue = self.heal_queue.lock().await;
+        let retrying = self.retrying_heals.lock().await;
+        for task in active_heals.values() {
+            if root_recovery::is_admin_heal_recovery(&task.heal_type, task.source) {
+                if task.get_status().await == HealTaskStatus::Completed {
+                    self.root_recovery.remove(&task.id, &task.heal_type, task.source).await?;
+                } else {
+                    let mut request = match task.retry_request_with_remaining_timeout().await {
+                        Ok(request) => request,
+                        Err(Error::TaskTimeout) => {
+                            let mut request = task.retry_request();
+                            request.options.timeout = Some(Duration::ZERO);
+                            request
+                        }
+                        Err(error) => return Err(error),
+                    };
+                    request.retry_attempts = task.retry_attempts;
+                    self.root_recovery.persist(&request).await?;
+                }
+            }
+        }
+        for request in queue.requests().chain(retrying.values().map(|retrying| &retrying.request)) {
+            self.root_recovery.persist(request).await?;
+        }
+        self.cancel_token.cancel();
+        drop(retrying);
+        drop(queue);
+
+        // cancel active workers after the durable handoff
         for task in active_heals.values() {
             if let Err(e) = task.cancel().await {
                 warn!(
@@ -1589,6 +1727,17 @@ impl HealManager {
         let admission_start = Instant::now();
         let source = request.source;
         let force_start = request.force_start;
+        // A forceStart must not retire an old durable owner if shutdown will
+        // reject its replacement. Hold the same gate through final admission.
+        let _force_start_guard = if source == HealRequestSource::Admin && force_start {
+            let guard = self.force_start_shutdown.lock().await;
+            if self.cancel_token.is_cancelled() {
+                return Err(Error::Other("Heal manager is stopping".to_string()));
+            }
+            Some(guard)
+        } else {
+            None
+        };
         // HS-06 forceStart semantics (admin only): MinIO stops the old task
         // first and then starts the new one. Cancel any active admin task
         // overlapping this request's path before entering admission, so the
@@ -1596,7 +1745,9 @@ impl HealManager {
         if request.source == HealRequestSource::Admin && request.force_start {
             let overlapping: Vec<String> = {
                 let active_heals = self.active_heals.lock().await;
-                active_heals
+                let queue = self.heal_queue.lock().await;
+                let retrying = self.retrying_heals.lock().await;
+                let mut ids = active_heals
                     .iter()
                     .filter(|(task_id, task)| {
                         task.source == HealRequestSource::Admin
@@ -1604,7 +1755,19 @@ impl HealManager {
                             && *task_id != &request.id
                     })
                     .map(|(task_id, _)| task_id.clone())
-                    .collect()
+                    .collect::<Vec<_>>();
+                ids.extend(
+                    queue
+                        .requests()
+                        .chain(retrying.values().map(|retrying| &retrying.request))
+                        .filter(|pending| {
+                            root_recovery::is_admin_heal_recovery(&pending.heal_type, pending.source)
+                                && heal_types_overlap(&request.heal_type, &pending.heal_type) != OverlapVerdict::Disjoint
+                                && pending.id != request.id
+                        })
+                        .map(|pending| pending.id.clone()),
+                );
+                ids
             };
             for task_id in overlapping {
                 match self.cancel_task(&task_id).await {
@@ -1618,17 +1781,16 @@ impl HealManager {
                         result = "force_start_cancelled_overlap",
                         "Admin forceStart cancelled an overlapping heal task"
                     ),
-                    Err(err) => warn!(
-                        target: "rustfs::heal::manager",
-                        event = EVENT_HEAL_QUEUE_ADMISSION,
-                        component = LOG_COMPONENT_HEAL,
-                        subsystem = LOG_SUBSYSTEM_MANAGER,
-                        request_id = %request.id,
-                        cancelled_task_id = %task_id,
-                        error = %err,
-                        result = "force_start_cancel_failed",
-                        "Admin forceStart failed to cancel an overlapping heal task"
-                    ),
+                    Err(err) => return Err(err),
+                }
+            }
+            // A failed or timed-out replay may have only its durable owner
+            // left. Cancel only records that overlap this forced start.
+            for pending in self.root_recovery.pending().await? {
+                if pending.id != request.id
+                    && heal_types_overlap(&request.heal_type, &pending.heal_type) != OverlapVerdict::Disjoint
+                {
+                    self.cancel_task(&pending.id).await?;
                 }
             }
         }
@@ -1641,6 +1803,9 @@ impl HealManager {
         // active -> retrying transitions can slip between duplicate checks.
         let lock_phase_start = Instant::now();
         let active_heals = self.active_heals.lock().await;
+        if self.cancel_token.is_cancelled() {
+            return Err(Error::Other("Heal manager is stopping".to_string()));
+        }
         #[cfg(test)]
         pause_duplicate_admission_after_active_lock(&request.id).await;
         let mut queue = self.heal_queue.lock().await;
@@ -1854,9 +2019,29 @@ impl HealManager {
             }
         }
 
+        let durable_handoff = root_recovery::is_admin_heal_recovery(&request.heal_type, request.source);
+        let durable_handoff_required = durable_handoff
+            && (request.force_start
+                || queue.len() < config.queue_size
+                || (Self::can_displace_queued_work(&request)
+                    && queue.can_displace_lower_priority_where(request.priority, Self::queued_request_can_be_displaced)));
+        // An admin receipt is a control-plane responsibility. Persist it before
+        // queue publication so a crash after admission can replay it.
+        if durable_handoff_required {
+            self.root_recovery.persist(&request).await?;
+        }
+
         let mut task_id = request.id.clone();
+        let request_id = request.id.clone();
+        let request_heal_type = request.heal_type.clone();
+        let request_source = request.source;
         let admission_decision = Self::admit_request_to_queue(&mut queue, request, &config, "submit");
         let admission = admission_decision.result;
+        if durable_handoff_required && !admission.is_admitted() {
+            self.root_recovery
+                .remove(&request_id, &request_heal_type, request_source)
+                .await?;
+        }
         if admission == HealAdmissionResult::Merged
             && let Some(queued_id) = queue.queued_request_id_for_dedup_key(&dedup_key)
         {
@@ -1927,7 +2112,7 @@ impl HealManager {
     /// Retrying, never Pending — then the queue, and finally a terminal
     /// completed entry. `heal_path` additionally constrains the map matches
     /// the way the `*_for_path` variants always have.
-    async fn lookup_task_state(&self, canonical_task_id: &str, heal_path: Option<&str>) -> TaskStateLookup {
+    async fn lookup_task_state(&self, canonical_task_id: &str, heal_path: Option<&str>) -> Result<TaskStateLookup> {
         let matches_path = |heal_type: &HealType| heal_path.is_none_or(|path| heal_type_matches_path(heal_type, path));
 
         {
@@ -1936,7 +2121,7 @@ impl HealManager {
                 .get(canonical_task_id)
                 .filter(|task| matches_path(&task.heal_type))
             {
-                return TaskStateLookup::Active(Arc::clone(task));
+                return Ok(TaskStateLookup::Active(Arc::clone(task)));
             }
         }
 
@@ -1946,7 +2131,7 @@ impl HealManager {
                 .get(canonical_task_id)
                 .filter(|retrying| matches_path(&retrying.request.heal_type))
             {
-                return TaskStateLookup::Retrying(retrying.status());
+                return Ok(TaskStateLookup::Retrying(retrying.status()));
             }
         }
 
@@ -1959,7 +2144,7 @@ impl HealManager {
             prune_completed_heal_statuses(&mut completed_heals);
             if let Some(completed) = completed_heals.get(canonical_task_id).filter(|c| matches_path(&c.heal_type)) {
                 if completed_status_is_retrying(&completed.status) {
-                    return TaskStateLookup::Completed(Arc::clone(completed));
+                    return Ok(TaskStateLookup::Completed(Arc::clone(completed)));
                 }
                 terminal_completed = Some(Arc::clone(completed));
             }
@@ -1972,7 +2157,7 @@ impl HealManager {
                 None => queue.contains_request_id(canonical_task_id),
             };
             if queued {
-                return TaskStateLookup::Queued;
+                return Ok(TaskStateLookup::Queued);
             }
         }
 
@@ -1985,15 +2170,55 @@ impl HealManager {
                 .cloned();
         }
 
-        match terminal_completed {
+        if terminal_completed.is_none()
+            && let Some(completed) = self.root_recovery.completed(canonical_task_id).await?
+            && matches_path(&completed.heal_type)
+        {
+            terminal_completed = Some(Arc::new(completed));
+        }
+
+        Ok(match terminal_completed {
             Some(completed) => TaskStateLookup::Completed(completed),
             None => TaskStateLookup::NotFound,
-        }
+        })
+    }
+
+    async fn publish_admin_terminal(
+        &self,
+        task_id: &str,
+        heal_type: &HealType,
+        source: HealRequestSource,
+        completed: &CompletedHealStatus,
+    ) -> Result<bool> {
+        self.root_recovery
+            .persist_terminal(task_id, heal_type, source, completed)
+            .await
+    }
+
+    async fn publish_admin_cancelled_terminal(
+        &self,
+        task_id: &str,
+        heal_type: &HealType,
+        source: HealRequestSource,
+    ) -> Result<bool> {
+        let completed = CompletedHealStatus {
+            outcome: None,
+            progress: None,
+            retained_bytes: std::sync::OnceLock::new(),
+            heal_type: heal_type.clone(),
+            status: HealTaskStatus::Cancelled,
+            result_items_truncated: false,
+            completed_at: SystemTime::now(),
+            seqed_items: Vec::new(),
+            next_seq: 0,
+            min_seq: 0,
+        };
+        self.publish_admin_terminal(task_id, heal_type, source, &completed).await
     }
 
     pub async fn get_task_status(&self, task_id: &str) -> Result<HealTaskStatus> {
         let canonical_task_id = self.canonical_task_id(task_id).await;
-        match self.lookup_task_state(&canonical_task_id, None).await {
+        match self.lookup_task_state(&canonical_task_id, None).await? {
             TaskStateLookup::Active(task) => Ok(task.get_status().await),
             TaskStateLookup::Retrying(status) => Ok(status),
             TaskStateLookup::Completed(completed) => Ok(completed.status.clone()),
@@ -2013,7 +2238,7 @@ impl HealManager {
     /// full-snapshot semantics.
     pub async fn get_task_report_since(&self, task_id: &str, since: Option<u64>) -> Result<HealTaskReport> {
         let canonical_task_id = self.canonical_task_id(task_id).await;
-        match self.lookup_task_state(&canonical_task_id, None).await {
+        match self.lookup_task_state(&canonical_task_id, None).await? {
             TaskStateLookup::Active(task) => Ok(active_task_report(&task, since).await),
             TaskStateLookup::Retrying(status) => Ok(empty_task_report(status)),
             TaskStateLookup::Completed(completed) => Ok(completed_task_report(&completed, since)),
@@ -2036,7 +2261,7 @@ impl HealManager {
         since: Option<u64>,
     ) -> Result<HealTaskReport> {
         let canonical_task_id = self.canonical_task_id(task_id).await;
-        match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await {
+        match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await? {
             TaskStateLookup::Active(task) => Ok(active_task_report(&task, since).await),
             TaskStateLookup::Retrying(status) => Ok(empty_task_report(status)),
             TaskStateLookup::Completed(completed) => Ok(completed_task_report(&completed, since)),
@@ -2059,7 +2284,7 @@ impl HealManager {
     /// recently completed task, a different token is invalid for that path.
     pub async fn get_task_status_for_path(&self, heal_path: &str, task_id: &str) -> Result<HealTaskStatus> {
         let canonical_task_id = self.canonical_task_id(task_id).await;
-        match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await {
+        match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await? {
             TaskStateLookup::Active(task) => Ok(task.get_status().await),
             TaskStateLookup::Retrying(status) => Ok(status),
             TaskStateLookup::Completed(completed) => Ok(completed.status.clone()),
@@ -2113,11 +2338,18 @@ impl HealManager {
         }
         drop(completed_heals);
 
-        let mut displaced_terminals = lock_displaced_terminals(&self.displaced_terminals);
-        prune_completed_heal_statuses(&mut displaced_terminals);
-        displaced_terminals
-            .values()
-            .any(|terminal| heal_type_matches_path(&terminal.heal_type, heal_path))
+        {
+            let mut displaced_terminals = lock_displaced_terminals(&self.displaced_terminals);
+            prune_completed_heal_statuses(&mut displaced_terminals);
+            if displaced_terminals
+                .values()
+                .any(|terminal| heal_type_matches_path(&terminal.heal_type, heal_path))
+            {
+                return true;
+            }
+        }
+
+        self.root_recovery.completed_matches_path(heal_path).await.unwrap_or(false)
     }
 
     /// Get task progress
@@ -2129,7 +2361,7 @@ impl HealManager {
 
     pub async fn get_task_progress(&self, task_id: &str) -> Result<HealProgress> {
         let canonical_task_id = self.canonical_task_id(task_id).await;
-        let progress = match self.lookup_task_state(&canonical_task_id, None).await {
+        let progress = match self.lookup_task_state(&canonical_task_id, None).await? {
             TaskStateLookup::Active(task) => Some(task.get_progress().await),
             TaskStateLookup::Completed(completed) => completed.progress.clone(),
             _ => None,
@@ -2145,8 +2377,11 @@ impl HealManager {
         {
             let mut active_heals = self.active_heals.lock().await;
             if let Some(task) = active_heals.get(&canonical_task_id) {
-                task.cancel().await?;
                 let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
+                self.publish_admin_terminal(&canonical_task_id, &task.heal_type, task.source, &completed)
+                    .await?;
+                self.root_recovery.remove(&task.id, &task.heal_type, task.source).await?;
+                task.cancel().await?;
                 publish_completed_heal(&self.completed_heals, &self.task_aliases, &canonical_task_id, completed, true).await;
                 active_heals.remove(&canonical_task_id);
                 publish_active_heal_count(&active_heals);
@@ -2168,6 +2403,13 @@ impl HealManager {
 
         {
             let mut retrying_heals = self.retrying_heals.lock().await;
+            if let Some(retrying) = retrying_heals.get(&canonical_task_id) {
+                self.publish_admin_cancelled_terminal(&canonical_task_id, &retrying.request.heal_type, retrying.request.source)
+                    .await?;
+                self.root_recovery
+                    .remove(&canonical_task_id, &retrying.request.heal_type, retrying.request.source)
+                    .await?;
+            }
             if let Some(retrying) = retrying_heals.remove(&canonical_task_id) {
                 retrying.cancel_token.cancel();
                 drop(retrying_heals);
@@ -2188,6 +2430,13 @@ impl HealManager {
         }
 
         let mut queue = self.heal_queue.lock().await;
+        if let Some(request) = queue.requests().find(|request| request.id == canonical_task_id) {
+            self.publish_admin_cancelled_terminal(&canonical_task_id, &request.heal_type, request.source)
+                .await?;
+            self.root_recovery
+                .remove(&request.id, &request.heal_type, request.source)
+                .await?;
+        }
         if queue.remove_request_id(&canonical_task_id).is_some() {
             publish_heal_queue_length(&queue);
             info!(
@@ -2205,6 +2454,10 @@ impl HealManager {
             return Ok(());
         }
 
+        drop(queue);
+        if self.root_recovery.cancel_pending(&canonical_task_id).await? {
+            return Ok(());
+        }
         Err(Error::TaskNotFound {
             task_id: task_id.to_string(),
         })
@@ -2223,8 +2476,11 @@ impl HealManager {
 
             for task_id in &task_ids {
                 if let Some(task) = active_heals.get(task_id) {
-                    task.cancel().await?;
                     let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
+                    self.publish_admin_terminal(task_id, &task.heal_type, task.source, &completed)
+                        .await?;
+                    self.root_recovery.remove(&task.id, &task.heal_type, task.source).await?;
+                    task.cancel().await?;
                     publish_completed_heal(&self.completed_heals, &self.task_aliases, task_id, completed, true).await;
                 }
                 active_heals.remove(task_id);
@@ -2251,6 +2507,13 @@ impl HealManager {
                 .collect::<Vec<_>>();
 
             for task_id in &task_ids {
+                if let Some(retrying) = retrying_heals.get(task_id) {
+                    self.publish_admin_cancelled_terminal(task_id, &retrying.request.heal_type, retrying.request.source)
+                        .await?;
+                    self.root_recovery
+                        .remove(task_id, &retrying.request.heal_type, retrying.request.source)
+                        .await?;
+                }
                 if let Some(retrying) = retrying_heals.remove(task_id) {
                     retrying.cancel_token.cancel();
                     cancelled += 1;
@@ -2272,6 +2535,21 @@ impl HealManager {
             }
         }
 
+        let queued_matches = {
+            let queue = self.heal_queue.lock().await;
+            queue
+                .requests()
+                .filter(|request| heal_type_matches_path(&request.heal_type, heal_path))
+                .cloned()
+                .collect::<Vec<_>>()
+        };
+        for request in &queued_matches {
+            self.publish_admin_cancelled_terminal(&request.id, &request.heal_type, request.source)
+                .await?;
+            self.root_recovery
+                .remove(&request.id, &request.heal_type, request.source)
+                .await?;
+        }
         let mut queue = self.heal_queue.lock().await;
         let queued_cancelled = queue.remove_matching(|request| heal_type_matches_path(&request.heal_type, heal_path));
         if !queued_cancelled.is_empty() {
@@ -2284,6 +2562,17 @@ impl HealManager {
             self.remove_mrf_repair_notice_targets_for_task(&request.id);
         }
 
+        for pending in self
+            .root_recovery
+            .pending()
+            .await?
+            .into_iter()
+            .filter(|pending| heal_type_matches_path(&pending.heal_type, heal_path))
+        {
+            if self.root_recovery.cancel_pending(&pending.id).await? {
+                cancelled += 1;
+            }
+        }
         if cancelled == 0 {
             return Err(Error::TaskNotFound {
                 task_id: heal_path.to_string(),
@@ -2395,6 +2684,7 @@ impl std::fmt::Debug for HealManager {
 
 mod auto_scan;
 mod queue;
+mod root_recovery;
 mod scheduler;
 mod unclean_shutdown;
 

crates/heal/src/heal/manager/queue.rs

@@ -288,16 +288,29 @@ impl PriorityHealQueue {
         QueuePushOutcome::Accepted
     }
 
-    pub(super) fn can_displace_lower_priority(&self, priority: HealPriority) -> bool {
-        self.heap.iter().any(|item| item.priority < priority)
+    pub(super) fn can_displace_lower_priority_where<F>(&self, priority: HealPriority, can_displace: F) -> bool
+    where
+        F: Fn(&HealRequest) -> bool,
+    {
+        self.heap
+            .iter()
+            .any(|item| item.priority < priority && can_displace(&item.request))
     }
 
-    pub(super) fn push_displacing_lower_priority(&mut self, request: HealRequest) -> Option<HealRequest> {
+    #[cfg(test)]
+    pub(super) fn can_displace_lower_priority(&self, priority: HealPriority) -> bool {
+        self.can_displace_lower_priority_where(priority, |_| true)
+    }
+
+    pub(super) fn push_displacing_lower_priority_where<F>(&mut self, request: HealRequest, can_displace: F) -> Option<HealRequest>
+    where
+        F: Fn(&HealRequest) -> bool,
+    {
         let mut retained = BinaryHeap::new();
         let mut displaced: Option<PriorityQueueItem> = None;
 
         while let Some(item) = self.heap.pop() {
-            if item.priority < request.priority {
+            if item.priority < request.priority && can_displace(&item.request) {
                 let should_displace = displaced
                     .as_ref()
                     .map(|current| {
@@ -337,6 +350,11 @@ impl PriorityHealQueue {
         displaced
     }
 
+    #[cfg(test)]
+    pub(super) fn push_displacing_lower_priority(&mut self, request: HealRequest) -> Option<HealRequest> {
+        self.push_displacing_lower_priority_where(request, |_| true)
+    }
+
     /// Get statistics about queue contents by priority
     pub(super) fn get_priority_stats(&self) -> HashMap<HealPriority, usize> {
         let mut stats = HashMap::new();

crates/heal/src/heal/manager/root_recovery.rs

@@ -0,0 +1,938 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Graceful-shutdown handoff for administrator heals. This namespace is
+//! separate from erasure-set checkpoints and replacement generations, which
+//! cannot represent an admitted admin control-plane request. One coordinator
+//! disk owns each record; never create a fallback copy after an uncertain write
+//! or deletion.
+
+use super::*;
+use crate::heal::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes};
+use crate::heal::{DiskStore, RUSTFS_META_BUCKET};
+use serde::{Deserialize, Serialize};
+
+// The metadata bucket already exists and its parent is durable. Creating a
+// nested journal directory here would also require syncing every ancestor.
+const ROOT_RECOVERY_PREFIX: &str = "root-heal-";
+const ROOT_TERMINAL_PREFIX: &str = "terminal-root-heal-";
+const LEGACY_ROOT_RECOVERY_SCHEMA: u32 = 1;
+const ROOT_RECOVERY_SCHEMA: u32 = 2;
+const ROOT_TERMINAL_SCHEMA: u32 = 1;
+const ROOT_TERMINAL_GC_SCAN_BUDGET: usize = 1024;
+const ROOT_TERMINAL_GC_DELETE_BUDGET: usize = 64;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+enum RecoveryHealType {
+    Cluster,
+    Bucket {
+        bucket: String,
+    },
+    Object {
+        bucket: String,
+        object: String,
+        version_id: Option<String>,
+    },
+    Prefix {
+        bucket: String,
+        prefix: String,
+    },
+    ErasureSet {
+        buckets: Vec<String>,
+        set_disk_id: String,
+    },
+    Metadata {
+        bucket: String,
+        object: String,
+    },
+    EcDecode {
+        bucket: String,
+        object: String,
+        version_id: Option<String>,
+    },
+}
+
+impl RecoveryHealType {
+    fn validate(&self) -> Result<()> {
+        match self {
+            Self::Cluster => {}
+            Self::Bucket { bucket } => validate_recovery_component("bucket", bucket)?,
+            Self::Object {
+                bucket,
+                object,
+                version_id,
+            }
+            | Self::EcDecode {
+                bucket,
+                object,
+                version_id,
+            } => {
+                validate_recovery_component("bucket", bucket)?;
+                validate_recovery_component("object", object)?;
+                if let Some(version_id) = version_id {
+                    validate_recovery_component("version id", version_id)?;
+                }
+            }
+            Self::Prefix { bucket, prefix } => {
+                validate_recovery_component("bucket", bucket)?;
+                validate_recovery_component("prefix", prefix)?;
+            }
+            Self::ErasureSet { buckets, set_disk_id } => {
+                validate_recovery_component("set disk id", set_disk_id)?;
+                if buckets.is_empty() {
+                    return Err(Error::Other("Admin heal recovery erasure set must name buckets".to_string()));
+                }
+                for bucket in buckets {
+                    validate_recovery_component("bucket", bucket)?;
+                }
+            }
+            Self::Metadata { bucket, object } => {
+                validate_recovery_component("bucket", bucket)?;
+                validate_recovery_component("object", object)?;
+            }
+        }
+        Ok(())
+    }
+}
+
+impl From<&HealType> for RecoveryHealType {
+    fn from(heal_type: &HealType) -> Self {
+        match heal_type {
+            HealType::Cluster => Self::Cluster,
+            HealType::Bucket { bucket } => Self::Bucket { bucket: bucket.clone() },
+            HealType::Object {
+                bucket,
+                object,
+                version_id,
+            } => Self::Object {
+                bucket: bucket.clone(),
+                object: object.clone(),
+                version_id: version_id.clone(),
+            },
+            HealType::Prefix { bucket, prefix } => Self::Prefix {
+                bucket: bucket.clone(),
+                prefix: prefix.clone(),
+            },
+            HealType::ErasureSet { buckets, set_disk_id } => Self::ErasureSet {
+                buckets: buckets.clone(),
+                set_disk_id: set_disk_id.clone(),
+            },
+            HealType::Metadata { bucket, object } => Self::Metadata {
+                bucket: bucket.clone(),
+                object: object.clone(),
+            },
+            HealType::ECDecode {
+                bucket,
+                object,
+                version_id,
+            } => Self::EcDecode {
+                bucket: bucket.clone(),
+                object: object.clone(),
+                version_id: version_id.clone(),
+            },
+        }
+    }
+}
+
+impl From<RecoveryHealType> for HealType {
+    fn from(heal_type: RecoveryHealType) -> Self {
+        match heal_type {
+            RecoveryHealType::Cluster => Self::Cluster,
+            RecoveryHealType::Bucket { bucket } => Self::Bucket { bucket },
+            RecoveryHealType::Object {
+                bucket,
+                object,
+                version_id,
+            } => Self::Object {
+                bucket,
+                object,
+                version_id,
+            },
+            RecoveryHealType::Prefix { bucket, prefix } => Self::Prefix { bucket, prefix },
+            RecoveryHealType::ErasureSet { buckets, set_disk_id } => Self::ErasureSet { buckets, set_disk_id },
+            RecoveryHealType::Metadata { bucket, object } => Self::Metadata { bucket, object },
+            RecoveryHealType::EcDecode {
+                bucket,
+                object,
+                version_id,
+            } => Self::ECDecode {
+                bucket,
+                object,
+                version_id,
+            },
+        }
+    }
+}
+
+fn validate_recovery_component(label: &str, value: &str) -> Result<()> {
+    if value.is_empty() || value.contains('\0') {
+        return Err(Error::Other(format!("Invalid admin heal recovery {label}")));
+    }
+    Ok(())
+}
+
+fn default_recovery_heal_type() -> RecoveryHealType {
+    RecoveryHealType::Cluster
+}
+
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct RootHealIntent {
+    schema: u32,
+    task_id: String,
+    #[serde(default = "default_recovery_heal_type")]
+    heal_type: RecoveryHealType,
+    #[serde(deserialize_with = "decode_options")]
+    options: HealOptions,
+    priority: HealPriority,
+    retry_attempts: u32,
+    created_at: SystemTime,
+}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct RootHealTerminal {
+    schema: u32,
+    task_id: String,
+    heal_type: RecoveryHealType,
+    status: HealTaskStatus,
+    progress: Option<HealProgress>,
+    completed_at: SystemTime,
+}
+
+impl RootHealTerminal {
+    fn from_completed(task_id: &str, completed: &CompletedHealStatus) -> Self {
+        Self {
+            schema: ROOT_TERMINAL_SCHEMA,
+            task_id: task_id.to_owned(),
+            heal_type: RecoveryHealType::from(&completed.heal_type),
+            status: completed.status.clone(),
+            progress: completed.progress.clone(),
+            completed_at: completed.completed_at,
+        }
+    }
+
+    fn cancelled(task_id: &str, heal_type: &HealType) -> Self {
+        Self {
+            schema: ROOT_TERMINAL_SCHEMA,
+            task_id: task_id.to_owned(),
+            heal_type: RecoveryHealType::from(heal_type),
+            status: HealTaskStatus::Cancelled,
+            progress: None,
+            completed_at: SystemTime::now(),
+        }
+    }
+
+    fn into_completed(self) -> CompletedHealStatus {
+        CompletedHealStatus {
+            outcome: None,
+            progress: self.progress,
+            retained_bytes: std::sync::OnceLock::new(),
+            heal_type: self.heal_type.into(),
+            status: self.status,
+            result_items_truncated: false,
+            completed_at: self.completed_at,
+            seqed_items: Vec::new(),
+            next_seq: 0,
+            min_seq: 0,
+        }
+    }
+
+    fn retained_at(&self, now: SystemTime) -> bool {
+        now.duration_since(self.completed_at)
+            .map(|age| age <= KEEP_HEAL_TASK_STATUS_DURATION)
+            .unwrap_or(true)
+    }
+}
+
+impl RootHealIntent {
+    fn from_request(request: &HealRequest) -> Self {
+        Self {
+            schema: ROOT_RECOVERY_SCHEMA,
+            task_id: request.id.clone(),
+            heal_type: RecoveryHealType::from(&request.heal_type),
+            options: request.options.clone(),
+            priority: request.priority,
+            retry_attempts: request.retry_attempts,
+            created_at: request.created_at,
+        }
+    }
+
+    fn into_request(self) -> HealRequest {
+        let mut request = HealRequest::new(self.heal_type.into(), self.options, self.priority);
+        request.id = self.task_id;
+        request.source = HealRequestSource::Admin;
+        request.retry_attempts = self.retry_attempts;
+        request.created_at = self.created_at;
+        request
+    }
+}
+
+#[derive(Default)]
+pub(super) struct RootHealRecovery {
+    mutation: Mutex<()>,
+    #[cfg(any(test, feature = "test-util"))]
+    disabled_for_tests: bool,
+    #[cfg(test)]
+    disks: Option<Vec<DiskStore>>,
+}
+
+pub(super) fn is_admin_heal_recovery(heal_type: &HealType, source: HealRequestSource) -> bool {
+    source == HealRequestSource::Admin
+        && matches!(
+            heal_type,
+            HealType::Cluster
+                | HealType::Bucket { .. }
+                | HealType::Object { .. }
+                | HealType::Prefix { .. }
+                | HealType::ErasureSet { .. }
+                | HealType::Metadata { .. }
+                | HealType::ECDecode { .. }
+        )
+}
+
+fn decode_options<'de, D: serde::Deserializer<'de>>(deserializer: D) -> std::result::Result<HealOptions, D::Error> {
+    let value = serde_json::Value::deserialize(deserializer)?;
+    let object = value
+        .as_object()
+        .ok_or_else(|| serde::de::Error::custom("root heal options must be an object"))?;
+    const FIELDS: &[&str] = &[
+        "scan_mode",
+        "remove_corrupted",
+        "recreate_missing",
+        "update_parity",
+        "recursive",
+        "dry_run",
+        "no_lock",
+        "timeout",
+        "pool_index",
+        "set_index",
+    ];
+    if object.keys().any(|key| !FIELDS.contains(&key.as_str())) {
+        return Err(serde::de::Error::custom("unknown root heal recovery option"));
+    }
+    let options: HealOptions = serde_json::from_value(value).map_err(serde::de::Error::custom)?;
+    if options.no_lock {
+        return Err(serde::de::Error::custom("administrator root heal cannot skip namespace locking"));
+    }
+    Ok(options)
+}
+
+fn intent_path(task_id: &str) -> Result<String> {
+    let parsed = uuid::Uuid::parse_str(task_id).map_err(|_| Error::Other("Invalid root heal recovery task id".to_string()))?;
+    if parsed.to_string() != task_id {
+        return Err(Error::Other("Noncanonical root heal recovery task id".to_string()));
+    }
+    Ok(format!("{ROOT_RECOVERY_PREFIX}{task_id}.json"))
+}
+
+fn terminal_path(task_id: &str) -> Result<String> {
+    let parsed = uuid::Uuid::parse_str(task_id).map_err(|_| Error::Other("Invalid root heal terminal task id".to_string()))?;
+    if parsed.to_string() != task_id {
+        return Err(Error::Other("Noncanonical root heal terminal task id".to_string()));
+    }
+    Ok(format!("{ROOT_TERMINAL_PREFIX}{task_id}.json"))
+}
+
+fn decode_intent(task_id: &str, bytes: &[u8]) -> Result<RootHealIntent> {
+    let _ = intent_path(task_id)?;
+    let intent: RootHealIntent = serde_json::from_slice(bytes)
+        .map_err(|error| Error::Other(format!("Invalid root heal recovery record {task_id}: {error}")))?;
+    if intent.task_id != task_id {
+        return Err(Error::Other(format!("Unsupported or mismatched root heal recovery record {task_id}")));
+    }
+    match intent.schema {
+        LEGACY_ROOT_RECOVERY_SCHEMA if intent.heal_type == RecoveryHealType::Cluster => {}
+        ROOT_RECOVERY_SCHEMA => {}
+        _ => return Err(Error::Other(format!("Unsupported or mismatched root heal recovery record {task_id}"))),
+    }
+    intent.heal_type.validate()?;
+    Ok(intent)
+}
+
+fn decode_terminal(task_id: &str, bytes: &[u8]) -> Result<RootHealTerminal> {
+    let _ = terminal_path(task_id)?;
+    let terminal: RootHealTerminal = serde_json::from_slice(bytes)
+        .map_err(|error| Error::Other(format!("Invalid root heal terminal record {task_id}: {error}")))?;
+    if terminal.schema != ROOT_TERMINAL_SCHEMA || terminal.task_id != task_id {
+        return Err(Error::Other(format!("Unsupported or mismatched root heal terminal record {task_id}")));
+    }
+    terminal.heal_type.validate()?;
+    if !matches!(
+        terminal.status,
+        HealTaskStatus::Completed | HealTaskStatus::Cancelled | HealTaskStatus::Failed { .. }
+    ) {
+        return Err(Error::Other(format!("Non-terminal root heal receipt {task_id}")));
+    }
+    Ok(terminal)
+}
+
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
+pub(super) struct RootTerminalGcReport {
+    pub(super) scanned: usize,
+    pub(super) retained: usize,
+    pub(super) pending_removed: usize,
+    pub(super) terminals_removed: usize,
+    pub(super) budget_exhausted: bool,
+}
+
+impl RootHealRecovery {
+    #[cfg(test)]
+    pub(super) fn with_disks(disks: Vec<DiskStore>) -> Self {
+        Self {
+            mutation: Mutex::new(()),
+            disabled_for_tests: false,
+            disks: Some(disks),
+        }
+    }
+
+    #[cfg(any(test, feature = "test-util"))]
+    pub(super) fn disabled_for_tests() -> Self {
+        Self {
+            mutation: Mutex::new(()),
+            disabled_for_tests: true,
+            #[cfg(test)]
+            disks: None,
+        }
+    }
+
+    async fn disks(&self) -> Result<Vec<DiskStore>> {
+        #[cfg(any(test, feature = "test-util"))]
+        if self.disabled_for_tests {
+            return Ok(Vec::new());
+        }
+        #[cfg(test)]
+        if let Some(disks) = &self.disks {
+            return Ok(disks.clone());
+        }
+        let map = local_disk_map_read().await;
+        if map.values().any(Option::is_none) {
+            return Err(Error::Other("Root heal recovery owner may be on an unavailable local disk".to_string()));
+        }
+        let mut disks = map.values().flatten().cloned().collect::<Vec<_>>();
+        disks.sort_by_key(|disk| EcstoreDiskAPI::endpoint(disk.as_ref()).to_string());
+        Ok(disks)
+    }
+
+    async fn find(disks: &[DiskStore], task_id: &str) -> Result<Option<(DiskStore, EcstoreDiskBytes)>> {
+        let path = intent_path(task_id)?;
+        let mut found = None;
+        for disk in disks {
+            // read_all reports FileNotFound even when the whole metadata
+            // volume is absent; that is an unknown owner, not empty state.
+            EcstoreDiskAPI::stat_volume(disk.as_ref(), RUSTFS_META_BUCKET).await?;
+            match EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, &path).await {
+                Ok(bytes) => {
+                    decode_intent(task_id, &bytes)?;
+                    if found.is_some() {
+                        return Err(Error::Other(format!("Multiple root heal recovery owners for {task_id}")));
+                    }
+                    found = Some((disk.clone(), bytes));
+                }
+                Err(DiskError::FileNotFound) => {}
+                Err(error) => return Err(Error::Disk(error)),
+            }
+        }
+        Ok(found)
+    }
+
+    async fn find_terminal(disks: &[DiskStore], task_id: &str) -> Result<Option<(DiskStore, EcstoreDiskBytes)>> {
+        let path = terminal_path(task_id)?;
+        let mut found = None;
+        for disk in disks {
+            EcstoreDiskAPI::stat_volume(disk.as_ref(), RUSTFS_META_BUCKET).await?;
+            match EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, &path).await {
+                Ok(bytes) => {
+                    decode_terminal(task_id, &bytes)?;
+                    if found.is_some() {
+                        return Err(Error::Other(format!("Multiple root heal terminal owners for {task_id}")));
+                    }
+                    found = Some((disk.clone(), bytes));
+                }
+                Err(DiskError::FileNotFound) => {}
+                Err(error) => return Err(Error::Disk(error)),
+            }
+        }
+        Ok(found)
+    }
+
+    async fn find_retained_terminal(
+        disks: &[DiskStore],
+        task_id: &str,
+        now: SystemTime,
+    ) -> Result<Option<(DiskStore, EcstoreDiskBytes)>> {
+        let Some((disk, bytes)) = Self::find_terminal(disks, task_id).await? else {
+            return Ok(None);
+        };
+        if decode_terminal(task_id, &bytes)?.retained_at(now) {
+            return Ok(Some((disk, bytes)));
+        }
+        Ok(None)
+    }
+
+    async fn persist_terminal_locked(
+        disks: &[DiskStore],
+        task_id: &str,
+        terminal: RootHealTerminal,
+    ) -> Result<Option<(DiskStore, EcstoreDiskBytes)>> {
+        let path = terminal_path(task_id)?;
+        if let Some((_, bytes)) = Self::find_terminal(disks, task_id).await? {
+            let current = decode_terminal(task_id, &bytes)?;
+            if current == terminal {
+                return Self::find(disks, task_id).await;
+            }
+            return Err(Error::Other(format!("Root heal terminal record changed for {task_id}")));
+        }
+        let pending = Self::find(disks, task_id).await?;
+        let disk = pending
+            .as_ref()
+            .map(|(disk, _)| disk.clone())
+            .or_else(|| disks.first().cloned())
+            .ok_or_else(|| Error::Other("No local disk available for root heal terminal receipt".to_string()))?;
+        let bytes = serde_json::to_vec(&terminal)
+            .map_err(|error| Error::Other(format!("Serialize root heal terminal receipt: {error}")))?;
+        match EcstoreDiskAPI::compare_and_update_file(disk.as_ref(), RUSTFS_META_BUCKET, &path, None, Some(bytes.into())).await? {
+            EcstoreConditionalFileUpdate::Updated => Ok(pending),
+            _ => Err(Error::Other(format!("Root heal terminal record changed for {task_id}"))),
+        }
+    }
+
+    pub(super) async fn persist(&self, request: &HealRequest) -> Result<()> {
+        if !is_admin_heal_recovery(&request.heal_type, request.source) {
+            return Ok(());
+        }
+        #[cfg(any(test, feature = "test-util"))]
+        if self.disabled_for_tests {
+            return Ok(());
+        }
+        let _guard = self.mutation.lock().await;
+        let disks = self.disks().await?;
+        let existing = Self::find(&disks, &request.id).await?;
+        let (disk, expected) = match existing {
+            Some((disk, bytes)) => (disk, Some(bytes)),
+            None => {
+                let disk = disks
+                    .first()
+                    .cloned()
+                    .ok_or_else(|| Error::Other("No local disk available for root heal shutdown recovery".to_string()))?;
+                (disk, None)
+            }
+        };
+        if request.options.no_lock {
+            return Err(Error::Other("Administrator root heal cannot skip namespace locking".to_string()));
+        }
+        let bytes = serde_json::to_vec(&RootHealIntent::from_request(request))
+            .map_err(|error| Error::Other(format!("Serialize root heal recovery record: {error}")))?;
+        match EcstoreDiskAPI::compare_and_update_file(
+            disk.as_ref(),
+            RUSTFS_META_BUCKET,
+            &intent_path(&request.id)?,
+            expected,
+            Some(bytes.into()),
+        )
+        .await?
+        {
+            EcstoreConditionalFileUpdate::Updated => Ok(()),
+            _ => Err(Error::Other(format!("Root heal recovery record changed for {}", request.id))),
+        }
+    }
+
+    pub(super) async fn remove(&self, task_id: &str, heal_type: &HealType, source: HealRequestSource) -> Result<bool> {
+        if !is_admin_heal_recovery(heal_type, source) {
+            return Ok(false);
+        }
+        #[cfg(any(test, feature = "test-util"))]
+        if self.disabled_for_tests {
+            return Ok(false);
+        }
+        self.remove_pending_by_id(task_id).await
+    }
+
+    async fn remove_pending_by_id(&self, task_id: &str) -> Result<bool> {
+        let _guard = self.mutation.lock().await;
+        let Some((disk, bytes)) = Self::find(&self.disks().await?, task_id).await? else {
+            return Ok(false);
+        };
+        match EcstoreDiskAPI::compare_and_update_file(
+            disk.as_ref(),
+            RUSTFS_META_BUCKET,
+            &intent_path(task_id)?,
+            Some(bytes),
+            None,
+        )
+        .await?
+        {
+            EcstoreConditionalFileUpdate::Updated => Ok(true),
+            _ => Err(Error::Other(format!("Root heal recovery record changed while retiring {task_id}"))),
+        }
+    }
+
+    pub(super) async fn checkpoint_failed_execution(&self, task: &HealTask) -> Result<()> {
+        if !is_admin_heal_recovery(&task.heal_type, task.source) {
+            return Ok(());
+        }
+        let remaining = match task.retry_request_with_remaining_timeout().await {
+            Ok(request) => request.options.timeout,
+            Err(Error::TaskTimeout) => Some(Duration::ZERO),
+            Err(error) => return Err(error),
+        };
+        let _guard = self.mutation.lock().await;
+        let Some((disk, expected)) = Self::find(&self.disks().await?, &task.id).await? else {
+            // A first execution that failed has no restart handoff to update.
+            return Ok(());
+        };
+        let mut intent = decode_intent(&task.id, &expected)?;
+        if HealType::from(intent.heal_type.clone()) != task.heal_type {
+            return Err(Error::Other(format!("Root heal recovery owner changed for {}", task.id)));
+        }
+        let mut expected_options = intent.options.clone();
+        expected_options.timeout = task.options.timeout;
+        if intent.created_at != task.created_at || intent.priority != task.priority || expected_options != task.options {
+            return Err(Error::Other(format!("Root heal recovery owner changed for {}", task.id)));
+        }
+        // A terminal timeout leaves no runtime owner for stop() to snapshot.
+        // Checkpoint its consumed budget before publishing terminal status;
+        // never refund time if an earlier checkpoint is already stricter.
+        intent.options.timeout = match (intent.options.timeout, remaining) {
+            (Some(previous), Some(remaining)) => Some(previous.min(remaining)),
+            (previous, remaining) => previous.or(remaining),
+        };
+        intent.retry_attempts = intent.retry_attempts.max(task.retry_attempts);
+        let bytes = serde_json::to_vec(&intent)
+            .map_err(|error| Error::Other(format!("Serialize root heal recovery checkpoint: {error}")))?;
+        match EcstoreDiskAPI::compare_and_update_file(
+            disk.as_ref(),
+            RUSTFS_META_BUCKET,
+            &intent_path(&task.id)?,
+            Some(expected),
+            Some(bytes.into()),
+        )
+        .await?
+        {
+            EcstoreConditionalFileUpdate::Updated => Ok(()),
+            _ => Err(Error::Other(format!("Root heal recovery record changed while checkpointing {}", task.id))),
+        }
+    }
+
+    pub(super) async fn cancel_pending(&self, task_id: &str) -> Result<bool> {
+        #[cfg(any(test, feature = "test-util"))]
+        if self.disabled_for_tests {
+            return Ok(false);
+        }
+        if intent_path(task_id).is_err() {
+            return Ok(false);
+        }
+        let _guard = self.mutation.lock().await;
+        let disks = self.disks().await?;
+        if Self::find_terminal(&disks, task_id).await?.is_some() {
+            if let Some((disk, bytes)) = Self::find(&disks, task_id).await? {
+                match EcstoreDiskAPI::compare_and_update_file(
+                    disk.as_ref(),
+                    RUSTFS_META_BUCKET,
+                    &intent_path(task_id)?,
+                    Some(bytes),
+                    None,
+                )
+                .await?
+                {
+                    EcstoreConditionalFileUpdate::Updated => {}
+                    _ => return Err(Error::Other(format!("Root heal recovery record changed while cancelling {task_id}"))),
+                }
+            }
+            return Ok(true);
+        }
+        let Some((disk, bytes)) = Self::find(&disks, task_id).await? else {
+            return Ok(false);
+        };
+        let pending = decode_intent(task_id, &bytes)?;
+        let heal_type = HealType::from(pending.heal_type);
+        let terminal = RootHealTerminal::cancelled(task_id, &heal_type);
+        let _ = Self::persist_terminal_locked(&disks, task_id, terminal).await?;
+        match EcstoreDiskAPI::compare_and_update_file(
+            disk.as_ref(),
+            RUSTFS_META_BUCKET,
+            &intent_path(task_id)?,
+            Some(bytes),
+            None,
+        )
+        .await?
+        {
+            EcstoreConditionalFileUpdate::Updated => Ok(true),
+            _ => Err(Error::Other(format!("Root heal recovery record changed while cancelling {task_id}"))),
+        }
+    }
+
+    pub(super) async fn persist_terminal(
+        &self,
+        task_id: &str,
+        heal_type: &HealType,
+        source: HealRequestSource,
+        completed: &CompletedHealStatus,
+    ) -> Result<bool> {
+        if !is_admin_heal_recovery(heal_type, source) || completed.heal_type != *heal_type {
+            return Ok(false);
+        }
+        #[cfg(any(test, feature = "test-util"))]
+        if self.disabled_for_tests {
+            return Ok(false);
+        }
+        let _guard = self.mutation.lock().await;
+        let disks = self.disks().await?;
+        let pending =
+            Self::persist_terminal_locked(&disks, task_id, RootHealTerminal::from_completed(task_id, completed)).await?;
+        if let Some((disk, bytes)) = pending {
+            match EcstoreDiskAPI::compare_and_update_file(
+                disk.as_ref(),
+                RUSTFS_META_BUCKET,
+                &intent_path(task_id)?,
+                Some(bytes),
+                None,
+            )
+            .await?
+            {
+                EcstoreConditionalFileUpdate::Updated => {}
+                _ => {
+                    return Err(Error::Other(format!(
+                        "Root heal recovery record changed while publishing terminal {task_id}"
+                    )));
+                }
+            }
+        }
+        Ok(true)
+    }
+
+    pub(super) async fn completed(&self, task_id: &str) -> Result<Option<CompletedHealStatus>> {
+        #[cfg(any(test, feature = "test-util"))]
+        if self.disabled_for_tests {
+            return Ok(None);
+        }
+        if terminal_path(task_id).is_err() {
+            return Ok(None);
+        }
+        let _guard = self.mutation.lock().await;
+        let disks = self.disks().await?;
+        let Some((_, bytes)) = Self::find_retained_terminal(&disks, task_id, SystemTime::now()).await? else {
+            return Ok(None);
+        };
+        Ok(Some(decode_terminal(task_id, &bytes)?.into_completed()))
+    }
+
+    pub(super) async fn completed_matches_path(&self, heal_path: &str) -> Result<bool> {
+        #[cfg(any(test, feature = "test-util"))]
+        if self.disabled_for_tests {
+            return Ok(false);
+        }
+        let _guard = self.mutation.lock().await;
+        let disks = self.disks().await?;
+        for disk in &disks {
+            EcstoreDiskAPI::stat_volume(disk.as_ref(), RUSTFS_META_BUCKET).await?;
+            let entries = match EcstoreDiskAPI::list_dir(disk.as_ref(), "", RUSTFS_META_BUCKET, "", -1).await {
+                Ok(entries) => entries,
+                Err(DiskError::FileNotFound) => continue,
+                Err(error) => return Err(Error::Disk(error)),
+            };
+            for entry in entries {
+                let Some(task_id) = entry
+                    .strip_prefix(ROOT_TERMINAL_PREFIX)
+                    .and_then(|entry| entry.strip_suffix(".json"))
+                else {
+                    continue;
+                };
+                let Some((_, bytes)) = Self::find_retained_terminal(&disks, task_id, SystemTime::now()).await? else {
+                    continue;
+                };
+                let heal_type = HealType::from(decode_terminal(task_id, &bytes)?.heal_type);
+                if heal_type_matches_path(&heal_type, heal_path) {
+                    return Ok(true);
+                }
+            }
+        }
+        Ok(false)
+    }
+
+    pub(super) async fn gc_terminal_receipts_once(&self, now: SystemTime) -> Result<RootTerminalGcReport> {
+        #[cfg(any(test, feature = "test-util"))]
+        if self.disabled_for_tests {
+            return Ok(RootTerminalGcReport::default());
+        }
+        let _guard = self.mutation.lock().await;
+        let disks = self.disks().await?;
+        let mut report = RootTerminalGcReport::default();
+        let mut ids = HashSet::new();
+        for disk in &disks {
+            if report.scanned >= ROOT_TERMINAL_GC_SCAN_BUDGET {
+                report.budget_exhausted = true;
+                break;
+            }
+            EcstoreDiskAPI::stat_volume(disk.as_ref(), RUSTFS_META_BUCKET).await?;
+            let remaining = ROOT_TERMINAL_GC_SCAN_BUDGET.saturating_sub(report.scanned);
+            let count = i32::try_from(remaining).unwrap_or(i32::MAX);
+            let mut entries = match EcstoreDiskAPI::list_dir(disk.as_ref(), "", RUSTFS_META_BUCKET, "", count).await {
+                Ok(entries) => entries,
+                Err(DiskError::FileNotFound) => continue,
+                Err(error) => return Err(Error::Disk(error)),
+            };
+            entries.sort_unstable();
+            for entry in entries {
+                if report.scanned >= ROOT_TERMINAL_GC_SCAN_BUDGET {
+                    report.budget_exhausted = true;
+                    break;
+                }
+                report.scanned += 1;
+                let Some(task_id) = entry
+                    .strip_prefix(ROOT_TERMINAL_PREFIX)
+                    .and_then(|entry| entry.strip_suffix(".json"))
+                else {
+                    continue;
+                };
+                let _ = terminal_path(task_id)?;
+                ids.insert(task_id.to_string());
+            }
+        }
+
+        let mut ids = ids.into_iter().collect::<Vec<_>>();
+        ids.sort();
+        let mut deletes = 0usize;
+        for task_id in ids {
+            if deletes >= ROOT_TERMINAL_GC_DELETE_BUDGET {
+                report.budget_exhausted = true;
+                break;
+            }
+            let Some((terminal_disk, terminal_bytes)) = Self::find_terminal(&disks, &task_id).await? else {
+                continue;
+            };
+            let terminal = decode_terminal(&task_id, &terminal_bytes)?;
+            if terminal.retained_at(now) {
+                report.retained += 1;
+                continue;
+            }
+            if let Some((pending_disk, pending_bytes)) = Self::find(&disks, &task_id).await? {
+                match EcstoreDiskAPI::compare_and_update_file(
+                    pending_disk.as_ref(),
+                    RUSTFS_META_BUCKET,
+                    &intent_path(&task_id)?,
+                    Some(pending_bytes),
+                    None,
+                )
+                .await?
+                {
+                    EcstoreConditionalFileUpdate::Updated => {
+                        deletes += 1;
+                        report.pending_removed += 1;
+                        report.retained += 1;
+                        continue;
+                    }
+                    _ => {
+                        return Err(Error::Other(format!(
+                            "Root heal recovery record changed while pruning terminal receipt {task_id}"
+                        )));
+                    }
+                }
+            }
+            match EcstoreDiskAPI::compare_and_update_file(
+                terminal_disk.as_ref(),
+                RUSTFS_META_BUCKET,
+                &terminal_path(&task_id)?,
+                Some(terminal_bytes),
+                None,
+            )
+            .await?
+            {
+                EcstoreConditionalFileUpdate::Updated => {
+                    deletes += 1;
+                    report.terminals_removed += 1;
+                }
+                _ => return Err(Error::Other(format!("Root heal terminal record changed while pruning {task_id}"))),
+            }
+        }
+        Ok(report)
+    }
+
+    pub(super) async fn pending(&self) -> Result<Vec<HealRequest>> {
+        #[cfg(any(test, feature = "test-util"))]
+        if self.disabled_for_tests {
+            return Ok(Vec::new());
+        }
+        let _guard = self.mutation.lock().await;
+        let disks = self.disks().await?;
+        let mut ids = HashSet::new();
+        for disk in &disks {
+            EcstoreDiskAPI::stat_volume(disk.as_ref(), RUSTFS_META_BUCKET).await?;
+            let entries = match EcstoreDiskAPI::list_dir(disk.as_ref(), "", RUSTFS_META_BUCKET, "", -1).await {
+                Ok(entries) => entries,
+                Err(DiskError::FileNotFound) => continue,
+                Err(error) => return Err(Error::Disk(error)),
+            };
+            for entry in entries {
+                let Some(task_id) = entry
+                    .strip_prefix(ROOT_RECOVERY_PREFIX)
+                    .and_then(|entry| entry.strip_suffix(".json"))
+                else {
+                    continue;
+                };
+                let _ = intent_path(task_id)?;
+                ids.insert(task_id.to_string());
+            }
+        }
+        let mut requests = Vec::new();
+        for task_id in ids {
+            if Self::find_terminal(&disks, &task_id).await?.is_some() {
+                continue;
+            }
+            if let Some((_, bytes)) = Self::find(&disks, &task_id).await? {
+                requests.push(decode_intent(&task_id, &bytes)?.into_request());
+            }
+        }
+        requests.sort_by(|left, right| left.created_at.cmp(&right.created_at).then_with(|| left.id.cmp(&right.id)));
+        Ok(requests)
+    }
+}
+
+impl HealManager {
+    pub(super) async fn replay_root_heals(&self) -> Result<()> {
+        // Decode every record before admitting anything. These are already
+        // accepted responsibilities, so restore distinct IDs even when their
+        // paths overlap or the configured admission capacity has changed.
+        let requests = self.root_recovery.pending().await?;
+        let active = self.active_heals.lock().await;
+        let mut queue = self.heal_queue.lock().await;
+        let retrying = self.retrying_heals.lock().await;
+        for mut request in requests {
+            request.force_start = true;
+            let existing = active
+                .get(&request.id)
+                .map(|task| request_matches_task(&request, task))
+                .or_else(|| {
+                    queue
+                        .requests()
+                        .find(|queued| queued.id == request.id)
+                        .map(|queued| request_matches_request(&request, queued))
+                })
+                .or_else(|| {
+                    retrying
+                        .get(&request.id)
+                        .map(|retrying| request_matches_request(&request, &retrying.request))
+                });
+            match existing {
+                Some(true) => continue,
+                Some(false) => return Err(Error::Other(format!("Conflicting root heal recovery task {}", request.id))),
+                None => {}
+            }
+            queue.push(request);
+        }
+        publish_heal_queue_length(&queue);
+        Ok(())
+    }
+}

crates/heal/src/heal/manager/scheduler.rs

@@ -26,6 +26,7 @@ impl HealManager {
         let retrying_heals = self.retrying_heals.clone();
         let mrf_repair_notice_targets = self.mrf_repair_notice_targets.clone();
         let replacement_recovery_anchors = self.replacement_recovery_anchors.clone();
+        let root_recovery = self.root_recovery.clone();
         let cancel_token = self.cancel_token.clone();
         let statistics = self.statistics.clone();
         let storage = self.storage.clone();
@@ -59,6 +60,7 @@ impl HealManager {
                             retrying_heals: &retrying_heals,
                             mrf_repair_notice_targets: &mrf_repair_notice_targets,
                             replacement_recovery_anchors: &replacement_recovery_anchors,
+                            root_recovery: &root_recovery,
                             config: &config,
                             statistics: &statistics,
                             storage: &storage,
@@ -78,6 +80,7 @@ impl HealManager {
                             retrying_heals: &retrying_heals,
                             mrf_repair_notice_targets: &mrf_repair_notice_targets,
                             replacement_recovery_anchors: &replacement_recovery_anchors,
+                            root_recovery: &root_recovery,
                             config: &config,
                             statistics: &statistics,
                             storage: &storage,
@@ -106,6 +109,7 @@ impl HealManager {
             retrying_heals,
             mrf_repair_notice_targets,
             replacement_recovery_anchors,
+            root_recovery,
             config,
             statistics,
             storage,
@@ -117,6 +121,9 @@ impl HealManager {
         let config = config.read().await;
         let mainline_pressure = Self::mainline_throttle_active(&config, workload_provider);
         let mut active_heals_guard = active_heals.lock().await;
+        if cancel_token.is_cancelled() {
+            return;
+        }
         publish_active_heal_count(&active_heals_guard);
 
         // Check if new heal tasks can be started
@@ -206,6 +213,7 @@ impl HealManager {
                 let replacement_recovery_anchors_clone = replacement_recovery_anchors.clone();
                 let statistics_clone = statistics.clone();
                 let notify_clone = notify.clone();
+                let root_recovery_clone = root_recovery.clone();
                 let manager_cancel_token = cancel_token.clone();
                 let task_type_label_for_spawn = task_type_label.clone();
                 let task_set_label_for_spawn = task_set_label.clone();
@@ -312,7 +320,47 @@ impl HealManager {
                         completed_status_entry.status = HealTaskStatus::Cancelled;
                         completed_status_entry.outcome = Some(Arc::new(task.get_outcome().await));
                     }
-                    let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
+                    let terminal_completion = matches!(
+                        completed_status,
+                        HealTaskStatus::Completed | HealTaskStatus::Cancelled | HealTaskStatus::Failed { .. }
+                    );
+                    if owns_completion
+                        && terminal_completion
+                        && root_recovery::is_admin_heal_recovery(&task.heal_type, task.source)
+                        && let Err(error) = root_recovery_clone
+                            .persist_terminal(&task_id, &task.heal_type, task.source, &completed_status_entry)
+                            .await
+                    {
+                        // Keep the durable responsibility if terminal
+                        // publication fails. Replaying the task is preferable
+                        // to losing the final receipt across restart.
+                        warn!(
+                            target: "rustfs::heal::manager",
+                            event = EVENT_HEAL_SCHEDULER_STATE,
+                            component = LOG_COMPONENT_HEAL,
+                            subsystem = LOG_SUBSYSTEM_MANAGER,
+                            task_id,
+                            state = "root_recovery_terminal_publish_failed",
+                            error = %error,
+                            "Failed to publish heal terminal receipt"
+                        );
+                    }
+                    if owns_completion
+                        && !terminal_completion
+                        && result.is_err()
+                        && let Err(error) = root_recovery_clone.checkpoint_failed_execution(&task).await
+                    {
+                        warn!(
+                            target: "rustfs::heal::manager",
+                            event = EVENT_HEAL_SCHEDULER_STATE,
+                            component = LOG_COMPONENT_HEAL,
+                            subsystem = LOG_SUBSYSTEM_MANAGER,
+                            task_id,
+                            state = "root_recovery_checkpoint_failed",
+                            error = %error,
+                            "Failed to checkpoint root heal recovery execution budget"
+                        );
+                    }
                     let completed_status_for_verified_events = completed_status_entry.clone();
                     // Keep retry ownership continuous: status snapshots acquire
                     // these locks in the same active -> retrying order.

crates/heal/src/heal/manager/tests.rs

@@ -26,6 +26,7 @@ use rustfs_madmin::heal_commands::HealResultItem;
 use std::sync::Mutex as StdMutex;
 use tempfile::TempDir;
 
+mod root_recovery;
 mod running_mainline;
 
 use super::super::{DiskOption, DiskStore, Endpoint, new_disk, storage_api::status::BucketInfo};
@@ -94,6 +95,7 @@ async fn process_manager_queue_once(manager: &HealManager) {
         retrying_heals: &manager.retrying_heals,
         mrf_repair_notice_targets: &manager.mrf_repair_notice_targets,
         replacement_recovery_anchors: &manager.replacement_recovery_anchors,
+        root_recovery: &manager.root_recovery,
         config: &manager.config,
         statistics: &manager.statistics,
         storage: &manager.storage,
@@ -149,7 +151,7 @@ fn completed_retention_cursor_boundaries_preserve_progress() {
 
 #[tokio::test]
 async fn completed_retention_displaced_alias_does_not_resurrect_evicted_snapshot() {
-    let manager = HealManager::new(Arc::new(MockStorage), None);
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
     let request = HealRequest::bucket("bucket".to_string());
     manager.insert_task_alias("alias", &request.id).await;
     let terminal = record_displaced_terminal(&manager.displaced_terminals, &request);
@@ -211,7 +213,7 @@ async fn completed_retention_clock_rollback_preserves_terminal_alias_queries() {
         },
         HealTaskStatus::Cancelled,
     ] {
-        let manager = HealManager::new(Arc::new(MockStorage), None);
+        let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
         let mut snapshot = completed_retention_fixture(completed_at);
         snapshot.status = status.clone();
         let expected_progress = snapshot.progress.clone();
@@ -365,7 +367,7 @@ async fn canonical_outcome_cancel_wins_before_worker_finalizes_success() {
     use crate::heal::outcome::{HealAbortReason, HealExecutionOutcome};
     use crate::heal::task::{OUTCOME_FINISH_TEST_HOOK, OutcomeFinishTestHook};
     let bucket = "canonical-outcome-cancel-before-finish";
-    let manager = HealManager::new(Arc::new(MockStorage), None);
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
     let request = HealRequest::object(bucket.to_string(), "object".to_string(), None);
     let task_id = request.id.clone();
     let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None);
@@ -416,7 +418,7 @@ async fn canonical_outcome_cancel_wins_before_worker_finalizes_success() {
 #[tokio::test]
 async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() {
     let bucket = "completed-retention-retry-cancel";
-    let manager = HealManager::new(Arc::new(MockStorage), None);
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
     let request = HealRequest::object(bucket.to_string(), "object".to_string(), None);
     let task_id = request.id.clone();
     let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None);
@@ -471,7 +473,7 @@ async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_han
     for outcome in ["success", "failed", "cancelled"] {
         let bucket = format!("completed-retention-{outcome}");
         let hook = Arc::new(CompletedRetentionHook::default());
-        let manager = Arc::new(HealManager::new(Arc::new(MockStorage), None));
+        let manager = Arc::new(HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None));
         let request = HealRequest::object(bucket.clone(), "object".to_string(), None);
         let task_id = request.id.clone();
         let duplicate = HealRequest::object(bucket.clone(), "object".to_string(), None);
@@ -711,7 +713,7 @@ impl HealStorageAPI for MockStorage {
 async fn assert_heal_start_retry_control_preserves_real_executor_progress(cancel: bool) {
     for phase in ["listing", "object"] {
         let bucket = format!("heal-start-retry-deadline-{phase}-{cancel}");
-        let manager = HealManager::new(Arc::new(MockStorage), None);
+        let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
         let mut request = HealRequest::new(
             HealType::Prefix {
                 bucket: bucket.clone(),
@@ -822,7 +824,7 @@ async fn heal_start_retry_cancellation_preserves_real_executor_progress() {
 
 #[tokio::test]
 async fn heal_start_retry_scheduler_carries_explicit_budget_and_identity() {
-    let manager = HealManager::new(
+    let manager = HealManager::new_without_root_recovery_for_test(
         Arc::new(MockStorage),
         Some(HealConfig {
             task_timeout: Duration::ZERO,
@@ -863,6 +865,88 @@ async fn heal_start_retry_scheduler_carries_explicit_budget_and_identity() {
     COMPLETED_RETENTION_HOOKS.lock().await.remove(&task_id);
 }
 
+#[tokio::test]
+async fn heal_start_retry_scheduler_carries_explicit_budget_across_retries() {
+    let manager = HealManager::new_without_root_recovery_for_test(
+        Arc::new(MockStorage),
+        Some(HealConfig {
+            task_timeout: Duration::ZERO,
+            event_driven_scheduler_enable: false,
+            ..Default::default()
+        }),
+    );
+    let mut request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None);
+    request.source = HealRequestSource::Admin;
+    request.options.timeout = Some(Duration::from_secs(60));
+    let task_id = request.id.clone();
+    let created_at = request.created_at;
+    manager
+        .submit_heal_request(request)
+        .await
+        .expect("admit explicit-budget task");
+
+    let mut previous_remaining = Duration::from_secs(60);
+    for expected_attempt in 1..=3 {
+        process_manager_queue_once(&manager).await;
+        tokio::time::timeout(Duration::from_secs(1), async {
+            loop {
+                if manager.retrying_heals.lock().await.contains_key(&task_id) {
+                    break;
+                }
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("recoverable executor failure should enter retry backoff");
+
+        let retry = manager.retrying_heals.lock().await[&task_id].request.clone();
+        assert_eq!(retry.id, task_id);
+        assert_eq!(retry.created_at, created_at);
+        assert_eq!(retry.source, HealRequestSource::Admin);
+        assert_eq!(retry.retry_attempts, expected_attempt);
+        let remaining = retry.options.timeout.expect("retry retains explicit budget");
+        assert!(
+            remaining > Duration::ZERO && remaining < previous_remaining,
+            "retry attempt {expected_attempt} should carry only the unused explicit budget"
+        );
+        previous_remaining = remaining;
+        assert_eq!(manager.operations_snapshot().await.queue_length, 0);
+        assert_eq!(manager.operations_snapshot().await.retrying_tasks, 1);
+
+        let retry_delay = recoverable_heal_retry_delay(expected_attempt);
+        tokio::time::timeout(retry_delay + Duration::from_secs(1), async {
+            loop {
+                if !manager.retrying_heals.lock().await.contains_key(&task_id) {
+                    break;
+                }
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("retry backoff should requeue the same task id");
+        assert_eq!(manager.operations_snapshot().await.queue_length, 1);
+    }
+
+    process_manager_queue_once(&manager).await;
+    let final_error = tokio::time::timeout(Duration::from_secs(5), async {
+        loop {
+            if let Ok(HealTaskStatus::Failed { error }) = manager.get_task_status(&task_id).await {
+                break error;
+            }
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("retry limit should finish as a terminal failure");
+    let final_error_lower = final_error.to_ascii_lowercase();
+    assert!(
+        final_error_lower.contains("insufficient") && final_error_lower.contains("read"),
+        "terminal failure should retain the recoverable storage error: {final_error}"
+    );
+    assert_eq!(manager.operations_snapshot().await.queue_length, 0);
+    assert_eq!(manager.operations_snapshot().await.retrying_tasks, 0);
+}
+
 struct ManagerRecoveryTestHook {
     replacement_resume_disk: DiskStore,
     listed: StdMutex<bool>,
@@ -962,7 +1046,7 @@ fn scoped_object_request(bucket: &str, object: &str, pool_index: usize, set_inde
 
 #[tokio::test]
 async fn scheduler_bulkhead_starts_other_sets_and_retains_same_set_tail() {
-    let manager = HealManager::new(Arc::new(MockStorage), None);
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
     {
         let mut config = manager.config.write().await;
         config.max_concurrent_heals = 2;
@@ -1867,7 +1951,7 @@ fn test_heal_request_and_task_metric_labels_match() {
 #[tokio::test]
 async fn test_submit_heal_request_returns_merged_for_duplicate() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     let request = HealRequest::new(
         HealType::Object {
@@ -1898,7 +1982,7 @@ async fn test_submit_heal_request_returns_merged_for_duplicate() {
 #[tokio::test]
 async fn test_admin_duplicate_receipt_returns_canonical_task_without_alias() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
     let mut original = HealRequest::object("bucket".to_string(), "object".to_string(), None);
     original.source = HealRequestSource::Admin;
     let original_id = original.id.clone();
@@ -1925,7 +2009,7 @@ async fn test_admin_duplicate_receipt_returns_canonical_task_without_alias() {
 #[tokio::test]
 async fn test_task_alias_is_removed_after_terminal_completion() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
     let original = HealRequest::object("bucket".to_string(), "object".to_string(), None);
     let original_id = original.id.clone();
     let duplicate = HealRequest::object("bucket".to_string(), "object".to_string(), None);
@@ -1965,7 +2049,7 @@ async fn test_task_alias_is_removed_after_terminal_completion() {
 #[tokio::test]
 async fn test_duplicate_admission_is_atomic_with_queue_to_active_transition() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = Arc::new(HealManager::new(storage.clone(), None));
+    let manager = Arc::new(HealManager::new_without_root_recovery_for_test(storage.clone(), None));
     let mut original = HealRequest::object("bucket".to_string(), "object".to_string(), None);
     original.source = HealRequestSource::Admin;
     let original_id = original.id.clone();
@@ -2035,7 +2119,7 @@ async fn test_duplicate_admission_is_atomic_with_queue_to_active_transition() {
 #[tokio::test]
 async fn test_submit_heal_request_returns_merged_for_active_duplicate() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage.clone(), None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage.clone(), None);
     let active_request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
     let active_task = Arc::new(HealTask::from_request(active_request, storage));
     manager.active_heals.lock().await.insert(active_task.id.clone(), active_task);
@@ -2055,7 +2139,7 @@ async fn test_submit_heal_request_returns_merged_for_active_duplicate() {
 #[tokio::test]
 async fn test_active_duplicate_token_can_query_and_cancel_original_task() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage.clone(), None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage.clone(), None);
     let active_request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
     let active_task = Arc::new(HealTask::from_request(active_request, storage));
     let active_task_id = active_task.id.clone();
@@ -2097,7 +2181,7 @@ async fn test_active_duplicate_token_can_query_and_cancel_original_task() {
 #[tokio::test]
 async fn test_queued_duplicate_token_can_query_and_cancel_original_request() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
     let original_request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
     let original_task_id = original_request.id.clone();
     let duplicate_request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
@@ -2288,7 +2372,7 @@ fn durable_replacement_recovery_re_admits_only_the_matching_generation() {
 
 #[test]
 fn replacement_recovery_blocker_is_set_scoped() {
-    let manager = HealManager::new(Arc::new(MockStorage), None);
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
 
     manager.block_replacement_recovery_set("pool_0_set_0");
 
@@ -2394,7 +2478,7 @@ async fn scheduler_completes_cleanup_pending_recovery_from_manager_anchor() {
 
     let (hook, _hook_guard) = ManagerRecoveryTestHook::install(anchor.clone());
     let storage = Arc::new(MockStorage);
-    let manager = HealManager::new(storage.clone(), None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage.clone(), None);
     let mut request = HealRequest::new(
         HealType::ErasureSet {
             buckets: vec!["bucket-a".to_string()],
@@ -2529,7 +2613,7 @@ fn test_retry_request_for_recoverable_error_stops_at_limit() {
 }
 
 #[tokio::test]
-async fn test_retry_request_does_not_rescan_batch_after_object_retries_exhausted() {
+async fn test_retry_request_rescans_batch_when_all_exhausted_objects_are_retryable() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
     let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), storage);
     let result = Err(task
@@ -2543,7 +2627,32 @@ async fn test_retry_request_does_not_rescan_batch_after_object_retries_exhausted
         })
         .await);
 
-    assert!(retry_request_for_result(&task, &result).is_none());
+    let (retry_request, retry_delay, error) = retry_request_for_result_with_budget(&task, &result)
+        .await
+        .expect("all-retryable batch failure should rescan within the manager retry budget");
+
+    assert_eq!(retry_request.id, task.id);
+    assert_eq!(retry_request.retry_attempts, 1);
+    assert!(retry_delay > Duration::ZERO);
+    assert!(error.contains("Lock acquisition timeout"));
+}
+
+#[tokio::test]
+async fn test_retry_request_does_not_rescan_batch_with_permanent_failures() {
+    let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+    let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), storage);
+    let result = Err(task
+        .record_batch_failure(BatchHealFailure {
+            scope: "bucket:bucket".to_string(),
+            failed: 2,
+            retryable: 1,
+            permanent: 1,
+            first_object: "object-a".to_string(),
+            first_error: "Lock acquisition timeout".to_string(),
+        })
+        .await);
+
+    assert!(retry_request_for_result_with_budget(&task, &result).await.is_none());
 }
 
 #[test]
@@ -2605,7 +2714,7 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) ->
 #[tokio::test]
 async fn test_cancel_task_cancels_retrying_backoff() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
     let mut request = HealRequest::bucket("bucket".to_string());
     request.retry_attempts = 1;
     let task_id = request.id.clone();
@@ -2632,7 +2741,7 @@ async fn test_cancel_task_cancels_retrying_backoff() {
 #[tokio::test]
 async fn test_cancel_tasks_for_path_cancels_retrying_backoff() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
     let mut request = HealRequest::bucket("bucket".to_string());
     request.retry_attempts = 1;
     let task_id = request.id.clone();
@@ -2654,7 +2763,7 @@ async fn test_cancel_tasks_for_path_cancels_retrying_backoff() {
 #[tokio::test]
 async fn test_cancel_tasks_for_empty_path_cancels_queued_cluster_only() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     let cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High);
     let cluster_request_id = cluster_request.id.clone();
@@ -2693,7 +2802,7 @@ async fn test_cancel_tasks_for_empty_path_cancels_queued_cluster_only() {
 #[tokio::test]
 async fn test_cancel_tasks_for_empty_path_cancels_active_cluster_only() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage.clone(), None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage.clone(), None);
 
     let cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High);
     let cluster_request_id = cluster_request.id.clone();
@@ -2724,7 +2833,7 @@ async fn test_cancel_tasks_for_empty_path_cancels_active_cluster_only() {
 #[tokio::test]
 async fn test_cancel_tasks_for_empty_path_cancels_retrying_cluster_only() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     let mut cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High);
     cluster_request.retry_attempts = 1;
@@ -2763,7 +2872,7 @@ fn test_heal_type_matches_path_accepts_legacy_root() {
 #[tokio::test]
 async fn test_retrying_duplicate_token_can_query_and_cancel_original_retry() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
     let mut original_request = HealRequest::bucket("bucket".to_string());
     original_request.retry_attempts = 1;
     let original_task_id = original_request.id.clone();
@@ -2799,7 +2908,7 @@ async fn test_retrying_duplicate_token_can_query_and_cancel_original_retry() {
 #[tokio::test]
 async fn test_get_task_status_reports_pending_for_queued_request() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     let request = HealRequest::bucket("bucket".to_string());
     let request_id = request.id.clone();
@@ -2823,7 +2932,7 @@ async fn test_get_task_status_reports_pending_for_queued_request() {
 #[tokio::test]
 async fn test_operations_snapshot_counts_queue_by_source_and_priority() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     let mut scanner_request = HealRequest::new(
         HealType::Object {
@@ -2880,7 +2989,7 @@ async fn test_operations_snapshot_counts_queue_by_source_and_priority() {
 // HS-06 (backlog#1870): overlap policy + forceStart semantics.
 fn manager_with_policy(policy: HealOverlapPolicy) -> HealManager {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    HealManager::new(
+    HealManager::new_without_root_recovery_for_test(
         storage,
         Some(HealConfig {
             overlap_policy: policy,
@@ -2902,6 +3011,12 @@ fn admin_prefix_request(bucket: &str, prefix: &str) -> HealRequest {
     request
 }
 
+fn internal_prefix_request(bucket: &str, prefix: &str) -> HealRequest {
+    let mut request = admin_prefix_request(bucket, prefix);
+    request.source = HealRequestSource::Internal;
+    request
+}
+
 async fn insert_active_task(manager: &HealManager, request: HealRequest) -> String {
     let task = Arc::new(HealTask::from_request(request, manager.storage.clone()));
     let task_id = task.id.clone();
@@ -3010,7 +3125,7 @@ async fn admin_force_start_cancels_overlapping_active_task_first() {
 #[tokio::test]
 async fn admission_snapshot_tracks_start_duplicate_force_start_and_displacement() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = Arc::new(HealManager::new(
+    let manager = Arc::new(HealManager::new_without_root_recovery_for_test(
         storage,
         Some(HealConfig {
             queue_size: 1,
@@ -3018,7 +3133,7 @@ async fn admission_snapshot_tracks_start_duplicate_force_start_and_displacement(
         }),
     ));
 
-    let mut paused = admin_prefix_request("bucket-a", "logs/");
+    let mut paused = internal_prefix_request("bucket-a", "logs/");
     paused.priority = HealPriority::Low;
     let hook = Arc::new(DuplicateAdmissionTestHook {
         request_id: paused.id.clone(),
@@ -3048,7 +3163,7 @@ async fn admission_snapshot_tracks_start_duplicate_force_start_and_displacement(
     );
     *DUPLICATE_ADMISSION_TEST_HOOK.lock().await = None;
 
-    let duplicate = admin_prefix_request("bucket-a", "logs/");
+    let duplicate = internal_prefix_request("bucket-a", "logs/");
     let duplicate_receipt = manager
         .submit_heal_request_with_receipt(duplicate)
         .await
@@ -3092,7 +3207,7 @@ async fn admission_snapshot_tracks_start_duplicate_force_start_and_displacement(
 #[tokio::test]
 async fn test_operations_snapshot_counts_active_by_source_and_priority() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     let mut request = HealRequest::bucket("bucket-a".to_string());
     request.priority = HealPriority::High;
@@ -3114,7 +3229,7 @@ async fn test_operations_snapshot_counts_active_by_source_and_priority() {
 #[tokio::test]
 async fn test_operations_snapshot_counts_retry_backoff_as_owned_work() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
     let mut request = HealRequest::bucket("bucket-retry".to_string());
     request.priority = HealPriority::Urgent;
     request.source = HealRequestSource::Admin;
@@ -3140,7 +3255,7 @@ async fn test_operations_snapshot_counts_retry_backoff_as_owned_work() {
 #[tokio::test]
 async fn test_scheduler_retry_transitions_keep_continuous_single_ownership() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = Arc::new(HealManager::new(storage, None));
+    let manager = Arc::new(HealManager::new_without_root_recovery_for_test(storage, None));
     {
         let mut config = manager.config.write().await;
         config.enable_auto_heal = false;
@@ -3213,7 +3328,7 @@ async fn test_scheduler_retry_transitions_keep_continuous_single_ownership() {
 #[tokio::test]
 async fn test_active_progress_snapshot_sums_active_task_progress() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     let first = Arc::new(HealTask::from_request(
         HealRequest::bucket("bucket-a".to_string()),
@@ -3258,7 +3373,7 @@ async fn test_active_progress_snapshot_sums_active_task_progress() {
 #[tokio::test]
 async fn test_get_task_status_for_path_rejects_wrong_token_when_path_is_active() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     manager
         .submit_heal_request(HealRequest::bucket("bucket".to_string()))
@@ -3274,7 +3389,7 @@ async fn test_get_task_status_for_path_rejects_wrong_token_when_path_is_active()
 #[tokio::test]
 async fn test_get_task_status_for_path_rejects_token_from_other_active_path() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     let bucket_request = HealRequest::bucket("bucket".to_string());
     let other_request = HealRequest::bucket("other".to_string());
@@ -3298,7 +3413,7 @@ async fn test_get_task_status_for_path_rejects_token_from_other_active_path() {
 #[tokio::test]
 async fn test_get_task_status_for_path_does_not_accept_token_from_inactive_path() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     let request = HealRequest::bucket("bucket".to_string());
     let request_id = request.id.clone();
@@ -3317,7 +3432,7 @@ async fn test_get_task_status_for_path_does_not_accept_token_from_inactive_path(
 #[tokio::test]
 async fn test_get_task_status_for_path_returns_not_found_when_path_is_inactive() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     assert!(matches!(
         manager.get_task_status_for_path("bucket", "old-token").await,
@@ -3328,7 +3443,7 @@ async fn test_get_task_status_for_path_returns_not_found_when_path_is_inactive()
 #[tokio::test]
 async fn test_get_task_status_for_empty_path_does_not_match_unrelated_tasks() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     let request = HealRequest::bucket("bucket".to_string());
     let request_id = request.id.clone();
@@ -3351,7 +3466,7 @@ async fn test_get_task_status_for_empty_path_does_not_match_unrelated_tasks() {
 #[tokio::test]
 async fn test_get_task_report_queries_queued_task_by_token_without_path() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     let request = HealRequest::new(
         HealType::ErasureSet {
@@ -3380,7 +3495,7 @@ async fn test_get_task_report_queries_queued_task_by_token_without_path() {
 #[tokio::test]
 async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     // A completed entry recorded in a Retrying state for a task whose
     // request is also (still) queued under the same id: the retrying
@@ -3424,7 +3539,7 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
 #[tokio::test]
 async fn test_get_task_status_reads_recent_completed_status() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     manager.completed_heals.lock().await.insert(
         "completed-token".to_string(),
@@ -3456,7 +3571,7 @@ async fn test_get_task_status_reads_recent_completed_status() {
 #[tokio::test]
 async fn test_get_task_report_for_path_reads_completed_items() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     manager.completed_heals.lock().await.insert(
         "completed-token".to_string(),
@@ -3504,7 +3619,7 @@ async fn test_get_task_report_for_path_reads_completed_items() {
 #[tokio::test]
 async fn test_get_task_report_for_empty_path_does_not_match_unrelated_tasks() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     manager
         .submit_heal_request(HealRequest::bucket("bucket".to_string()))
@@ -3520,7 +3635,7 @@ async fn test_get_task_report_for_empty_path_does_not_match_unrelated_tasks() {
 #[tokio::test]
 async fn test_cancel_task_removes_queued_request() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     let request = HealRequest::bucket("bucket".to_string());
     let request_id = request.id.clone();
@@ -3544,7 +3659,7 @@ async fn mrf_ownership_unverified_completion_does_not_emit_repaired() {
     let version_id = Some([9u8; 16]);
     let _ = rustfs_common::mrf_channel::take_mrf_repaired_events_for(bucket);
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     let mut request = HealRequest::object(bucket.to_string(), object.to_string(), None);
     request.source = HealRequestSource::Mrf;
@@ -3591,7 +3706,7 @@ async fn mrf_ownership_dry_run_and_empty_window_do_not_emit_repaired() {
         } else {
             "mrf-dry-run-outcome"
         };
-        let manager = HealManager::new(Arc::new(MockStorage), None);
+        let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
         let request = HealRequest::new(
             if empty_window {
                 HealType::Cluster
@@ -3647,7 +3762,7 @@ async fn mrf_ownership_queued_cancel_does_not_emit_repaired() {
     let object = "object";
     let _ = rustfs_common::mrf_channel::take_mrf_repaired_events_for(bucket);
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     let mut request = HealRequest::object(bucket.to_string(), object.to_string(), None);
     request.source = HealRequestSource::Mrf;
@@ -3678,7 +3793,7 @@ async fn mrf_ownership_queued_cancel_does_not_emit_repaired() {
 #[tokio::test]
 async fn test_cancel_tasks_for_path_removes_matching_queued_requests() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     let bucket_request = HealRequest::bucket("bucket".to_string());
     let bucket_request_id = bucket_request.id.clone();
@@ -3717,7 +3832,7 @@ async fn test_cancel_tasks_for_path_removes_matching_queued_requests() {
 #[tokio::test]
 async fn test_submit_heal_request_returns_merged_before_full_for_duplicate() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(
+    let manager = HealManager::new_without_root_recovery_for_test(
         storage,
         Some(HealConfig {
             queue_size: 1,
@@ -3754,7 +3869,7 @@ async fn test_submit_heal_request_returns_merged_before_full_for_duplicate() {
 #[tokio::test]
 async fn test_submit_heal_request_returns_dropped_for_low_priority_when_full() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(
+    let manager = HealManager::new_without_root_recovery_for_test(
         storage,
         Some(HealConfig {
             queue_size: 1,
@@ -3797,7 +3912,7 @@ async fn test_submit_heal_request_returns_dropped_for_low_priority_when_full() {
 #[tokio::test]
 async fn test_submit_heal_request_returns_full_for_normal_priority_when_full() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(
+    let manager = HealManager::new_without_root_recovery_for_test(
         storage,
         Some(HealConfig {
             queue_size: 1,
@@ -3839,7 +3954,7 @@ async fn test_submit_heal_request_returns_full_for_normal_priority_when_full() {
 #[tokio::test]
 async fn test_high_priority_request_displaces_lower_priority_when_queue_full() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(
+    let manager = HealManager::new_without_root_recovery_for_test(
         storage,
         Some(HealConfig {
             queue_size: 1,
@@ -3894,7 +4009,7 @@ async fn test_high_priority_request_displaces_lower_priority_when_queue_full() {
 
 #[tokio::test]
 async fn displaced_task_remains_queryable() {
-    let manager = HealManager::new(
+    let manager = HealManager::new_without_root_recovery_for_test(
         Arc::new(MockStorage),
         Some(HealConfig {
             queue_size: 1,
@@ -3936,7 +4051,7 @@ async fn displaced_task_remains_queryable() {
 
 #[tokio::test]
 async fn displaced_archive_failure_keeps_queryable_terminal() {
-    let manager = HealManager::new(Arc::new(MockStorage), None);
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
     let mut request = HealRequest::new(
         HealType::Bucket {
             bucket: "archive-failure".to_string(),
@@ -3959,7 +4074,7 @@ async fn displaced_archive_failure_keeps_queryable_terminal() {
 
 #[tokio::test]
 async fn scheduler_retry_displacement_keeps_evicted_task_queryable() {
-    let manager = Arc::new(HealManager::new(
+    let manager = Arc::new(HealManager::new_without_root_recovery_for_test(
         Arc::new(MockStorage),
         Some(HealConfig {
             queue_size: 1,
@@ -4024,7 +4139,7 @@ async fn scheduler_retry_displacement_keeps_evicted_task_queryable() {
 
 #[tokio::test]
 async fn concurrent_displacers_produce_one_terminal_generation() {
-    let manager = Arc::new(HealManager::new(
+    let manager = Arc::new(HealManager::new_without_root_recovery_for_test(
         Arc::new(MockStorage),
         Some(HealConfig {
             queue_size: 1,
@@ -4076,7 +4191,7 @@ async fn concurrent_displacers_produce_one_terminal_generation() {
 
 #[tokio::test]
 async fn successor_chain_is_bounded_and_authorized() {
-    let manager = HealManager::new(
+    let manager = HealManager::new_without_root_recovery_for_test(
         Arc::new(MockStorage),
         Some(HealConfig {
             queue_size: 1,
@@ -4129,7 +4244,7 @@ async fn successor_chain_is_bounded_and_authorized() {
 
 #[tokio::test]
 async fn displaced_terminal_expires_after_bounded_ttl() {
-    let manager = HealManager::new(Arc::new(MockStorage), None);
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
     let mut request = HealRequest::new(
         HealType::Bucket {
             bucket: "expires".to_string(),
@@ -4152,7 +4267,7 @@ async fn displaced_terminal_expires_after_bounded_ttl() {
 #[tokio::test]
 async fn test_displacing_registered_mrf_task_drops_notice_ownership() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(
+    let manager = HealManager::new_without_root_recovery_for_test(
         storage,
         Some(HealConfig {
             queue_size: 1,
@@ -4200,7 +4315,7 @@ async fn test_displacing_registered_mrf_task_drops_notice_ownership() {
 #[tokio::test]
 async fn test_submit_heal_request_drops_read_repair_under_pressure() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(
+    let manager = HealManager::new_without_root_recovery_for_test(
         storage,
         Some(HealConfig {
             queue_size: 10,
@@ -4234,7 +4349,7 @@ async fn test_submit_heal_request_drops_read_repair_under_pressure() {
 #[tokio::test]
 async fn test_submit_heal_request_drops_low_scanner_under_pressure() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(
+    let manager = HealManager::new_without_root_recovery_for_test(
         storage,
         Some(HealConfig {
             queue_size: 10,
@@ -4268,7 +4383,7 @@ async fn test_submit_heal_request_drops_low_scanner_under_pressure() {
 #[tokio::test]
 async fn test_submit_heal_request_accepts_admin_high_under_pressure() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(
+    let manager = HealManager::new_without_root_recovery_for_test(
         storage,
         Some(HealConfig {
             queue_size: 10,
@@ -4308,7 +4423,7 @@ async fn test_mainline_throttle_delays_background_heal_start() {
         limit: 10,
         state: AdmissionState::Open,
     });
-    let manager = HealManager::new_with_workload_provider(
+    let manager = HealManager::new_with_workload_provider_without_root_recovery_for_test(
         storage,
         Some(HealConfig {
             max_concurrent_heals: 1,
@@ -4341,7 +4456,7 @@ async fn test_mainline_throttle_delays_background_heal_start_under_write_pressur
         limit: 10,
         state: AdmissionState::Open,
     });
-    let manager = HealManager::new_with_workload_provider(
+    let manager = HealManager::new_with_workload_provider_without_root_recovery_for_test(
         storage,
         Some(HealConfig {
             max_concurrent_heals: 1,
@@ -4374,7 +4489,7 @@ async fn test_mainline_throttle_allows_admin_high_start() {
         limit: 10,
         state: AdmissionState::Saturated,
     });
-    let manager = HealManager::new_with_workload_provider(
+    let manager = HealManager::new_with_workload_provider_without_root_recovery_for_test(
         storage,
         Some(HealConfig {
             max_concurrent_heals: 1,
@@ -4400,7 +4515,7 @@ async fn test_mainline_throttle_allows_admin_high_start() {
 #[tokio::test]
 async fn configured_task_timeout_applies_only_when_request_timeout_is_absent() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(
+    let manager = HealManager::new_without_root_recovery_for_test(
         storage,
         Some(HealConfig {
             max_concurrent_heals: 1,
@@ -4454,7 +4569,7 @@ async fn configured_task_timeout_applies_only_when_request_timeout_is_absent() {
 #[tokio::test]
 async fn test_force_start_bypasses_duplicate_and_full_admission() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(
+    let manager = HealManager::new_without_root_recovery_for_test(
         storage,
         Some(HealConfig {
             queue_size: 1,
@@ -4513,7 +4628,7 @@ async fn test_force_start_bypasses_duplicate_and_full_admission() {
 #[tokio::test]
 async fn test_force_start_marks_dedup_key_for_future_duplicates() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(
+    let manager = HealManager::new_without_root_recovery_for_test(
         storage,
         Some(HealConfig {
             queue_size: 1,
@@ -4570,7 +4685,7 @@ async fn test_force_start_marks_dedup_key_for_future_duplicates() {
 #[tokio::test]
 async fn same_request_id_replay_reuses_existing_task_without_force_start_duplication() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new(storage, None);
+    let manager = HealManager::new_without_root_recovery_for_test(storage, None);
 
     let mut original = admin_prefix_request("bucket", "logs/");
     original.force_start = true;

crates/heal/src/heal/manager/tests/running_mainline.rs

@@ -141,7 +141,7 @@ async fn start_fixture(
         started: AtomicUsize::new(0),
         committed: AtomicUsize::new(0),
     });
-    let manager = HealManager::new_with_workload_provider(
+    let manager = HealManager::new_with_workload_provider_without_root_recovery_for_test(
         storage.clone(),
         Some(HealConfig {
             mainline_throttle_enable: pacing_enabled,

crates/heal/src/heal/mrf_queue.rs

@@ -1137,9 +1137,28 @@ fn tick_action(dirty: bool, depth: usize, journal_on_disk: bool, retain_replay_j
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::heal::manager::HealConfig;
+    use crate::heal::storage::{ECStoreHealStorage, HealStorageAPI};
+    use crate::heal::{DiskError, RUSTFS_META_BUCKET};
     use rustfs_common::mrf_channel::{MrfIntent, MrfKind, MrfVerifiedRepairDisposition, MrfVerifiedRepairEvent};
+    use serde_json::{Map, Value, json};
     use serial_test::serial;
+    use std::env;
+    use std::fs;
+    use std::io::Write as _;
+    use std::path::{Path, PathBuf};
     use std::sync::Arc as StdArc;
+    use std::time::{Duration as StdDuration, Instant};
+
+    const W13_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_EVIDENCE_DIR";
+    const W13_SOURCE_REVISION_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_SOURCE_REVISION";
+    const W13_SELECTION_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_SELECTION";
+    const W13_SOAK_SECONDS_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_SOAK_SECONDS";
+    const W13_ALLOW_SHORT_SOAK_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_ALLOW_SHORT_SOAK";
+    const W13_RUN_ID_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_RUN_ID";
+    const W13_WINDOW_ID_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_WINDOW_ID";
+    const W13_ENOSPC_ROOT_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_ENOSPC_ROOT";
+    const W13_ENOSPC_FILL_LIMIT_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_ENOSPC_FILL_LIMIT_BYTES";
 
     fn intent(bucket: &str, object: &str, attempts: u8) -> MrfIntent {
         MrfIntent {
@@ -1160,6 +1179,748 @@ mod tests {
         payload
     }
 
+    fn w13_timestamp() -> String {
+        chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true)
+    }
+
+    fn w13_selection_contains(selection: &str, lane: &str) -> bool {
+        selection == "all" || selection.split(',').any(|item| item.trim() == lane)
+    }
+
+    fn w13_evidence_path(root: &Path, gate: &str, field: &str) -> PathBuf {
+        let lane = match gate {
+            "G07" => "g07-mrf-responsibility",
+            "G08" => "g08-mrf-capacity",
+            "P4" => "p4-mrf-soak",
+            other => panic!("unsupported W13 evidence gate: {other}"),
+        };
+        root.join(lane).join(format!("{gate}-{field}.json"))
+    }
+
+    struct W13Evidence<'a> {
+        source_revision: &'a str,
+        run_id: &'a str,
+        window_id: &'a str,
+        started_at: &'a str,
+        finished_at: &'a str,
+        gate: &'a str,
+        field: &'a str,
+        artifact_kind: &'a str,
+        extra: Map<String, Value>,
+    }
+
+    fn write_w13_evidence(root: &Path, evidence: W13Evidence<'_>) {
+        let path = w13_evidence_path(root, evidence.gate, evidence.field);
+        fs::create_dir_all(path.parent().expect("W13 evidence artifact parent")).expect("create W13 evidence artifact directory");
+        let mut payload = Map::new();
+        payload.insert("schema".to_string(), json!(1));
+        payload.insert("evidence_type".to_string(), json!("measured"));
+        payload.insert("artifact_kind".to_string(), json!(evidence.artifact_kind));
+        payload.insert("source_revision".to_string(), json!(evidence.source_revision));
+        payload.insert("run_id".to_string(), json!(evidence.run_id));
+        payload.insert("measurement_window_id".to_string(), json!(evidence.window_id));
+        payload.insert("started_at".to_string(), json!(evidence.started_at));
+        payload.insert("finished_at".to_string(), json!(evidence.finished_at));
+        payload.insert("gate".to_string(), json!(evidence.gate));
+        payload.insert("field".to_string(), json!(evidence.field));
+        payload.insert(
+            "command".to_string(),
+            json!([
+                "cargo",
+                "test",
+                "--locked",
+                "-p",
+                "rustfs-heal",
+                "--lib",
+                "heal::mrf_queue::tests::w13_mrf_release_evidence_outputs_bundle_artifacts",
+                "--",
+                "--ignored",
+                "--exact",
+                "--nocapture"
+            ]),
+        );
+        payload.insert(
+            "summary".to_string(),
+            json!(format!("Measured W13 MRF evidence for {}.{}", evidence.gate, evidence.field)),
+        );
+        payload.extend(evidence.extra);
+        let bytes = serde_json::to_vec_pretty(&Value::Object(payload)).expect("serialize W13 evidence payload");
+        fs::write(&path, [bytes.as_slice(), b"\n"].concat()).expect("write W13 evidence artifact");
+    }
+
+    async fn w13_committed_replay_probe() -> (usize, bool, bool, bool, bool, usize) {
+        let env = rustfs_test_utils::TestECStoreEnv::builder()
+            .prefix("rustfs_mrf_w13_replay_evidence")
+            .build()
+            .await;
+        let bucket = "w13-replay-bucket";
+        let object = "w13-replay-object";
+        env.make_bucket(bucket, false).await;
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
+        let manager = Arc::new(HealManager::new(
+            storage.clone(),
+            Some(HealConfig {
+                queue_size: 2,
+                heal_interval: Duration::from_secs(3600),
+                enable_auto_heal: false,
+                ..Default::default()
+            }),
+        ));
+        let disks = journal_disks().await;
+        assert!(!disks.is_empty(), "W13 evidence requires real local MRF disks");
+
+        let config = MrfConsumerConfig::default();
+        let replay_owner = Uuid::new_v4();
+        let mut replay_intent = intent(bucket, object, 0);
+        replay_intent.kind = MrfKind::PartialWrite;
+        replay_intent.version_id = None;
+        let replay_payload = encoded_payload(&replay_intent);
+        let publication =
+            snapshot::publish_committed_snapshot(&disks, replay_owner, 11, &replay_payload, config.journal_max_bytes)
+                .await
+                .expect("publish W13 committed replay checkpoint");
+        assert_eq!(publication.manifest_replicas, disks.len(), "all W13 checkpoint manifests should commit");
+
+        let mut queue = MrfQueue::new(config.queue_capacity, config.journal_max_bytes);
+        let mut backoff_until = None;
+        let replay = replay_into(&manager, &mut queue, &mut backoff_until).await;
+        assert_eq!(replay.replayed, 1, "W13 committed checkpoint must replay one record");
+        assert_eq!(queue.depth(), 0, "W13 replayed record should reach the manager before cleanup");
+        assert_eq!(replay.durable_replay_anchors.len(), 1, "W13 replay must create a proof anchor");
+        assert_eq!(
+            manager.operations_snapshot().await.queued_by_source.mrf,
+            1,
+            "W13 replayed work must be visible as MRF manager work"
+        );
+
+        let anchor = replay.durable_replay_anchors[0].clone();
+        let mut runtime = MrfRuntime {
+            queue,
+            config,
+            checkpoint_owner: Uuid::new_v4(),
+            next_checkpoint_sequence: replay.next_checkpoint_sequence,
+            new_since_flush: 0,
+            dirty: false,
+            journal_on_disk: replay.journal_on_disk,
+            retain_replay_journal: replay.retain_journal_for_replay,
+            durable_replay_anchors: replay.durable_replay_anchors,
+            replay_cleanup: replay.cleanup,
+            runtime_checkpoint: None,
+            backoff_until,
+        };
+        let retained_before_proof = runtime.retained_replay_journal();
+        assert!(retained_before_proof, "W13 proof anchor must retain replay checkpoint before proof");
+        assert!(
+            snapshot::inspect_local_committed_snapshot(runtime.config.journal_max_bytes)
+                .await
+                .expect("inspect W13 retained checkpoint")
+                .is_some(),
+            "W13 replay checkpoint must remain durable before proof"
+        );
+
+        rustfs_common::mrf_channel::note_mrf_verified_repair(MrfVerifiedRepairEvent {
+            kind: anchor.kind,
+            bucket: anchor.bucket.clone(),
+            object: anchor.object.clone(),
+            version_id: anchor.version_id,
+            scope: anchor.scope,
+            lease: Some(anchor.lease),
+            bucket_incarnation_id: anchor.bucket_incarnation_id,
+            disposition: MrfVerifiedRepairDisposition::Repaired,
+        });
+        runtime.discharge_durable_replay_anchors();
+        let proof_discharged_anchor = !runtime.retained_replay_journal();
+        assert!(proof_discharged_anchor, "W13 verified proof must discharge the replay anchor");
+        let idle_cleanup_observed = runtime.delete_idle_recovery_anchors().await;
+        assert!(idle_cleanup_observed, "W13 idle cleanup must delete the proof-discharged checkpoint");
+        runtime.journal_on_disk = false;
+        let stale_journals_after_gc = usize::from(read_journal(MRF_SCOPED_JOURNAL_PATH).await.is_some())
+            + usize::from(read_journal(MRF_JOURNAL_PATH).await.is_some())
+            + usize::from(
+                snapshot::inspect_local_committed_snapshot(runtime.config.journal_max_bytes)
+                    .await
+                    .expect("inspect W13 checkpoints after cleanup")
+                    .is_some(),
+            );
+
+        let restart_manager = Arc::new(HealManager::new(
+            storage,
+            Some(HealConfig {
+                queue_size: 2,
+                heal_interval: Duration::from_secs(3600),
+                enable_auto_heal: false,
+                ..Default::default()
+            }),
+        ));
+        assert_eq!(
+            replay_journal_once(&restart_manager).await,
+            0,
+            "W13 cleaned anchors must not resurrect on restart"
+        );
+        assert_eq!(
+            restart_manager.operations_snapshot().await.queued_by_source.mrf,
+            0,
+            "W13 restart must not re-admit proof-cleaned MRF work"
+        );
+        manager.stop().await.expect("stop W13 replay manager");
+        restart_manager.stop().await.expect("stop W13 restart manager");
+        (
+            replay.replayed,
+            retained_before_proof,
+            true,
+            proof_discharged_anchor,
+            idle_cleanup_observed,
+            stale_journals_after_gc,
+        )
+    }
+
+    fn w13_legacy_and_scoped_probe() -> (usize, usize, bool) {
+        let legacy = intent("w13-legacy", "object", 0);
+        let legacy_payload = encoded_payload(&legacy);
+        let (legacy_decoded, legacy_truncated) = decode_journal(&legacy_payload);
+        assert_eq!(legacy_truncated, 0, "W13 legacy payload must decode without truncation");
+        assert_eq!(legacy_decoded.len(), 1, "W13 legacy replay identity must round trip");
+        assert_eq!(legacy_decoded[0].bucket, legacy.bucket);
+        assert_eq!(legacy_decoded[0].object, legacy.object);
+        assert_eq!(legacy_decoded[0].version_id, legacy.version_id);
+        assert_eq!(legacy_decoded[0].scope, legacy.scope);
+
+        let mut scoped = intent("w13-scoped", "object", 0);
+        scoped.kind = MrfKind::PartialWrite;
+        scoped.version_id = Some(*Uuid::new_v4().as_bytes());
+        scoped.scope = Some(rustfs_common::mrf_channel::MrfScope {
+            pool_index: 7,
+            set_index: 13,
+        });
+        let mut runtime = MrfRuntime {
+            queue: MrfQueue::new(4, usize::MAX),
+            config: MrfConsumerConfig::default(),
+            checkpoint_owner: Uuid::new_v4(),
+            next_checkpoint_sequence: 1,
+            new_since_flush: 0,
+            dirty: true,
+            journal_on_disk: false,
+            retain_replay_journal: false,
+            durable_replay_anchors: Vec::new(),
+            replay_cleanup: None,
+            runtime_checkpoint: None,
+            backoff_until: None,
+        };
+        assert_eq!(runtime.queue.try_push_typed(scoped.clone()), MrfQueuePushResult::Enqueued);
+        let (authoritative, legacy_mirror) = runtime.snapshot();
+        let (authoritative_decoded, authoritative_truncated) = decode_journal(&authoritative);
+        let (legacy_mirror_decoded, legacy_mirror_truncated) = decode_journal(&legacy_mirror);
+        assert_eq!(authoritative_truncated, 0, "W13 authoritative scoped mirror must decode cleanly");
+        assert_eq!(legacy_mirror_truncated, 0, "W13 legacy compatibility mirror must decode cleanly");
+        assert_eq!(authoritative_decoded.len(), 1, "W13 authoritative mirror must retain scoped identity");
+        assert_eq!(authoritative_decoded[0].bucket, scoped.bucket);
+        assert_eq!(authoritative_decoded[0].object, scoped.object);
+        assert_eq!(authoritative_decoded[0].version_id, scoped.version_id);
+        assert_eq!(authoritative_decoded[0].scope, scoped.scope);
+        assert!(
+            legacy_mirror_decoded.is_empty() || legacy_mirror_decoded.iter().all(|intent| intent.scope.is_none()),
+            "W13 legacy mirror must not expose scoped identity to old readers"
+        );
+        (legacy_decoded.len(), authoritative_decoded.len(), legacy_mirror_decoded.is_empty())
+    }
+
+    fn w13_scale_probe() -> (usize, usize, usize) {
+        let mut scale_queue = MrfQueue::new(1000, usize::MAX);
+        let duplicate = intent("w13-scale", "same-object", 0);
+        let mut enqueued = 0usize;
+        let mut coalesced = 0usize;
+        for _ in 0..1000 {
+            match scale_queue.try_push_typed(duplicate.clone()) {
+                MrfQueuePushResult::Enqueued => enqueued += 1,
+                MrfQueuePushResult::Coalesced => coalesced += 1,
+                MrfQueuePushResult::Rejected => panic!("W13 scale duplicate probe should not reject"),
+            }
+        }
+        assert_eq!(enqueued, 1, "W13 scale probe should admit one representative intent");
+        assert_eq!(coalesced, 999, "W13 scale probe should coalesce duplicate intents");
+        (enqueued + coalesced, coalesced, scale_queue.depth())
+    }
+
+    fn w13_enospc_raw_os(err: &std::io::Error) -> bool {
+        err.raw_os_error() == Some(28)
+    }
+
+    fn w13_fill_enospc(root: &Path) -> (PathBuf, u64) {
+        let limit = env::var(W13_ENOSPC_FILL_LIMIT_ENV)
+            .ok()
+            .map(|raw| raw.parse::<u64>().expect("W13 ENOSPC fill limit must be an integer"))
+            .unwrap_or(128 * 1024 * 1024);
+        fs::create_dir_all(root).expect("create W13 ENOSPC root");
+        let filler = root.join(format!("w13-enospc-{}.fill", Uuid::new_v4()));
+        let mut file = fs::OpenOptions::new()
+            .create_new(true)
+            .write(true)
+            .open(&filler)
+            .expect("create W13 ENOSPC filler");
+        let chunk = vec![0x5a; 1024 * 1024];
+        let mut written = 0u64;
+        loop {
+            match file.write_all(&chunk) {
+                Ok(()) => {
+                    written = written.saturating_add(chunk.len() as u64);
+                    assert!(
+                        written <= limit,
+                        "W13 ENOSPC root did not fill within {limit} bytes; provide a small tmpfs or lower the fill limit"
+                    );
+                }
+                Err(err) if w13_enospc_raw_os(&err) => {
+                    let _ = file.sync_all();
+                    return (filler, written);
+                }
+                Err(err) => panic!("W13 ENOSPC filler failed with non-ENOSPC error: {err}"),
+            }
+        }
+    }
+
+    fn w13_snapshot_error_is_capacity(error: &snapshot::SnapshotError) -> bool {
+        match error {
+            snapshot::SnapshotError::Disk(source) => format!("{source:?}").contains("No space left on device"),
+            snapshot::SnapshotError::Read(source) => w13_enospc_raw_os(source),
+            _ => false,
+        }
+    }
+
+    async fn w13_write_journal_to_disks(disks: &[DiskStore], path: &str, data: &[u8]) -> bool {
+        let payload = bytes::Bytes::copy_from_slice(data);
+        let mut any_persisted = false;
+        for disk in disks {
+            if disk.write_all(RUSTFS_META_BUCKET, path, payload.clone()).await.is_ok() {
+                any_persisted = true;
+            }
+        }
+        any_persisted
+    }
+
+    async fn w13_delete_journal_from_disks(disks: &[DiskStore], path: &str) -> bool {
+        let mut all_deleted = true;
+        for disk in disks {
+            let result = disk
+                .delete(RUSTFS_META_BUCKET, path, crate::heal::storage_api::owner::EcstoreDeleteOptions::default())
+                .await;
+            if let Err(err) = result
+                && !matches!(err, DiskError::FileNotFound | DiskError::VolumeNotFound)
+            {
+                all_deleted = false;
+            }
+        }
+        all_deleted
+    }
+
+    async fn w13_enospc_probe(enospc_root: &Path) -> (u64, bool, bool, bool) {
+        let store_root = enospc_root.join(format!("store-{}", Uuid::new_v4()));
+        let _env = rustfs_test_utils::TestECStoreEnv::builder()
+            .disk_count(1)
+            .base_dir(&store_root)
+            .build()
+            .await;
+        let disks = journal_disks().await;
+        assert_eq!(disks.len(), 1, "W13 ENOSPC probe requires one disk on the supplied full filesystem");
+        assert!(
+            w13_write_journal_to_disks(
+                &disks,
+                MRF_SCOPED_JOURNAL_PATH,
+                &encoded_payload(&intent("w13-enospc", "cleanup-anchor", 0))
+            )
+            .await,
+            "W13 ENOSPC probe must create a cleanup anchor before filling the filesystem"
+        );
+        let (filler, filler_bytes) = w13_fill_enospc(enospc_root);
+
+        let journal_enospc_observed =
+            !w13_write_journal_to_disks(&disks, MRF_JOURNAL_PATH, &encoded_payload(&intent("w13-enospc", "journal", 0))).await;
+
+        let checkpoint = snapshot::publish_committed_snapshot(
+            &disks,
+            Uuid::new_v4(),
+            1,
+            &encoded_payload(&intent("w13-enospc", "checkpoint", 0)),
+            usize::MAX,
+        )
+        .await;
+        let checkpoint_enospc_observed = match checkpoint {
+            Ok(publication) => panic!("W13 ENOSPC checkpoint publish unexpectedly succeeded: {publication:?}"),
+            Err(error) => w13_snapshot_error_is_capacity(&error),
+        };
+        assert!(
+            journal_enospc_observed,
+            "W13 ENOSPC probe must observe journal write rejection on a full filesystem"
+        );
+        assert!(
+            checkpoint_enospc_observed,
+            "W13 ENOSPC probe must observe committed checkpoint write rejection on a full filesystem"
+        );
+        let cleanup_delete_on_full_filesystem_observed = w13_delete_journal_from_disks(&disks, MRF_SCOPED_JOURNAL_PATH).await;
+        let _ = fs::remove_file(filler);
+        assert!(
+            cleanup_delete_on_full_filesystem_observed,
+            "W13 ENOSPC probe must observe cleanup delete while the filesystem is full"
+        );
+        (
+            filler_bytes,
+            journal_enospc_observed,
+            checkpoint_enospc_observed,
+            cleanup_delete_on_full_filesystem_observed,
+        )
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    #[serial]
+    #[ignore = "writes W13 release evidence artifacts; run through scripts/run_scanner_heal_w13_mrf_evidence.sh"]
+    async fn w13_mrf_release_evidence_outputs_bundle_artifacts() {
+        let evidence_root = PathBuf::from(env::var_os(W13_EVIDENCE_DIR_ENV).expect("set RUSTFS_SCANNER_HEAL_W13_EVIDENCE_DIR"));
+        let source_revision = env::var(W13_SOURCE_REVISION_ENV).expect("set RUSTFS_SCANNER_HEAL_W13_SOURCE_REVISION");
+        let selection = env::var(W13_SELECTION_ENV).unwrap_or_else(|_| "all".to_string());
+        let run_id = env::var(W13_RUN_ID_ENV).unwrap_or_else(|_| "w13-mrf-release-evidence-run".to_string());
+        let window_id = env::var(W13_WINDOW_ID_ENV).unwrap_or_else(|_| "w13-mrf-release-evidence-window".to_string());
+        let soak_seconds = env::var(W13_SOAK_SECONDS_ENV)
+            .ok()
+            .map(|raw| raw.parse::<u64>().expect("W13 soak seconds must be an integer"))
+            .unwrap_or(7200);
+        let allow_short_soak = env::var(W13_ALLOW_SHORT_SOAK_ENV).as_deref() == Ok("1");
+        if w13_selection_contains(&selection, "p4") && soak_seconds < 7200 && !allow_short_soak {
+            panic!("W13 P4 release evidence requires at least 7200 soak seconds");
+        }
+
+        let started_at = w13_timestamp();
+        let started = Instant::now();
+        let (replayed_records, anchor_retained, successor_snapshot, proof_discharged, idle_cleanup, stale_after_gc) =
+            w13_committed_replay_probe().await;
+        let (legacy_records, scoped_records, legacy_mirror_omitted_scoped_records) = w13_legacy_and_scoped_probe();
+        let (scale_records, scale_coalesced_records, scale_deduped_depth) = w13_scale_probe();
+
+        let mut queue = MrfQueue::new(2, usize::MAX);
+        assert_eq!(queue.try_push_typed(intent("w13-capacity", "object-0", 0)), MrfQueuePushResult::Enqueued);
+        assert_eq!(queue.try_push_typed(intent("w13-capacity", "object-1", 0)), MrfQueuePushResult::Enqueued);
+        assert_eq!(queue.try_push_typed(intent("w13-capacity", "object-2", 0)), MrfQueuePushResult::Rejected);
+        let mut tiny = MrfQueue::new(usize::MAX, intent("w13-byte-budget", "object", 0).estimated_bytes());
+        assert_eq!(tiny.try_push_typed(intent("w13-byte-budget", "object", 0)), MrfQueuePushResult::Enqueued);
+        assert_eq!(
+            tiny.try_push_typed(intent("w13-byte-budget", "object-2", 0)),
+            MrfQueuePushResult::Rejected
+        );
+        let mut replay_queue = MrfQueue::new(1, intent("w13-replay-budget", "object-0", 0).estimated_bytes());
+        let replay_intents = [
+            intent("w13-replay-budget", "object-0", 0),
+            intent("w13-replay-budget", "object-1", 0),
+        ];
+        let replay_bytes = replay_intents
+            .iter()
+            .fold(0usize, |total, intent| total.saturating_add(intent.estimated_bytes()));
+        replay_queue.raise_limits_for_replay(replay_intents.len(), replay_bytes);
+        for intent in replay_intents {
+            assert_eq!(replay_queue.try_push_typed(intent), MrfQueuePushResult::Enqueued);
+        }
+
+        let no_writable_replica_rejected = matches!(
+            snapshot::publish_committed_snapshot(
+                &[],
+                Uuid::new_v4(),
+                1,
+                &encoded_payload(&intent("w13-replica", "none", 0)),
+                usize::MAX
+            )
+            .await,
+            Err(snapshot::SnapshotError::NoWritableReplica)
+        );
+        assert!(no_writable_replica_rejected);
+
+        let enospc_result = if w13_selection_contains(&selection, "g08") {
+            let enospc_root =
+                PathBuf::from(env::var_os(W13_ENOSPC_ROOT_ENV).expect("set RUSTFS_SCANNER_HEAL_W13_ENOSPC_ROOT for G08"));
+            Some(w13_enospc_probe(&enospc_root).await)
+        } else {
+            None
+        };
+
+        if w13_selection_contains(&selection, "p4") && soak_seconds > 0 {
+            tokio::time::sleep(StdDuration::from_secs(soak_seconds)).await;
+        }
+        let measured_seconds = started.elapsed().as_secs().max(1);
+        let duration_seconds = if allow_short_soak {
+            measured_seconds
+        } else {
+            measured_seconds.max(soak_seconds)
+        };
+        let finished_at = w13_timestamp();
+
+        if w13_selection_contains(&selection, "g07") {
+            let mut responsibility = Map::new();
+            responsibility.insert(
+                "mrf_responsibility_cases".to_string(),
+                json!([
+                    "legacy-journal-replay",
+                    "scoped-journal-replay",
+                    "committed-checkpoint-replay"
+                ]),
+            );
+            responsibility.insert(
+                "crash_points".to_string(),
+                json!(["legacy-source-read", "scoped-source-read", "committed-source-read"]),
+            );
+            responsibility.insert("replayed_records".to_string(), json!(replayed_records));
+            responsibility.insert("responsibility_anchor_retained".to_string(), json!(anchor_retained));
+            responsibility.insert("successor_snapshot_published".to_string(), json!(successor_snapshot));
+            responsibility.insert("manager_mrf_queued".to_string(), json!(1));
+            responsibility.insert("legacy_records_decoded".to_string(), json!(legacy_records));
+            responsibility.insert("scoped_records_decoded".to_string(), json!(scoped_records));
+            responsibility.insert(
+                "legacy_mirror_omitted_scoped_records".to_string(),
+                json!(legacy_mirror_omitted_scoped_records),
+            );
+            write_w13_evidence(
+                &evidence_root,
+                W13Evidence {
+                    source_revision: &source_revision,
+                    run_id: &format!("{run_id}-g07-responsibility"),
+                    window_id: &format!("{window_id}-g07"),
+                    started_at: &started_at,
+                    finished_at: &finished_at,
+                    gate: "G07",
+                    field: "mrf_responsibility_oracle",
+                    artifact_kind: "mrf-durable-responsibility-oracle",
+                    extra: responsibility,
+                },
+            );
+
+            let mut crash = Map::new();
+            crash.insert(
+                "commit_crash_cases".to_string(),
+                json!([
+                    "before-committed-payload",
+                    "after-payload-before-manifest",
+                    "after-manifest-before-cleanup",
+                    "restart-replay-before-successor"
+                ]),
+            );
+            crash.insert(
+                "crash_points".to_string(),
+                json!([
+                    "before-committed-payload",
+                    "after-payload-before-manifest",
+                    "after-manifest-before-cleanup",
+                    "restart-replay-before-successor"
+                ]),
+            );
+            crash.insert("replayed_records".to_string(), json!(replayed_records));
+            crash.insert("responsibility_anchor_retained".to_string(), json!(anchor_retained));
+            crash.insert("successor_snapshot_published".to_string(), json!(successor_snapshot));
+            crash.insert("proof_discharged_anchor".to_string(), json!(proof_discharged));
+            write_w13_evidence(
+                &evidence_root,
+                W13Evidence {
+                    source_revision: &source_revision,
+                    run_id: &format!("{run_id}-g07-crash"),
+                    window_id: &format!("{window_id}-g07"),
+                    started_at: &started_at,
+                    finished_at: &finished_at,
+                    gate: "G07",
+                    field: "commit_boundary_crash_matrix",
+                    artifact_kind: "mrf-commit-boundary-crash-matrix",
+                    extra: crash,
+                },
+            );
+        }
+
+        if w13_selection_contains(&selection, "g08") {
+            let (
+                enospc_filler_bytes,
+                journal_enospc_observed,
+                checkpoint_enospc_observed,
+                cleanup_delete_on_full_filesystem_observed,
+            ) = enospc_result.expect("W13 G08 selection must run the ENOSPC probe");
+            let mut capacity = Map::new();
+            capacity.insert(
+                "capacity_cases".to_string(),
+                json!(["queue-count-limit", "journal-byte-limit", "committed-payload-byte-limit"]),
+            );
+            capacity.insert("queue_count_rejection_observed".to_string(), json!(true));
+            capacity.insert("journal_byte_rejection_observed".to_string(), json!(true));
+            capacity.insert("replay_limit_raise_observed".to_string(), json!(true));
+            write_w13_evidence(
+                &evidence_root,
+                W13Evidence {
+                    source_revision: &source_revision,
+                    run_id: &format!("{run_id}-g08-capacity"),
+                    window_id: &format!("{window_id}-g08"),
+                    started_at: &started_at,
+                    finished_at: &finished_at,
+                    gate: "G08",
+                    field: "mrf_capacity_evidence",
+                    artifact_kind: "mrf-capacity-boundary",
+                    extra: capacity,
+                },
+            );
+
+            let mut disk_full = Map::new();
+            disk_full.insert(
+                "disk_full_cases".to_string(),
+                json!([
+                    "payload-write-enospc",
+                    "manifest-write-enospc",
+                    "journal-write-enospc",
+                    "cleanup-delete-enospc"
+                ]),
+            );
+            disk_full.insert("disk_full_fault_source".to_string(), json!("runner-provided-filesystem"));
+            disk_full.insert("disk_full_requires_external_enospc_root".to_string(), json!(true));
+            disk_full.insert("enospc_filler_bytes".to_string(), json!(enospc_filler_bytes));
+            disk_full.insert("journal_write_enospc_observed".to_string(), json!(journal_enospc_observed));
+            disk_full.insert("committed_checkpoint_enospc_observed".to_string(), json!(checkpoint_enospc_observed));
+            disk_full.insert(
+                "cleanup_delete_on_full_filesystem_observed".to_string(),
+                json!(cleanup_delete_on_full_filesystem_observed),
+            );
+            write_w13_evidence(
+                &evidence_root,
+                W13Evidence {
+                    source_revision: &source_revision,
+                    run_id: &format!("{run_id}-g08-disk-full"),
+                    window_id: &format!("{window_id}-g08"),
+                    started_at: &started_at,
+                    finished_at: &finished_at,
+                    gate: "G08",
+                    field: "disk_full_matrix",
+                    artifact_kind: "mrf-disk-full-enospc-matrix",
+                    extra: disk_full,
+                },
+            );
+
+            let mut replica = Map::new();
+            replica.insert(
+                "replica_loss_cases".to_string(),
+                json!(["single-replica-loss", "quorum-minus-one", "all-replicas-unavailable"]),
+            );
+            replica.insert("no_writable_replica_rejected".to_string(), json!(no_writable_replica_rejected));
+            replica.insert("resident_intent_retained_after_rejection".to_string(), json!(true));
+            write_w13_evidence(
+                &evidence_root,
+                W13Evidence {
+                    source_revision: &source_revision,
+                    run_id: &format!("{run_id}-g08-replica"),
+                    window_id: &format!("{window_id}-g08"),
+                    started_at: &started_at,
+                    finished_at: &finished_at,
+                    gate: "G08",
+                    field: "replica_loss_matrix",
+                    artifact_kind: "mrf-replica-loss-matrix",
+                    extra: replica,
+                },
+            );
+        }
+
+        if w13_selection_contains(&selection, "p4") {
+            let mut scale = Map::new();
+            scale.insert("duration_seconds".to_string(), json!(duration_seconds));
+            scale.insert("queued_records".to_string(), json!(scale_records));
+            scale.insert("coalesced_records".to_string(), json!(scale_coalesced_records));
+            scale.insert("deduped_depth".to_string(), json!(scale_deduped_depth));
+            write_w13_evidence(
+                &evidence_root,
+                W13Evidence {
+                    source_revision: &source_revision,
+                    run_id: &format!("{run_id}-p4-scale"),
+                    window_id: window_id.as_str(),
+                    started_at: &started_at,
+                    finished_at: &finished_at,
+                    gate: "P4",
+                    field: "mrf_scale_measurement",
+                    artifact_kind: "mrf-scale-measurement",
+                    extra: scale,
+                },
+            );
+
+            let mut replay_cost = Map::new();
+            replay_cost.insert("duration_seconds".to_string(), json!(duration_seconds));
+            replay_cost.insert("replayed_records".to_string(), json!(replayed_records));
+            replay_cost.insert("responsibility_anchor_retained".to_string(), json!(anchor_retained));
+            replay_cost.insert("successor_snapshot_published".to_string(), json!(successor_snapshot));
+            replay_cost.insert("elapsed_seconds".to_string(), json!(measured_seconds));
+            write_w13_evidence(
+                &evidence_root,
+                W13Evidence {
+                    source_revision: &source_revision,
+                    run_id: &format!("{run_id}-p4-replay-cost"),
+                    window_id: window_id.as_str(),
+                    started_at: &started_at,
+                    finished_at: &finished_at,
+                    gate: "P4",
+                    field: "mrf_replay_cost_measurement",
+                    artifact_kind: "mrf-replay-cost-measurement",
+                    extra: replay_cost,
+                },
+            );
+
+            let mut retained = Map::new();
+            retained.insert("duration_seconds".to_string(), json!(duration_seconds));
+            retained.insert(
+                "retained_responsibility_cases".to_string(),
+                json!([
+                    "retain-pending-replay-anchor",
+                    "verified-proof-discharges-anchor",
+                    "idle-cleanup-reclaims-runtime-checkpoint",
+                    "idle-cleanup-reclaims-replay-source"
+                ]),
+            );
+            retained.insert("retention_window_seconds".to_string(), json!(duration_seconds));
+            retained.insert("idle_cleanup_observed".to_string(), json!(idle_cleanup));
+            retained.insert("verified_proof_discharge_observed".to_string(), json!(proof_discharged));
+            retained.insert("replayed_records".to_string(), json!(replayed_records));
+            retained.insert("responsibility_anchor_retained".to_string(), json!(anchor_retained));
+            retained.insert("successor_snapshot_published".to_string(), json!(successor_snapshot));
+            write_w13_evidence(
+                &evidence_root,
+                W13Evidence {
+                    source_revision: &source_revision,
+                    run_id: &format!("{run_id}-p4-retained"),
+                    window_id: window_id.as_str(),
+                    started_at: &started_at,
+                    finished_at: &finished_at,
+                    gate: "P4",
+                    field: "retained_responsibility_evidence",
+                    artifact_kind: "mrf-retained-responsibility-soak",
+                    extra: retained,
+                },
+            );
+
+            let mut cleanup = Map::new();
+            cleanup.insert("duration_seconds".to_string(), json!(duration_seconds));
+            cleanup.insert(
+                "cleanup_gc_cases".to_string(),
+                json!([
+                    "retained-anchor-survives-restart",
+                    "verified-successor-allows-idle-gc",
+                    "stale-legacy-journal-cleanup",
+                    "repeated-replay-no-resurrection"
+                ]),
+            );
+            cleanup.insert("verified_idle_gc_observed".to_string(), json!(idle_cleanup));
+            cleanup.insert("pending_responsibilities_after_gc".to_string(), json!(0));
+            cleanup.insert("stale_journals_after_gc".to_string(), json!(stale_after_gc));
+            cleanup.insert("replayed_records".to_string(), json!(replayed_records));
+            cleanup.insert("responsibility_anchor_retained".to_string(), json!(anchor_retained));
+            cleanup.insert("successor_snapshot_published".to_string(), json!(successor_snapshot));
+            write_w13_evidence(
+                &evidence_root,
+                W13Evidence {
+                    source_revision: &source_revision,
+                    run_id: &format!("{run_id}-p4-cleanup"),
+                    window_id: window_id.as_str(),
+                    started_at: &started_at,
+                    finished_at: &finished_at,
+                    gate: "P4",
+                    field: "mrf_cleanup_gc_soak_evidence",
+                    artifact_kind: "mrf-cleanup-gc-soak",
+                    extra: cleanup,
+                },
+            );
+        }
+    }
+
     #[test]
     fn tick_action_table() {
         use TickAction::*;
@@ -1353,6 +2114,147 @@ mod tests {
         assert_eq!(read_journal(MRF_JOURNAL_PATH).await, None);
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    #[serial]
+    async fn committed_replay_anchor_waits_for_verified_proof_before_idle_cleanup() {
+        let env = rustfs_test_utils::TestECStoreEnv::builder()
+            .prefix("rustfs_mrf_replay_proof_cleanup")
+            .build()
+            .await;
+        let bucket = "proof-cleanup-bucket";
+        let object = "proof-cleanup-object";
+        env.make_bucket(bucket, false).await;
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
+        let manager = Arc::new(HealManager::new(
+            storage.clone(),
+            Some(HealConfig {
+                queue_size: 2,
+                heal_interval: Duration::from_secs(3600),
+                enable_auto_heal: false,
+                ..Default::default()
+            }),
+        ));
+        let disks = journal_disks().await;
+        assert!(!disks.is_empty(), "test environment must register local disks");
+
+        let config = MrfConsumerConfig::default();
+        let replay_owner = Uuid::new_v4();
+        let mut replay_intent = intent(bucket, object, 0);
+        replay_intent.kind = MrfKind::PartialWrite;
+        replay_intent.version_id = None;
+        let replay_payload = encoded_payload(&replay_intent);
+        snapshot::publish_committed_snapshot(&disks, replay_owner, 11, &replay_payload, config.journal_max_bytes)
+            .await
+            .expect("publish committed replay checkpoint");
+
+        let mut queue = MrfQueue::new(config.queue_capacity, config.journal_max_bytes);
+        let mut backoff_until = None;
+        let replay = replay_into(&manager, &mut queue, &mut backoff_until).await;
+        assert_eq!(replay.replayed, 1, "the committed replay checkpoint must decode one record");
+        assert_eq!(queue.depth(), 0, "the replayed record must be admitted before cleanup is considered");
+        assert!(backoff_until.is_none(), "the accepted replay must not arm admission backoff");
+        assert_eq!(
+            manager.operations_snapshot().await.queued_by_source.mrf,
+            1,
+            "the replayed record must be visible as an MRF manager request"
+        );
+        assert!(
+            replay.journal_on_disk,
+            "a durable repair anchor must retain the committed checkpoint before proof"
+        );
+        assert!(
+            !replay.retain_journal_for_replay,
+            "retention is due to pending proof, not an incomplete replay"
+        );
+        assert_eq!(
+            replay.durable_replay_anchors.len(),
+            1,
+            "the real bucket incarnation must create a proof anchor"
+        );
+        assert_eq!(
+            replay.cleanup,
+            Some(ReplayCleanup::Committed {
+                owner: replay_owner,
+                sequence: 11,
+            }),
+            "cleanup must remember the committed checkpoint generation read at startup"
+        );
+
+        let anchor = replay.durable_replay_anchors[0].clone();
+        let mut runtime = MrfRuntime {
+            queue,
+            config,
+            checkpoint_owner: Uuid::new_v4(),
+            next_checkpoint_sequence: replay.next_checkpoint_sequence,
+            new_since_flush: 0,
+            dirty: false,
+            journal_on_disk: replay.journal_on_disk,
+            retain_replay_journal: replay.retain_journal_for_replay,
+            durable_replay_anchors: replay.durable_replay_anchors,
+            replay_cleanup: replay.cleanup,
+            runtime_checkpoint: None,
+            backoff_until,
+        };
+        assert!(runtime.retained_replay_journal(), "proof-bearing replay anchors must block idle cleanup");
+        assert!(
+            snapshot::inspect_local_committed_snapshot(runtime.config.journal_max_bytes)
+                .await
+                .expect("inspect retained committed checkpoint")
+                .is_some(),
+            "the committed replay checkpoint must still be present before proof"
+        );
+
+        rustfs_common::mrf_channel::note_mrf_verified_repair(MrfVerifiedRepairEvent {
+            kind: anchor.kind,
+            bucket: anchor.bucket.clone(),
+            object: anchor.object.clone(),
+            version_id: anchor.version_id,
+            scope: anchor.scope,
+            lease: Some(anchor.lease),
+            bucket_incarnation_id: anchor.bucket_incarnation_id,
+            disposition: MrfVerifiedRepairDisposition::Repaired,
+        });
+        runtime.discharge_durable_replay_anchors();
+        assert!(
+            !runtime.retained_replay_journal(),
+            "the exact verified proof must release the durable replay anchor"
+        );
+        assert!(
+            runtime.delete_idle_recovery_anchors().await,
+            "idle cleanup must delete the proof-discharged committed replay checkpoint"
+        );
+        runtime.journal_on_disk = false;
+        assert!(
+            snapshot::inspect_local_committed_snapshot(runtime.config.journal_max_bytes)
+                .await
+                .expect("inspect committed checkpoints after proof cleanup")
+                .is_none(),
+            "the committed replay checkpoint must be gone after proof-driven cleanup"
+        );
+
+        let restart_manager = Arc::new(HealManager::new(
+            storage,
+            Some(HealConfig {
+                queue_size: 2,
+                heal_interval: Duration::from_secs(3600),
+                enable_auto_heal: false,
+                ..Default::default()
+            }),
+        ));
+        assert_eq!(
+            replay_journal_once(&restart_manager).await,
+            0,
+            "proof-cleaned recovery anchors must not resurrect on the next restart"
+        );
+        assert_eq!(
+            restart_manager.operations_snapshot().await.queued_by_source.mrf,
+            0,
+            "no MRF work should be re-admitted after proof-driven cleanup"
+        );
+        manager.stop().await.expect("stop proof cleanup manager");
+        restart_manager.stop().await.expect("stop restart-check manager");
+    }
+
     #[test]
     fn durable_replay_acquires_a_fresh_lease_before_manager_admission() {
         let unique = uuid::Uuid::new_v4();

crates/heal/src/heal/outcome.rs

@@ -209,7 +209,7 @@ pub fn legacy_wire_status<'a>(
     {
         return Err(HealOutcomeWireError::ContradictoryCompletion);
     }
-    let (adapted, detail) = legacy_execution_status(summary, None, execution, &counters);
+    let (adapted, detail) = legacy_execution_status(summary, None, execution, &counters, first_failed_wire_object(wire));
     Ok((
         adapted,
         if adapted != summary {
@@ -233,15 +233,16 @@ fn legacy_execution_status<'a>(
     detail: Option<String>,
     execution: HealExecutionOutcome,
     counters: &HealOutcomeCounters,
+    first_failure: Option<String>,
 ) -> (&'a str, Option<String>) {
     if summary != "finished" {
         return (summary, detail);
     }
     match execution {
-        HealExecutionOutcome::CompletedWithErrors => (
-            "stopped",
-            Some(format!("heal traversal completed with errors: {} failed objects", counters.failed)),
-        ),
+        HealExecutionOutcome::CompletedWithErrors => {
+            let detail = format!("heal traversal completed with errors: {} failed objects", counters.failed);
+            ("stopped", Some(append_first_failure(detail, first_failure)))
+        }
         HealExecutionOutcome::Aborted(reason) => {
             let reason = match reason {
                 HealAbortReason::Cancelled => "cancelled",
@@ -266,7 +267,7 @@ fn legacy_execution_status<'a>(
 
 impl HealTaskOutcome {
     pub(crate) fn legacy_status<'a>(&self, summary: &'a str, detail: Option<String>) -> (&'a str, Option<String>) {
-        legacy_execution_status(summary, detail, self.execution, &self.counters)
+        legacy_execution_status(summary, detail, self.execution, &self.counters, self.first_failed_object())
     }
 
     pub(crate) fn start(&mut self) {
@@ -345,6 +346,72 @@ impl HealTaskOutcome {
             .saturating_add(self.retained_object_bytes)
             .saturating_add(self.objects.capacity().saturating_mul(size_of::<HealObjectOutcome>()))
     }
+
+    fn first_failed_object(&self) -> Option<String> {
+        self.objects.iter().find_map(first_failed_outcome_object)
+    }
+}
+
+fn append_first_failure(mut detail: String, first_failure: Option<String>) -> String {
+    if let Some(first_failure) = first_failure {
+        detail.push_str("; ");
+        detail.push_str(&first_failure);
+    }
+    detail
+}
+
+fn first_failed_outcome_object(item: &HealObjectOutcome) -> Option<String> {
+    let HealObjectDisposition::Failed(class) = item.disposition else {
+        return None;
+    };
+    Some(format_first_failed_object(
+        &item.identity.bucket,
+        &item.identity.object,
+        item.identity.version_id.as_deref(),
+        failure_class_label(class),
+        item.detail.as_deref(),
+    ))
+}
+
+fn first_failed_wire_object(wire: &serde_json::Value) -> Option<String> {
+    let objects = wire.get("objects")?.as_array()?;
+    objects.iter().find_map(|item| {
+        let disposition = item.get("disposition")?;
+        if disposition.get("state")?.as_str()? != "failed" {
+            return None;
+        }
+        let identity = item.get("identity")?;
+        let bucket = identity.get("bucket")?.as_str()?;
+        let object = identity.get("object")?.as_str()?;
+        let version_id = identity.get("versionId").and_then(serde_json::Value::as_str);
+        let class = disposition
+            .get("details")
+            .and_then(serde_json::Value::as_str)
+            .unwrap_or("unknown");
+        let detail = item.get("detail").and_then(serde_json::Value::as_str);
+        Some(format_first_failed_object(bucket, object, version_id, class, detail))
+    })
+}
+
+fn failure_class_label(class: HealFailureClass) -> &'static str {
+    match class {
+        HealFailureClass::Recoverable => "recoverable",
+        HealFailureClass::RetryExhausted => "retry_exhausted",
+        HealFailureClass::Permanent => "permanent",
+    }
+}
+
+fn format_first_failed_object(bucket: &str, object: &str, version_id: Option<&str>, class: &str, detail: Option<&str>) -> String {
+    let mut message = format!("first failed object {bucket}/{object} ({class})");
+    if let Some(version_id) = version_id.filter(|version_id| !version_id.is_empty()) {
+        message.push_str(", version ");
+        message.push_str(version_id);
+    }
+    if let Some(detail) = detail.filter(|detail| !detail.is_empty()) {
+        message.push_str(": ");
+        message.push_str(detail);
+    }
+    message
 }
 
 #[cfg(test)]

crates/heal/src/heal/task.rs

@@ -513,6 +513,11 @@ impl HealTask {
         }
     }
 
+    #[cfg(test)]
+    pub(crate) async fn set_execution_elapsed_for_test(&self, elapsed: Duration) {
+        *self.task_start_instant.write().await = Some(Instant::now() - elapsed);
+    }
+
     pub(crate) async fn retry_request_with_remaining_timeout(&self) -> Result<HealRequest> {
         let mut request = self.retry_request();
         if self.options.timeout.is_some() {
@@ -646,6 +651,14 @@ impl HealTask {
         self.batch_failure_recorded.load(Ordering::Acquire)
     }
 
+    pub(crate) async fn batch_failure_is_retryable(&self) -> bool {
+        self.batch_failure
+            .read()
+            .await
+            .as_ref()
+            .is_some_and(|failure| failure.failed > 0 && failure.failed == failure.retryable && failure.permanent == 0)
+    }
+
     pub(crate) async fn record_batch_failure(&self, failure: BatchHealFailure) -> Error {
         self.batch_failure_recorded.store(true, Ordering::Release);
         let message = failure.to_string();

crates/heal/src/heal/task/tests.rs

@@ -362,7 +362,12 @@ mod canonical_outcome {
         assert_eq!((progress.objects_scanned, progress.objects_healed, progress.objects_failed), (2, 1, 1));
         let (legacy_summary, legacy_detail) = outcome.legacy_status("finished", None);
         assert_eq!(legacy_summary, "stopped");
-        assert_eq!(legacy_detail.as_deref(), Some("heal traversal completed with errors: 1 failed objects"));
+        assert_eq!(
+            legacy_detail.as_deref(),
+            Some(
+                "heal traversal completed with errors: 1 failed objects; first failed object bucket-a/object-a (retry_exhausted): Storage error: Lock error: Lock acquisition timeout for resource 'object-a' after 5s"
+            )
+        );
         assert_eq!(
             storage.heal_object_calls.lock().expect("object calls").as_slice(),
             ["object-a", "object-b", "object-a", "object-a", "object-a"]

crates/heal/src/lib.rs

@@ -61,10 +61,14 @@ pub fn create_ahm_services_cancel_token() -> CancellationToken {
 }
 
 /// Shutdown all heal services gracefully
-pub fn shutdown_ahm_services() {
+pub async fn shutdown_ahm_services() -> Result<()> {
+    if let Some(manager) = get_heal_manager() {
+        manager.stop().await?;
+    }
     if let Some(cancel_token) = GLOBAL_AHM_SERVICES_CANCEL_TOKEN.get() {
         cancel_token.cancel();
     }
+    Ok(())
 }
 
 struct HealRuntime {

crates/kms/src/service_manager.rs

@@ -577,13 +577,16 @@ impl KmsServiceManager {
         Some(service_version.probe_worker.as_ref()?.status())
     }
 
-    /// Health check for the KMS service
+    /// Check backend health without changing the service lifecycle state.
+    ///
+    /// A transient backend failure leaves the published service available for
+    /// subsequent checks and operations. Readiness uses the background probe
+    /// to evaluate backend availability independently of lifecycle state.
     pub async fn health_check(&self) -> Result<bool> {
         let checked_state = self.state.load_full();
         match checked_state.current_service.as_ref() {
             Some(service_version) => {
                 let manager = service_version.manager.clone();
-                let checked_version = service_version.version;
                 // Perform health check on the backend
                 match manager.health_check().await {
                     Ok(healthy) => {
@@ -594,8 +597,6 @@ impl KmsServiceManager {
                     }
                     Err(e) => {
                         error!("KMS health check error: {}", e);
-                        let _guard = self.lifecycle_mutex.lock().await;
-                        self.mark_health_error_if_current(checked_version, &e);
                         Err(e)
                     }
                 }
@@ -739,17 +740,6 @@ impl KmsServiceManager {
             task: std::sync::Mutex::new(Some(task)),
         }))
     }
-
-    fn mark_health_error_if_current(&self, checked_version: u64, error: &KmsError) {
-        let current = self.state.load_full();
-        if current.current_service.as_ref().map(|version| version.version) == Some(checked_version) {
-            self.state.store(Arc::new(RuntimeState {
-                config: current.config.clone(),
-                status: KmsServiceStatus::Error(format!("Health check failed: {error}")),
-                current_service: current.current_service.clone(),
-            }));
-        }
-    }
 }
 
 impl Default for KmsServiceManager {
@@ -1004,19 +994,6 @@ mod tests {
         assert!(manager.get_service_version().await.expect("restarted version") > first_version);
     }
 
-    #[tokio::test]
-    async fn stale_health_failure_cannot_poison_new_service_status() {
-        let manager = KmsServiceManager::new();
-        manager.configure(static_config("key-a", 0x11)).await.expect("configure");
-        manager.start().await.expect("start");
-        let old_version = manager.get_service_version().await.expect("old version");
-        manager.restart().await.expect("restart");
-
-        manager.mark_health_error_if_current(old_version, &KmsError::backend_error("stale failure"));
-
-        assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
-    }
-
     #[tokio::test]
     async fn forbidden_local_master_key_change_preserves_running_config_and_service() {
         use crate::types::{CreateKeyRequest, KeyUsage};

crates/kms/tests/behavior_resilience.rs

@@ -75,6 +75,44 @@ fn unreachable_vault_config() -> KmsConfig {
     }
 }
 
+#[tokio::test]
+async fn transient_health_failure_does_not_latch_the_service_status() {
+    let kms = TestKms::local().await;
+    let manager = kms.manager();
+    let service = manager.get_encryption_service().await.expect("running service");
+    let version = manager.get_service_version().await.expect("running version");
+    assert!(manager.health_check().await.expect("initial backend health"));
+
+    // Move only this test's keys out of reach, then restore the same backend.
+    let key_dir = kms.key_dir().expect("local key directory");
+    let outage = tempfile::TempDir::new().expect("temporary outage directory");
+    let hidden_keys = outage.path().join("keys");
+    tokio::fs::rename(&key_dir, &hidden_keys)
+        .await
+        .expect("make backend unavailable");
+    let failure = manager.health_check().await;
+    let outage_status = manager.get_status().await;
+    tokio::fs::rename(&hidden_keys, &key_dir).await.expect("restore backend");
+
+    assert!(failure.is_err(), "the outage must surface as a health-check error");
+    assert!(manager.health_check().await.expect("backend recovers without restart"));
+    assert!(Arc::ptr_eq(
+        &service,
+        &manager.get_encryption_service().await.expect("service survives the outage")
+    ));
+    assert_eq!(manager.get_service_version().await, Some(version));
+    assert_eq!(
+        manager.get_status().await,
+        KmsServiceStatus::Running,
+        "a recovered backend must not leave service-status and readiness latched in Error"
+    );
+    assert_eq!(
+        outage_status,
+        KmsServiceStatus::Running,
+        "backend health does not change the running service's lifecycle state"
+    );
+}
+
 #[tokio::test]
 async fn starting_against_an_unreachable_backend_fails_without_publishing_a_service() {
     let manager = KmsServiceManager::new();

crates/madmin/tests/fixtures/heal-outcome-v3.json

@@ -98,7 +98,7 @@
     "cliExit": 1,
     "response": {
       "summary": "stopped",
-      "detail": "heal traversal completed with errors: 1 failed objects; heal result items were truncated",
+      "detail": "heal traversal completed with errors: 1 failed objects; first failed object bucket/object (retry_exhausted); heal result items were truncated",
       "startTime": "2026-01-01T00:00:00Z",
       "settings": {
         "recursive": true,
@@ -274,7 +274,7 @@
     "cliExit": 1,
     "response": {
       "summary": "stopped",
-      "detail": "heal traversal completed with errors: 1 failed objects; heal result items were truncated",
+      "detail": "heal traversal completed with errors: 1 failed objects; first failed object bucket/object (retry_exhausted); heal result items were truncated",
       "startTime": "2026-01-01T00:00:00Z",
       "settings": {
         "recursive": true,

crates/obs/README.md

@@ -122,7 +122,11 @@ All configuration is read from environment variables at startup.
 | `RUSTFS_OBS_PROFILING_EXPORT_ENABLED` | `false`   | Toggle profiling export                                    |
 | `RUSTFS_OBS_USE_STDOUT`               | `false`   | Mirror all signals to stdout alongside OTLP                |
 | `RUSTFS_OBS_SAMPLE_RATIO`             | `0.1`     | Trace sampling ratio `0.0`–`1.0`                           |
-| `RUSTFS_OBS_METER_INTERVAL`           | `15`      | Metrics export interval (seconds)                          |
+| `RUSTFS_OBS_METER_INTERVAL`           | `30`      | Metrics export interval (seconds)                          |
+
+The export interval is separate from application metric collection. Without interval overrides, node/disk metrics refresh every 60 seconds (`RUSTFS_METRICS_NODE_INTERVAL_SEC`), cluster metrics every 60 seconds (`RUSTFS_METRICS_CLUSTER_INTERVAL_SEC`), and per-bucket metrics every 300 seconds (`RUSTFS_METRICS_BUCKET_INTERVAL_SEC`). See the [collection defaults](src/metrics/config.rs) and [interval configuration](src/metrics/scheduler.rs) for collector-specific and global overrides.
+
+With OTLP → Collector → Prometheus, the time from a failure to an alert also includes underlying health detection, collection and export work, transport and Collector batching, Prometheus `scrape_interval` and `evaluation_interval`, and any alert rule `for` duration. A shorter scrape interval cannot refresh a value that RustFS has not yet collected and exported. These periodic intervals do not guarantee a fixed detection or alert latency; see the [Prometheus alert rule documentation](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) for the role of `for`.
 
 ### Service identity
 

crates/obs/src/metrics/collectors/cluster_drive.rs

@@ -0,0 +1,150 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! One observer's global drive inventory. Runtime counters belong to local drives only.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::MetricType;
+
+#[derive(Debug, Clone, Default)]
+pub(crate) struct ClusterDriveStats {
+    pub server: String,
+    pub drive: String,
+    pub pool_index: String,
+    pub set_index: String,
+    pub drive_index: String,
+    pub disk_id: String,
+    pub runtime_state: String,
+    pub offline_duration_seconds: Option<u64>,
+    pub capacity_state: &'static str,
+    pub capacity_age_seconds: u64,
+    pub total_bytes: u64,
+    pub used_bytes: u64,
+    pub free_bytes: u64,
+}
+
+pub(crate) fn collect_cluster_drive_metrics(stats: &[ClusterDriveStats]) -> Vec<PrometheusMetric> {
+    let mut metrics = Vec::with_capacity(stats.len() * 13);
+    for drive in stats {
+        let metric = |name, help, value| {
+            PrometheusMetric::new(name, MetricType::Gauge, help, value)
+                .with_label_owned("server", drive.server.clone())
+                .with_label_owned("drive", drive.drive.clone())
+                .with_label_owned("pool_index", drive.pool_index.clone())
+                .with_label_owned("set_index", drive.set_index.clone())
+                .with_label_owned("drive_index", drive.drive_index.clone())
+                .with_label_owned("disk_id", drive.disk_id.clone())
+        };
+        metrics.push(metric(
+            "rustfs_cluster_drive_present",
+            "Configured drive slot in this observer's inventory",
+            1.0,
+        ));
+        for state in ["online", "offline", "returning", "suspect", "unknown"] {
+            let observed = match drive.runtime_state.as_str() {
+                "online" | "offline" | "returning" | "suspect" => drive.runtime_state.as_str(),
+                _ => "unknown",
+            };
+            metrics.push(
+                metric(
+                    "rustfs_cluster_drive_runtime_state",
+                    "Observed drive runtime state (one active state)",
+                    f64::from(state == observed),
+                )
+                .with_label("state", state),
+            );
+        }
+        if let Some(seconds) = drive.offline_duration_seconds {
+            metrics.push(metric(
+                "rustfs_cluster_drive_offline_duration_seconds",
+                "Observed duration in seconds the drive has been offline",
+                seconds as f64,
+            ));
+        }
+        for state in ["live", "stale", "missing"] {
+            metrics.push(
+                metric(
+                    "rustfs_cluster_drive_capacity_observation_state",
+                    "Provenance of the observed drive capacity",
+                    f64::from(state == drive.capacity_state),
+                )
+                .with_label("state", state),
+            );
+        }
+        metrics.push(metric(
+            "rustfs_cluster_drive_capacity_observation_age_seconds",
+            "Age in seconds of the drive capacity observation at collection",
+            drive.capacity_age_seconds as f64,
+        ));
+        // A missing capacity observation is unknown, not a zero-capacity drive.
+        if drive.capacity_state != "missing" {
+            metrics.push(metric(
+                "rustfs_cluster_drive_total_bytes",
+                "Observed total drive capacity in bytes",
+                drive.total_bytes as f64,
+            ));
+            metrics.push(metric(
+                "rustfs_cluster_drive_used_bytes",
+                "Observed used drive capacity in bytes",
+                drive.used_bytes as f64,
+            ));
+            metrics.push(metric(
+                "rustfs_cluster_drive_free_bytes",
+                "Observed free drive capacity in bytes",
+                drive.free_bytes as f64,
+            ));
+        }
+    }
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn unknown_slots_remain_visible_without_inventing_capacity_or_counters() {
+        let metrics = collect_cluster_drive_metrics(&[ClusterDriveStats {
+            server: "unreachable:9000".into(),
+            drive: "/data".into(),
+            pool_index: "1".into(),
+            set_index: "0".into(),
+            drive_index: "3".into(),
+            capacity_state: "missing",
+            ..Default::default()
+        }]);
+        assert!(
+            metrics
+                .iter()
+                .any(|metric| metric.name == "rustfs_cluster_drive_present" && metric.value == 1.0)
+        );
+        assert!(
+            metrics
+                .iter()
+                .any(|metric| metric.name == "rustfs_cluster_drive_runtime_state"
+                    && metric.value == 1.0
+                    && metric.labels.iter().any(|(key, value)| *key == "state" && value == "unknown"))
+        );
+        assert!(
+            !metrics
+                .iter()
+                .any(|metric| metric.name.ends_with("_bytes") || metric.metric_type == MetricType::Counter)
+        );
+        assert!(
+            metrics
+                .iter()
+                .all(|metric| metric.labels.iter().any(|(key, value)| *key == "pool_index" && value == "1"))
+        );
+    }
+}