feat(scanner): surface cold segment reuse oracle

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=775825dcb2b4997c4fa24bd9ba9c0546316c503f4d5e369c39abff0678c95e8e
-sha256-darwin=775825dcb2b4997c4fa24bd9ba9c0546316c503f4d5e369c39abff0678c95e8e
+sha256-linux=4696a43b167ac608b3b8677027c9fe9fdac3396d37c8cca11dce531c720ac6d2
+sha256-darwin=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=874c881d7b45f12378a5817c7f42c95c4981960a2ec9ce12dcf4af239ae1f9d5
-sha256-linux=9351e25b45bf7dfce18b951a5e3740225f457cacc53b8bf9f500f6947763ec0e
+sha256-darwin=f0c78fdb93471575d9a64c5c46eae6c806bdd0bc10a6e33d7fb574aabd8db5a3
+sha256-linux=03ed7016cab672de9320e31375a0358eceacb4408b0e79cf063614fa7c878b87

.config/e2e-nightly-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=83a7dcaffd5a789517ae9f02a224f66a9713937885cff96fca2ad7e216f197ae
-sha256-linux=626c10f8c964507ff987b6c86069e9019dc6d2ae7fb02db9be5df5aa8cc5145b
+sha256-darwin=364f2329a7b72eb9f1608dbe1a3af37af4095354014f3cbe23ca448492d89961
+sha256-linux=60983f1ebe7068cf660d473c5f76c76a650410ccc99d71934ddca7fd67607987

.config/make/lint-fmt.mak

@@ -89,7 +89,6 @@ offline-enrollment-e2e-check: core-deps ## Build and exercise the dedicated offl
 test-wiring-check: ## Check tests stay registered and selected by their intended runners
 	@echo "🧪 Checking test wiring..."
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py
-	$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --check-workflow
 
 .PHONY: log-analyzer-rules-check
 log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source

.config/make/tests.mak

@@ -32,16 +32,6 @@ script-tests: ## Run shell script tests
 	./scripts/test_hotpath_warp_ab_gate.sh
 	./scripts/test_hotpath_warp_abba.sh
 	./scripts/test_scanner_validation_harness.sh
-	./scripts/test_scanner_heal_checkpoint_crash_evidence.sh
-	./scripts/test_scanner_heal_authority_evidence.sh
-	./scripts/test_scanner_heal_scoped_ack_evidence.sh
-	./scripts/test_scanner_heal_legacy_rollback_evidence.sh
-	./scripts/test_scanner_heal_g14_multiset_evidence.sh
-	./scripts/test_scanner_heal_scheduler_pressure_evidence.sh
-	./scripts/test_scanner_heal_status_outcome_evidence.sh
-	./scripts/test_scanner_heal_maintenance_evidence.sh
-	./scripts/test_scanner_heal_w13_mrf_evidence.sh
-	./scripts/test_scanner_heal_w16_recovery_evidence.sh
 	./scripts/test_exact_1mib_handoff_abba.sh
 	./scripts/test_pinned_paired_abba_bench.sh
 	./scripts/test_manual_transition_runbooks.sh
@@ -49,7 +39,6 @@ script-tests: ## Run shell script tests
 	./scripts/test_python_bin.sh
 	./scripts/check_embedded_secrets.sh --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
-	$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py

.config/nextest.toml

@@ -197,12 +197,6 @@ test-group = 'e2e-cluster-nightly'
 filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
 test-group = 'e2e-vault'
 
-# This four-disk, 65-member rollback probe already drives up to 32 concurrent
-# durable deletions. Reserve this nextest run's capacity for its progress oracle.
-[[profile.default.overrides]]
-filter = 'package(rustfs-ecstore) & test(=store::init::tests::dispatch_manifest_rollback_bounded_concurrency_reaches_tail_behind_slow_member)'
-threads-required = "num-test-threads"
-
 # ---------------------------------------------------------------------------
 # ci profile — the strict CI gate (ci.yml `cargo nextest run --profile ci`)
 # ---------------------------------------------------------------------------

.config/scanner-heal-required-tests.json

@@ -78,43 +78,6 @@
       "erasure": {"data_blocks": 8, "parity_blocks": 4},
       "erasure_set_drive_count": 12,
       "scope": "Target process killed during partial background rebuild on a single 3x4 EC8+4 set; real unclean-shutdown marker, exact unversioned S3 bodies and replacement-drive shards; not power loss, multi-set, or multi-pool."
-    },
-    "background-target-restart-ec8-4-multi-set": {
-      "gate": "G14",
-      "task": "W21",
-      "lane": "e2e-nightly",
-      "suite": "e2e_test",
-      "name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart",
-      "oracle": "background-target-restart-ec8-4-multi-set.json",
-      "evidence": "process-restart",
-      "unclean_shutdown_marker": false,
-      "min_objects": 9,
-      "max_objects": 65,
-      "topology": {"nodes": 3, "drives_per_node": 8},
-      "erasure": {"data_blocks": 8, "parity_blocks": 4},
-      "erasure_set_drive_count": 12,
-      "sets": 2,
-      "pools": 1,
-      "scope": "Target process restart during partial background rebuild on a 3x8 EC8+4 layout with two erasure sets in one pool; exact unversioned S3 bodies and replacement-drive shards; not power loss or multi-pool."
-    },
-    "background-target-crash-ec8-4-multi-pool": {
-      "gate": "G14",
-      "task": "W21",
-      "lane": "e2e-nightly",
-      "suite": "e2e_test",
-      "name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_pool_after_background_target_crash",
-      "oracle": "background-target-crash-ec8-4-multi-pool.json",
-      "evidence": "process-crash-restart",
-      "unclean_shutdown_marker": true,
-      "min_objects": 9,
-      "max_objects": 65,
-      "topology": {"nodes": 3, "drives_per_node": 12},
-      "erasure": {"data_blocks": 8, "parity_blocks": 4},
-      "erasure_set_drive_count": 12,
-      "sets": 3,
-      "pools": 3,
-      "outage_target_manifest_required": false,
-      "scope": "Target process crash during partial background rebuild on three single-node EC8+4 pools; exact baseline S3 bodies and replacement-drive shards, with down-window outage PUT refusal recorded and a deferred post-rejoin outage object verified through S3 when the full target pool was offline."
     }
   },
   "release_lanes": {
@@ -171,11 +134,7 @@
       "lane": "authority-coverage",
       "status": "pending",
       "description": "Complete root and quota authority coverage",
-      "requires": ["root authority evidence", "quota authority evidence"],
-      "evidence_fields": [
-        "root_authority_evidence",
-        "quota_authority_evidence"
-      ]
+      "requires": ["root authority evidence", "quota authority evidence"]
     },
     {
       "gate": "G02",
@@ -183,11 +142,7 @@
       "lane": "checkpoint-and-crash",
       "status": "pending",
       "description": "Bounded checkpoint progress and independent version inventory",
-      "requires": ["bounded checkpoint oracle", "independent version inventory"],
-      "evidence_fields": [
-        "bounded_checkpoint_oracle",
-        "independent_version_inventory"
-      ]
+      "requires": ["bounded checkpoint oracle", "independent version inventory"]
     },
     {
       "gate": "G03",
@@ -209,11 +164,7 @@
       "lane": "checkpoint-and-crash",
       "status": "pending",
       "description": "Crash at every cache, root, floor and intent boundary",
-      "requires": ["cache boundary crash evidence", "root/floor/intent crash evidence"],
-      "evidence_fields": [
-        "cache_boundary_crash_evidence",
-        "root_floor_intent_crash_evidence"
-      ]
+      "requires": ["cache boundary crash evidence", "root/floor/intent crash evidence"]
     },
     {
       "gate": "G05",
@@ -221,11 +172,7 @@
       "lane": "status-and-outcome",
       "status": "pending",
       "description": "Per-object outcomes and bounded terminal retention",
-      "requires": ["per-object outcome oracle", "terminal retention bounds"],
-      "evidence_fields": [
-        "per_object_outcome_oracle",
-        "terminal_retention_bounds"
-      ]
+      "requires": ["per-object outcome oracle", "terminal retention bounds"]
     },
     {
       "gate": "G06",
@@ -233,12 +180,7 @@
       "lane": "status-and-outcome",
       "status": "pending",
       "description": "Concurrent status, legacy clients and truncation",
-      "requires": ["concurrent status evidence", "legacy client compatibility", "truncation behavior"],
-      "evidence_fields": [
-        "concurrent_status_evidence",
-        "legacy_client_compatibility",
-        "truncation_behavior"
-      ]
+      "requires": ["concurrent status evidence", "legacy client compatibility", "truncation behavior"]
     },
     {
       "gate": "G07",
@@ -284,11 +226,7 @@
       "lane": "scheduler-pressure",
       "status": "pending",
       "description": "Bounded scheduling and pressure recovery",
-      "requires": ["scheduler bound evidence", "pressure recovery evidence"],
-      "evidence_fields": [
-        "scheduler_bound_evidence",
-        "pressure_recovery_evidence"
-      ]
+      "requires": ["scheduler bound evidence", "pressure recovery evidence"]
     },
     {
       "gate": "G11",
@@ -313,11 +251,7 @@
       "lane": "authority-coverage",
       "status": "pending",
       "description": "Both quota paths during reset and settlement",
-      "requires": ["reset quota-path evidence", "settlement quota-path evidence"],
-      "evidence_fields": [
-        "reset_quota_path_evidence",
-        "settlement_quota_path_evidence"
-      ]
+      "requires": ["reset quota-path evidence", "settlement quota-path evidence"]
     },
     {
       "gate": "G13",
@@ -325,12 +259,7 @@
       "lane": "maintenance-producers",
       "status": "pending",
       "description": "Quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail",
-      "requires": ["quorum-minus-one matrix", "unknown-disk/remount matrix", "Object Lock dry-run grace evidence"],
-      "evidence_fields": [
-        "quorum_minus_one_matrix",
-        "unknown_disk_remount_matrix",
-        "object_lock_dry_run_grace_evidence"
-      ]
+      "requires": ["quorum-minus-one matrix", "unknown-disk/remount matrix", "Object Lock dry-run grace evidence"]
     },
     {
       "gate": "G14",
@@ -359,12 +288,7 @@
       "lane": "scheduler-pressure",
       "status": "pending",
       "description": "Measured cold-walk share and foreground latency/throughput",
-      "requires": ["cold-walk share measurement", "foreground latency/throughput measurement"],
-      "evidence_fields": [
-        "cold_walk_share_measurement",
-        "foreground_latency_throughput_measurement",
-        "profile_evidence"
-      ]
+      "requires": ["cold-walk share measurement", "foreground latency/throughput measurement"]
     },
     {
       "gate": "P2",
@@ -384,12 +308,7 @@
       "lane": "scheduler-pressure",
       "status": "pending",
       "description": "Measured two-hour pressure/heal capacity and recovery window",
-      "requires": ["two-hour pressure measurement", "heal capacity measurement", "recovery-window measurement"],
-      "evidence_fields": [
-        "two_hour_pressure_measurement",
-        "heal_capacity_measurement",
-        "recovery_window_measurement"
-      ]
+      "requires": ["two-hour pressure measurement", "heal capacity measurement", "recovery-window measurement"]
     },
     {
       "gate": "P4",
@@ -399,9 +318,6 @@
       "description": "Measured MRF scale and replay cost with retained responsibility",
       "requires": ["MRF scale measurement", "MRF replay-cost measurement", "retained responsibility evidence", "cleanup/GC soak evidence"],
       "evidence_fields": [
-        "mrf_scale_measurement",
-        "mrf_replay_cost_measurement",
-        "retained_responsibility_evidence",
         "mrf_cleanup_gc_soak_evidence"
       ]
     },
@@ -411,12 +327,7 @@
       "lane": "checkpoint-and-crash",
       "status": "pending",
       "description": "Fixed-budget real process restart through enumeration and classification",
-      "requires": ["fixed-budget restart evidence", "enumeration evidence", "classification evidence"],
-      "evidence_fields": [
-        "fixed_budget_restart_evidence",
-        "enumeration_evidence",
-        "classification_evidence"
-      ]
+      "requires": ["fixed-budget restart evidence", "enumeration evidence", "classification evidence"]
     },
     {
       "gate": "R-D",
@@ -424,13 +335,7 @@
       "lane": "status-and-outcome",
       "status": "pending",
       "description": "Manager-to-event-to-ledger exact disposition, including grace",
-      "requires": ["manager disposition evidence", "event disposition evidence", "ledger disposition evidence", "grace handling"],
-      "evidence_fields": [
-        "manager_disposition_evidence",
-        "event_disposition_evidence",
-        "ledger_disposition_evidence",
-        "grace_handling"
-      ]
+      "requires": ["manager disposition evidence", "event disposition evidence", "ledger disposition evidence", "grace handling"]
     },
     {
       "gate": "R-L",
@@ -438,12 +343,7 @@
       "lane": "mixed-version-rollback",
       "status": "pending",
       "description": "Legacy source conflicts, migration gaps and crash-safe source retirement",
-      "requires": ["legacy source-conflict evidence", "migration-gap evidence", "crash-safe source retirement evidence"],
-      "evidence_fields": [
-        "legacy_source_conflict_evidence",
-        "migration_gap_evidence",
-        "crash_safe_source_retirement_evidence"
-      ]
+      "requires": ["legacy source-conflict evidence", "migration-gap evidence", "crash-safe source retirement evidence"]
     }
   ]
 }

.docker/compose/docker-compose.cluster.local-build.profiling-amd64.yml

@@ -43,7 +43,6 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
-      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       # `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
       # should show richer nested spans during request-path verification.
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
@@ -90,7 +89,6 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
-      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       # `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
       # should show richer nested spans during request-path verification.
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
@@ -137,7 +135,6 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
-      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       # `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
       # should show richer nested spans during request-path verification.
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
@@ -184,7 +181,6 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
-      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       # `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
       # should show richer nested spans during request-path verification.
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}

.docker/compose/docker-compose.cluster.local-build.yml

@@ -26,7 +26,6 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
-      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
       - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
       - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
@@ -73,7 +72,6 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
-      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
       - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
       - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
@@ -120,7 +118,6 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
-      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
       - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
       - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
@@ -167,7 +164,6 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
-      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
       - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
       - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}

.docker/compose/docker-compose.observability.yaml

@@ -201,7 +201,6 @@ services:
       - RUSTFS_ADDRESS=:9000
       - RUSTFS_CONSOLE_ENABLE=true
       - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
-      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:
@@ -221,7 +220,6 @@ services:
       - RUSTFS_ADDRESS=:9000
       - RUSTFS_CONSOLE_ENABLE=true
       - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
-      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:
@@ -241,7 +239,6 @@ services:
       - RUSTFS_ADDRESS=:9000
       - RUSTFS_CONSOLE_ENABLE=true
       - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
-      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:
@@ -261,7 +258,6 @@ services:
       - RUSTFS_ADDRESS=:9000
       - RUSTFS_CONSOLE_ENABLE=true
       - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
-      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:

.docker/observability/README.md

@@ -47,10 +47,6 @@ Three pre-built Grafana dashboards are included for monitoring RustFS GET perfor
 | **GET Resource Impact** | `grafana-get-resource-impact.json` | Monitors resource usage: concurrent requests, IO queue utilization, disk permit wait, RSS trend |
 | **Object Data Cache** | `grafana-object-data-cache.json` | Monitors the GET body cache (`rustfs_object_data_cache_*`): hit ratio, lookup/plan/fill outcomes, fill duration quantiles, hit vs fill throughput, entries/weighted bytes, inflight fills, memory-pressure skips, invalidations, and size-class breakdowns |
 
-### Storage Metrics
-
-Storage panels require `prometheus-rules/rustfs-storage.yml` and the cluster resource attribute. See the [storage metrics guide](../../docs/operations/storage-metrics.md) for ownership, observer selection, freshness, and rolling upgrades.
-
 ### Prometheus Alert Rules
 
 The file `prometheus-rules/rustfs-get-optimization-alerts.yaml` contains pre-configured alerting rules:

.docker/observability/docker-compose-example-for-rustfs.yml

@@ -32,7 +32,6 @@ services:
       - RUSTFS_SECRET_KEY=rustfsadmin
       - RUSTFS_OBS_LOGGER_LEVEL=info
       - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
-      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_PROFILING_ENDPOINT=http://pyroscope:4040
     volumes:
       - rustfs-data:/data/rustfs

.docker/observability/grafana/dashboards/rustfs.json

@@ -223,15 +223,14 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_buckets_total\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
+          "expr": "sum(rustfs_cluster_buckets_total{job=~\"$job\"})",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
         }
       ],
       "title": "Total Buckets",
-      "type": "stat",
-      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
+      "type": "stat"
     },
     {
       "datasource": {
@@ -290,15 +289,14 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_objects_total\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
+          "expr": "sum(rustfs_cluster_objects_total{job=~\"$job\"})",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
         }
       ],
       "title": "Total Objects",
-      "type": "stat",
-      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
+      "type": "stat"
     },
     {
       "datasource": {
@@ -429,7 +427,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_used_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
+          "expr": "sum(rustfs_cluster_capacity_used_bytes{job=~\"$job\"})",
           "legendFormat": "Used",
           "range": true,
           "refId": "A"
@@ -440,7 +438,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_raw_total_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
+          "expr": "sum(rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})",
           "hide": false,
           "legendFormat": "Total",
           "range": true,
@@ -452,7 +450,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_used_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"} / ignoring(source_metric) rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_raw_total_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
+          "expr": "sum(rustfs_cluster_capacity_used_bytes{job=~\"$job\"}) / sum(rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})",
           "hide": false,
           "instant": false,
           "legendFormat": "Percent",
@@ -461,8 +459,7 @@
         }
       ],
       "title": "Capacity",
-      "type": "stat",
-      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
+      "type": "stat"
     },
     {
       "datasource": {
@@ -528,7 +525,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_stale_drives\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
+          "expr": "sum(rustfs_cluster_capacity_stale_drives{job=~\"$job\"})",
           "legendFormat": "Stale Drives",
           "range": true,
           "refId": "A"
@@ -539,15 +536,14 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_missing_drives\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
+          "expr": "sum(rustfs_cluster_capacity_missing_drives{job=~\"$job\"})",
           "legendFormat": "Missing Drives",
           "range": true,
           "refId": "B"
         }
       ],
       "title": "Capacity Observation",
-      "type": "stat",
-      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
+      "type": "stat"
     },
     {
       "datasource": {
@@ -1993,8 +1989,8 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}",
-          "legendFormat": "{{server}} | {{drive}} (bytes)",
+          "expr": "sum by (drive) (rustfs_system_drive_used_bytes{job=~\"$job\", drive=~\"$drive\"})",
+          "legendFormat": "{{drive}} (bytes)",
           "range": true,
           "refId": "A"
         },
@@ -2004,17 +2000,16 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"} / ignoring(source_metric) rustfs:storage:current{source_metric=\"rustfs_system_drive_total_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}",
+          "expr": "sum by (drive) (rustfs_system_drive_used_bytes{job=~\"$job\", drive=~\"$drive\"}) / sum by (drive)(rustfs_system_drive_total_bytes{job=~\"$job\", drive=~\"$drive\"})",
           "hide": false,
           "instant": false,
-          "legendFormat": "{{server}} | {{drive}} (percent)",
+          "legendFormat": "{{drive}} (percent)",
           "range": true,
           "refId": "B"
         }
       ],
       "title": "System Drive Usage",
-      "type": "timeseries",
-      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
+      "type": "timeseries"
     },
     {
       "datasource": {
@@ -2102,15 +2097,14 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_capacity_observation_age_seconds\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}",
-          "legendFormat": "{{server}} | {{drive}}",
+          "expr": "max by (drive) (rustfs_system_drive_capacity_observation_age_seconds{job=~\"$job\", drive=~\"$drive\"})",
+          "legendFormat": "{{drive}}",
           "range": true,
           "refId": "A"
         }
       ],
       "title": "Drive Capacity Observation Age",
-      "type": "timeseries",
-      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
+      "type": "timeseries"
     },
     {
       "datasource": {
@@ -2196,8 +2190,8 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_capacity_observation_state\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",state=\"stale\"}",
-          "legendFormat": "{{server}} | {{drive}} stale",
+          "expr": "max by (drive) (rustfs_system_drive_capacity_observation_state{job=~\"$job\", drive=~\"$drive\", state=\"stale\"})",
+          "legendFormat": "{{drive}} stale",
           "range": true,
           "refId": "A"
         },
@@ -2207,15 +2201,14 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_capacity_observation_state\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",state=\"missing\"}",
-          "legendFormat": "{{server}} | {{drive}} missing",
+          "expr": "max by (drive) (rustfs_system_drive_capacity_observation_state{job=~\"$job\", drive=~\"$drive\", state=\"missing\"})",
+          "legendFormat": "{{drive}} missing",
           "range": true,
           "refId": "B"
         }
       ],
       "title": "Drive Capacity Observation State",
-      "type": "timeseries",
-      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
+      "type": "timeseries"
     },
     {
       "datasource": {
@@ -4558,7 +4551,7 @@
                   "index": 0,
                   "text": "INACTIVE"
                 },
-                "to": 1e-09
+                "to": 1e-9
               },
               "type": "range"
             }
@@ -6570,7 +6563,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_health_drives_online_count\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
+          "expr": "rustfs_cluster_health_drives_online_count{job=~\"$job\"}",
           "legendFormat": "online - {{job}}",
           "range": true,
           "refId": "A"
@@ -6581,7 +6574,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_health_drives_offline_count\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
+          "expr": "rustfs_cluster_health_drives_offline_count{job=~\"$job\"}",
           "legendFormat": "offline - {{job}}",
           "range": true,
           "refId": "B"
@@ -6592,15 +6585,14 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_health_drives_count\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
+          "expr": "rustfs_cluster_health_drives_count{job=~\"$job\"}",
           "legendFormat": "total - {{job}}",
           "range": true,
           "refId": "C"
         }
       ],
       "title": "Cluster Drive Health Counts",
-      "type": "timeseries",
-      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
+      "type": "timeseries"
     },
     {
       "collapsed": false,
@@ -8562,15 +8554,14 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs:storage:current{source_metric=~\"rustfs_system_drive_.*\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}",
-          "legendFormat": "{{server}} | {{source_metric}} | {{drive}}",
+          "expr": "{__name__=~\"rustfs_system_drive_.*\",job=~\"$job\",drive=~\"$drive\"}",
+          "legendFormat": "{{__name__}} | {{drive}}",
           "range": true,
           "refId": "A"
         }
       ],
       "title": "System Drive (All)",
-      "type": "timeseries",
-      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
+      "type": "timeseries"
     },
     {
       "datasource": {
@@ -8951,15 +8942,14 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs:storage:current{source_metric=~\"rustfs_cluster_erasure_set_.*\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
-          "legendFormat": "{{source_metric}}",
+          "expr": "{__name__=~\"rustfs_cluster_erasure_set_.*\",job=~\"$job\"}",
+          "legendFormat": "{{__name__}}",
           "range": true,
           "refId": "A"
         }
       ],
       "title": "Cluster Erasure Set (All)",
-      "type": "timeseries",
-      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
+      "type": "timeseries"
     },
     {
       "datasource": {
@@ -11726,7 +11716,7 @@
           "editorMode": "code",
           "range": true,
           "refId": "A",
-          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_drive_runtime_state\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\",drive=~\"$drive\"}",
+          "expr": "max by (server, drive, pool_index, set_index, drive_index, state) (rustfs_system_drive_runtime_state{job=~\"$job\",server=~\"$server\",drive=~\"$drive\"})",
           "legendFormat": "{{server}} | {{drive}} | p{{pool_index}}/s{{set_index}}/d{{drive_index}} | {{state}}"
         },
         {
@@ -11737,13 +11727,12 @@
           "editorMode": "code",
           "range": true,
           "refId": "B",
-          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_drive_offline_duration_seconds\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\",drive=~\"$drive\"}",
+          "expr": "max by (server, drive, pool_index, set_index, drive_index) (rustfs_system_drive_offline_duration_seconds{job=~\"$job\",server=~\"$server\",drive=~\"$drive\"})",
           "legendFormat": "{{server}} | {{drive}} | offline seconds"
         }
       ],
-      "title": "Observed Cluster Drive State and Offline Duration",
-      "type": "timeseries",
-      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
+      "title": "Drive Runtime State and Offline Duration",
+      "type": "timeseries"
     },
     {
       "datasource": {
@@ -11836,13 +11825,12 @@
           "editorMode": "code",
           "range": true,
           "refId": "A",
-          "expr": "rate(rustfs_system_drive_api_calls_total{collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}[$__rate_interval]) and ignoring(source_metric) rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}",
+          "expr": "sum by (server, drive, pool_index, set_index, drive_index, api) (rate(rustfs_system_drive_api_calls_total{job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}[$__rate_interval]))",
           "legendFormat": "{{server}} | {{drive}} | p{{pool_index}}/s{{set_index}}/d{{drive_index}} | {{api}}"
         }
       ],
       "title": "Drive API Calls by Operation",
-      "type": "timeseries",
-      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
+      "type": "timeseries"
     },
     {
       "datasource": {
@@ -12388,50 +12376,6 @@
         "sort": 1,
         "type": "query"
       },
-      {
-        "current": {},
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${datasource}"
-        },
-        "definition": "label_values(rustfs:storage_snapshot:fresh, rustfs_cluster_id)",
-        "includeAll": false,
-        "label": "Storage cluster",
-        "multi": false,
-        "name": "storage_cluster",
-        "options": [],
-        "query": {
-          "qryType": 1,
-          "query": "label_values(rustfs:storage_snapshot:fresh, rustfs_cluster_id)",
-          "refId": "PrometheusVariableQueryEditor-storage_cluster"
-        },
-        "refresh": 2,
-        "regex": "",
-        "sort": 1,
-        "type": "query"
-      },
-      {
-        "current": {},
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${datasource}"
-        },
-        "definition": "query_result(rustfs:storage_snapshot:fresh{collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\"})",
-        "includeAll": false,
-        "label": "Cluster observer",
-        "multi": false,
-        "name": "storage_observer",
-        "options": [],
-        "query": {
-          "qryType": 3,
-          "query": "query_result(rustfs:storage_snapshot:fresh{collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\"})",
-          "refId": "PrometheusVariableQueryEditor-storage_observer"
-        },
-        "refresh": 2,
-        "regex": "/observer=\"([^\"]+)\"/",
-        "sort": 1,
-        "type": "query"
-      },
       {
         "allValue": ".*",
         "current": {
@@ -12442,7 +12386,7 @@
           "type": "prometheus",
           "uid": "${datasource}"
         },
-        "definition": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, api)",
+        "definition": "label_values(rustfs_system_drive_api_calls_total,api)",
         "includeAll": true,
         "label": "Drive API",
         "multi": true,
@@ -12450,7 +12394,7 @@
         "options": [],
         "query": {
           "qryType": 1,
-          "query": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, api)",
+          "query": "label_values(rustfs_system_drive_api_calls_total,api)",
           "refId": "PrometheusVariableQueryEditor-drive_api"
         },
         "refresh": 2,
@@ -12511,7 +12455,7 @@
           "text": "All",
           "value": "$__all"
         },
-        "definition": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, drive)",
+        "definition": "label_values(rustfs_system_drive_used_bytes,drive)",
         "includeAll": true,
         "label": "Drive",
         "multi": true,
@@ -12519,7 +12463,7 @@
         "options": [],
         "query": {
           "qryType": 1,
-          "query": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, drive)",
+          "query": "label_values(rustfs_system_drive_used_bytes,drive)",
           "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 2,

.docker/observability/prometheus-rules/rustfs-storage.yml

@@ -1,575 +0,0 @@
-# Preserve OTLP timestamps in the Collector. Apply timestamp() directly to
-# each raw selector, before label rewriting; functions such as label_replace()
-# replace the evaluation timestamp and would make a cached value look new.
-# Compare publication times at Prometheus millisecond precision so points
-# published and exported within the same millisecond are not withheld.
-groups:
-  - name: rustfs-storage-snapshots
-    interval: 15s
-    rules:
-      - record: rustfs:storage_snapshot:fresh
-        expr: |
-          rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"}
-          and ((time() - rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"}) <= rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"})
-          and ((time() - rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"}) >= 0)
-          and (rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"} > 0)
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_buckets_total
-        expr: |
-          rustfs_cluster_buckets_total{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_buckets_total{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_capacity_free_bytes
-        expr: |
-          rustfs_cluster_capacity_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_capacity_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_capacity_missing_drives
-        expr: |
-          rustfs_cluster_capacity_missing_drives{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_capacity_missing_drives{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_capacity_raw_total_bytes
-        expr: |
-          rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_capacity_stale_drives
-        expr: |
-          rustfs_cluster_capacity_stale_drives{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_capacity_stale_drives{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_capacity_usable_total_bytes
-        expr: |
-          rustfs_cluster_capacity_usable_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_capacity_usable_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_capacity_used_bytes
-        expr: |
-          rustfs_cluster_capacity_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_capacity_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_drive_capacity_observation_age_seconds
-        expr: |
-          rustfs_cluster_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_drive_capacity_observation_state
-        expr: |
-          rustfs_cluster_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_drive_free_bytes
-        expr: |
-          rustfs_cluster_drive_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_drive_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_drive_offline_duration_seconds
-        expr: |
-          rustfs_cluster_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_drive_present
-        expr: |
-          rustfs_cluster_drive_present{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_drive_present{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_drive_runtime_state
-        expr: |
-          rustfs_cluster_drive_runtime_state{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_drive_runtime_state{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_drive_total_bytes
-        expr: |
-          rustfs_cluster_drive_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_drive_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_drive_used_bytes
-        expr: |
-          rustfs_cluster_drive_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_drive_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_erasure_set_data_shards
-        expr: |
-          rustfs_cluster_erasure_set_data_shards{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_erasure_set_data_shards{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_erasure_set_healing_drives_count
-        expr: |
-          rustfs_cluster_erasure_set_healing_drives_count{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_erasure_set_healing_drives_count{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_erasure_set_health
-        expr: |
-          rustfs_cluster_erasure_set_health{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_erasure_set_health{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_erasure_set_online_drives_count
-        expr: |
-          rustfs_cluster_erasure_set_online_drives_count{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_erasure_set_online_drives_count{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_erasure_set_overall_health
-        expr: |
-          rustfs_cluster_erasure_set_overall_health{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_erasure_set_overall_health{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_erasure_set_overall_write_quorum
-        expr: |
-          rustfs_cluster_erasure_set_overall_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_erasure_set_overall_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_erasure_set_parity
-        expr: |
-          rustfs_cluster_erasure_set_parity{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_erasure_set_parity{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_erasure_set_read_health
-        expr: |
-          rustfs_cluster_erasure_set_read_health{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_erasure_set_read_health{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_erasure_set_read_quorum
-        expr: |
-          rustfs_cluster_erasure_set_read_quorum{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_erasure_set_read_quorum{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_erasure_set_read_tolerance
-        expr: |
-          rustfs_cluster_erasure_set_read_tolerance{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_erasure_set_read_tolerance{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_erasure_set_size
-        expr: |
-          rustfs_cluster_erasure_set_size{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_erasure_set_size{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_erasure_set_write_health
-        expr: |
-          rustfs_cluster_erasure_set_write_health{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_erasure_set_write_health{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_erasure_set_write_quorum
-        expr: |
-          rustfs_cluster_erasure_set_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_erasure_set_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_erasure_set_write_tolerance
-        expr: |
-          rustfs_cluster_erasure_set_write_tolerance{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_erasure_set_write_tolerance{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_health_drives_count
-        expr: |
-          rustfs_cluster_health_drives_count{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_health_drives_count{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_health_drives_offline_count
-        expr: |
-          rustfs_cluster_health_drives_offline_count{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_health_drives_offline_count{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_health_drives_online_count
-        expr: |
-          rustfs_cluster_health_drives_online_count{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_health_drives_online_count{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_cluster_objects_total
-        expr: |
-          rustfs_cluster_objects_total{rustfs_cluster_id!="",collection_scope="cluster"}
-          and (timestamp(rustfs_cluster_objects_total{rustfs_cluster_id!="",collection_scope="cluster"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_node_disk_free_bytes
-        expr: |
-          rustfs_node_disk_free_bytes{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_node_disk_free_bytes{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_node_disk_total_bytes
-        expr: |
-          rustfs_node_disk_total_bytes{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_node_disk_total_bytes{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_node_disk_used_bytes
-        expr: |
-          rustfs_node_disk_used_bytes{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_node_disk_used_bytes{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_api_calls_total
-        expr: |
-          rustfs_system_drive_api_calls_total{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_api_calls_total{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_api_latency_by_api_micros
-        expr: |
-          rustfs_system_drive_api_latency_by_api_micros{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_api_latency_by_api_micros{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_api_latency_micros
-        expr: |
-          rustfs_system_drive_api_latency_micros{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_api_latency_micros{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_availability_errors_total
-        expr: |
-          rustfs_system_drive_availability_errors_total{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_availability_errors_total{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_capacity_observation_age_seconds
-        expr: |
-          rustfs_system_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_capacity_observation_state
-        expr: |
-          rustfs_system_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_count
-        expr: |
-          rustfs_system_drive_count{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_count{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_deletes_total
-        expr: |
-          rustfs_system_drive_deletes_total{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_deletes_total{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_free_bytes
-        expr: |
-          rustfs_system_drive_free_bytes{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_free_bytes{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_free_inodes
-        expr: |
-          rustfs_system_drive_free_inodes{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_free_inodes{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_healing
-        expr: |
-          rustfs_system_drive_healing{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_healing{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_health
-        expr: |
-          rustfs_system_drive_health{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_health{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_info
-        expr: |
-          rustfs_system_drive_info{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_info{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_io_errors_total
-        expr: |
-          rustfs_system_drive_io_errors_total{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_io_errors_total{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_offline_count
-        expr: |
-          rustfs_system_drive_offline_count{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_offline_count{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_offline_duration_seconds
-        expr: |
-          rustfs_system_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_online_count
-        expr: |
-          rustfs_system_drive_online_count{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_online_count{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_perc_util
-        expr: |
-          rustfs_system_drive_perc_util{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_perc_util{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_present
-        expr: |
-          rustfs_system_drive_present{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_present{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_reads_await
-        expr: |
-          rustfs_system_drive_reads_await{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_reads_await{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_reads_kb_per_sec
-        expr: |
-          rustfs_system_drive_reads_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_reads_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_reads_per_sec
-        expr: |
-          rustfs_system_drive_reads_per_sec{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_reads_per_sec{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_runtime_state
-        expr: |
-          rustfs_system_drive_runtime_state{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_runtime_state{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_scanning
-        expr: |
-          rustfs_system_drive_scanning{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_scanning{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_timeout_errors_total
-        expr: |
-          rustfs_system_drive_timeout_errors_total{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_timeout_errors_total{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_total_bytes
-        expr: |
-          rustfs_system_drive_total_bytes{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_total_bytes{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_total_inodes
-        expr: |
-          rustfs_system_drive_total_inodes{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_total_inodes{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_used_bytes
-        expr: |
-          rustfs_system_drive_used_bytes{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_used_bytes{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_used_inodes
-        expr: |
-          rustfs_system_drive_used_inodes{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_used_inodes{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_waiting_io
-        expr: |
-          rustfs_system_drive_waiting_io{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_waiting_io{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_writes_await
-        expr: |
-          rustfs_system_drive_writes_await{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_writes_await{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_writes_kb_per_sec
-        expr: |
-          rustfs_system_drive_writes_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_writes_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_writes_per_sec
-        expr: |
-          rustfs_system_drive_writes_per_sec{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_writes_per_sec{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
-      - record: rustfs:storage:current
-        labels:
-          source_metric: rustfs_system_drive_writes_total
-        expr: |
-          rustfs_system_drive_writes_total{rustfs_cluster_id!="",collection_scope="local"}
-          and (timestamp(rustfs_system_drive_writes_total{rustfs_cluster_id!="",collection_scope="local"})
-            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
-            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))

.docker/observability/tests/storage-rules.test.yml

@@ -1,189 +0,0 @@
-rule_files:
-- ../prometheus-rules/rustfs-storage.yml
-evaluation_interval: 15s
-tests:
-- name: owners, pools, clusters and observer views remain distinct in mixed versions
-  interval: 1m
-  input_series:
-  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
-    values: 0+60x8
-  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
-    values: 180x8
-  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",pool_index="0",set_index="0",drive_index="0"}
-    values: 100x8
-  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
-    values: 0+60x8
-  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
-    values: 180x8
-  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data",pool_index="0",set_index="0",drive_index="1"}
-    values: 100x8
-  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n2",collection_scope="local",job="rustfs",instance="collector"}
-    values: 0+60x8
-  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n2",collection_scope="local",job="rustfs",instance="collector"}
-    values: 180x8
-  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n2",collection_scope="local",job="rustfs",instance="collector",server="n2",drive="/data",pool_index="1",set_index="0",drive_index="0"}
-    values: 100x8
-  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n3",collection_scope="local",job="rustfs",instance="collector"}
-    values: 0+60x8
-  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n3",collection_scope="local",job="rustfs",instance="collector"}
-    values: 180x8
-  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n3",collection_scope="local",job="rustfs",instance="collector",server="n3",drive="/data",pool_index="1",set_index="0",drive_index="1"}
-    values: 100x8
-  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
-    values: 0+60x8
-  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
-    values: 180x8
-  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",pool_index="0",set_index="0",drive_index="0"}
-    values: 1000x8
-  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
-    values: 0+60x8
-  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
-    values: 180x8
-  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data",pool_index="0",set_index="0",drive_index="1"}
-    values: 1000x8
-  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n2",collection_scope="local",job="rustfs",instance="collector"}
-    values: 0+60x8
-  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n2",collection_scope="local",job="rustfs",instance="collector"}
-    values: 180x8
-  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n2",collection_scope="local",job="rustfs",instance="collector",server="n2",drive="/data",pool_index="1",set_index="0",drive_index="0"}
-    values: 1000x8
-  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n3",collection_scope="local",job="rustfs",instance="collector"}
-    values: 0+60x8
-  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n3",collection_scope="local",job="rustfs",instance="collector"}
-    values: 180x8
-  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n3",collection_scope="local",job="rustfs",instance="collector",server="n3",drive="/data",pool_index="1",set_index="0",drive_index="1"}
-    values: 1000x8
-  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="cluster",job="rustfs",instance="collector"}
-    values: 0+60x8
-  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="cluster",job="rustfs",instance="collector"}
-    values: 180x8
-  - series: rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id="a",observer="n0",collection_scope="cluster",job="rustfs",instance="collector"}
-    values: 400x8
-  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="cluster",job="rustfs",instance="collector"}
-    values: 0+60x8
-  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="cluster",job="rustfs",instance="collector"}
-    values: 180x8
-  - series: rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id="a",observer="n1",collection_scope="cluster",job="rustfs",instance="collector"}
-    values: 300x8
-  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="old",server="n0",drive="/data",job="rustfs",instance="collector"}
-    values: 999x8
-  promql_expr_test:
-  - expr: sum by (rustfs_cluster_id) (rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes",collection_scope="local"})
-    eval_time: 2m
-    exp_samples:
-    - labels: '{rustfs_cluster_id="a"}'
-      value: 400
-    - labels: '{rustfs_cluster_id="b"}'
-      value: 4000
-  - expr: count by (rustfs_cluster_id,pool_index) (rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes"})
-    eval_time: 2m
-    exp_samples:
-    - labels: '{rustfs_cluster_id="a",pool_index="0"}'
-      value: 2
-    - labels: '{rustfs_cluster_id="a",pool_index="1"}'
-      value: 2
-    - labels: '{rustfs_cluster_id="b",pool_index="0"}'
-      value: 2
-    - labels: '{rustfs_cluster_id="b",pool_index="1"}'
-      value: 2
-  - expr: sum(rustfs:storage:current{source_metric="rustfs_cluster_capacity_raw_total_bytes",rustfs_cluster_id="a",observer="n0"})
-    eval_time: 2m
-    exp_samples:
-    - labels: '{}'
-      value: 400
-  - expr: sum(rustfs:storage:current{source_metric="rustfs_cluster_capacity_raw_total_bytes",rustfs_cluster_id="a",observer="n1"})
-    eval_time: 2m
-    exp_samples:
-    - labels: '{}'
-      value: 300
-- name: removed values and identities cannot rejoin new snapshots; stalled sources expire
-  interval: 1m
-  input_series:
-  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
-    values: 0+60x8
-  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
-    values: 180x8
-  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
-    values: 0 60 _ _ _ _ _ _ _
-  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
-    values: 180x8
-  - series: rustfs_system_drive_info{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",disk_id="old"}
-    values: 1 1 _ _ _ _ _ _ _
-  - series: rustfs_system_drive_info{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",disk_id="new"}
-    values: _ _ 1 1 1 1 1 1 1
-  - series: rustfs_system_drive_waiting_io{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data"}
-    values: 7 7 _ _ _ _ _ _ _
-  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data"}
-    values: 100 100 _ _ _ _ _ _ _
-  promql_expr_test:
-  - expr: rustfs:storage:current{source_metric="rustfs_system_drive_info",disk_id="old"}
-    eval_time: 2m
-    exp_samples: []
-  - expr: rustfs:storage:current{source_metric="rustfs_system_drive_waiting_io"}
-    eval_time: 2m
-    exp_samples: []
-  - expr: count(rustfs:storage:current{source_metric="rustfs_system_drive_info",disk_id="new"})
-    eval_time: 2m
-    exp_samples:
-    - labels: '{}'
-      value: 1
-  - expr: rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes"}
-    eval_time: 5m
-    exp_samples: []
-  - expr: rustfs:storage_snapshot:fresh{observer="n1"}
-    eval_time: 5m
-    exp_samples: []
-- name: counters reset independently; filter versions before rate and sum rates across owners
-  interval: 1m
-  input_series:
-  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
-    values: 0+60x8
-  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
-    values: 180x8
-  - series: rustfs_system_drive_api_calls_total{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",disk_id="n0-disk",api="read_all"}
-    values: 0 60 120 30 90 150 210
-  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
-    values: 0+60x8
-  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
-    values: 180x8
-  - series: rustfs_system_drive_api_calls_total{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data",disk_id="n1-disk",api="read_all"}
-    values: 0+120x6
-  - series: rustfs_system_drive_api_calls_total{rustfs_cluster_id="a",server="n0",drive="/data",observer="old"}
-    values: 0+999x6
-  promql_expr_test:
-  - expr: sum(resets(rustfs_system_drive_api_calls_total{collection_scope="local"}[5m]))
-    eval_time: 5m
-    exp_samples:
-    - labels: '{}'
-      value: 1
-  - expr: sum(rate(rustfs_system_drive_api_calls_total{collection_scope="local"}[2m]) and ignoring(source_metric) rustfs:storage:current{source_metric="rustfs_system_drive_api_calls_total"})
-    eval_time: 5m
-    exp_samples:
-    - labels: '{}'
-      value: 3
-- name: completed slow collection has no validity budget
-  interval: 1m
-  input_series:
-  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="slow",collection_scope="cluster",job="rustfs",instance="collector"}
-    values: 0+60x3
-  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="slow",collection_scope="cluster",job="rustfs",instance="collector"}
-    values: '0x3'
-  promql_expr_test:
-  - expr: rustfs:storage_snapshot:fresh{observer="slow"}
-    eval_time: 2m
-    exp_samples: []
-- name: submillisecond publication and millisecond OTLP samples share a cutoff
-  interval: 1m
-  input_series:
-  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="submillisecond",collection_scope="local",job="rustfs",instance="collector"}
-    values: 0.0009+60x3
-  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="submillisecond",collection_scope="local",job="rustfs",instance="collector"}
-    values: 180x3
-  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="submillisecond",collection_scope="local",job="rustfs",instance="collector",server="submillisecond",drive="/data"}
-    values: 100x3
-  promql_expr_test:
-  - expr: count(rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes",observer="submillisecond"})
-    eval_time: 2m15s
-    exp_samples:
-    - labels: '{}'
-      value: 1

.github/actions/quick-checks/action.yml

@@ -111,6 +111,10 @@ runs:
       shell: bash
       run: ./scripts/check_no_planning_docs.sh
 
+    - name: Check CI paths stay in sync
+      shell: bash
+      run: ./scripts/check_ci_paths_sync.sh
+
     - name: Check io_uring lane --lib precondition
       shell: bash
       run: ./scripts/check_uring_lane_lib_only.sh

.github/s3tests/s3tests.conf

@@ -55,10 +55,6 @@ secret_key = ${S3_SECRET_KEY}
 
 ## replace with key id obtained when secret is created, or delete if KMS not tested
 #kms_keyid = 01234567-89ab-cdef-0123-456789abcdef
-#kms_keyid2 = fedcba98-7654-3210-fedc-ba9876543210
-
-## Expected service default for SSE-KMS requests without a key id; empty means none
-#kms_default_keyid =
 
 ## Storage classes
 #storage_classes = "LUKEWARM, FROZEN"

.github/workflows/ci-docs-only.yml

@@ -0,0 +1,79 @@
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Reports the existing required checks for paths excluded by ci.yml.
+# Mixed PRs can trigger both workflows; their Quick Checks jobs use one shared
+# action to keep validation coverage aligned. Keep this paths list in sync with
+# ci.yml's pull_request.paths-ignore via scripts/check_ci_paths_sync.sh.
+
+name: Continuous Integration (docs only)
+
+on:
+  pull_request:
+    types: [ opened, synchronize, reopened ]
+    branches: [ main, release ]
+    paths:
+      - "**.md"
+      - "docs/**"
+      - "deploy/**"
+      - "scripts/dev_*.sh"
+      - "scripts/probe.sh"
+      - "LICENSE*"
+      - ".gitignore"
+      - ".dockerignore"
+      - "README*"
+      - "**/*.png"
+      - "**/*.jpg"
+      - "**/*.svg"
+      - ".github/workflows/build.yml"
+      - ".github/workflows/docker.yml"
+      - ".github/workflows/audit.yml"
+      - "flake.lock"
+
+permissions:
+  contents: read
+
+jobs:
+  quick-checks:
+    name: Quick Checks
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Run shared quick checks
+        uses: ./.github/actions/quick-checks
+
+  test-and-lint:
+    name: Test and Lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      # Docs-only PRs skip the full code CI, but they are exactly where a
+      # planning-type document could be slipped in (git add -f bypasses
+      # .gitignore). Run the guard here so the required "Test and Lint" check
+      # stays meaningful for docs-only changes.
+      - name: Check no planning docs committed
+        run: ./scripts/check_no_planning_docs.sh
+
+      - name: Satisfy required check for docs-only changes
+        run: echo "Docs-only change — code CI is skipped by paths-ignore; planning-docs guard passed, reporting success for the required 'Test and Lint' check."

.github/workflows/ci.yml

@@ -37,6 +37,25 @@ on:
   pull_request:
     types: [ opened, synchronize, reopened, closed ]
     branches: [ main, release ]
+    # Keep this list in sync with the `paths` list in ci-docs-only.yml, which
+    # reports the required "Test and Lint" check for PRs skipped here.
+    paths-ignore:
+      - "**.md"
+      - "docs/**"
+      - "deploy/**"
+      - "scripts/dev_*.sh"
+      - "scripts/probe.sh"
+      - "LICENSE*"
+      - ".gitignore"
+      - ".dockerignore"
+      - "README*"
+      - "**/*.png"
+      - "**/*.jpg"
+      - "**/*.svg"
+      - ".github/workflows/build.yml"
+      - ".github/workflows/docker.yml"
+      - ".github/workflows/audit.yml"
+      - "flake.lock"
   merge_group:
     types: [ checks_requested ]
   schedule:
@@ -69,32 +88,6 @@ jobs:
       - name: Explain cancellation run
         run: echo "PR closed; this run only cancels older runs in the same concurrency group."
 
-  classify-changes:
-    name: Select CI scope
-    if: github.event_name != 'pull_request' || github.event.action != 'closed'
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    outputs:
-      mode: ${{ steps.scope.outputs.mode }}
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          fetch-depth: 2
-          persist-credentials: false
-      - name: Select scope using the base revision's policy
-        id: scope
-        env:
-          CI_BASE_SHA: ${{ github.event.pull_request.base.sha }}
-        run: |
-          if [[ "$GITHUB_EVENT_NAME" != "pull_request" ]]; then
-            printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
-          elif [[ "$CI_BASE_SHA" =~ ^[0-9a-f]{40}$ ]] && git show "$CI_BASE_SHA:scripts/ci_gate.py" > "$RUNNER_TEMP/ci-gate-base.py"; then
-            python3 -I "$RUNNER_TEMP/ci-gate-base.py" select
-          else
-            printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
-            echo "Base CI policy unavailable; running the full matrix."
-          fi
-
   typos:
     name: Typos
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
@@ -107,7 +100,7 @@ jobs:
       - name: Typos check with custom config file
         uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master
 
-  # Fail early with compile-free checks for every pull request.
+  # Fail early with compile-free checks shared with docs-only CI.
   quick-checks:
     name: Quick Checks
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
@@ -123,9 +116,9 @@ jobs:
         uses: ./.github/actions/quick-checks
 
   test-and-lint:
-    name: Workspace Test and Lint
-    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
-    needs: [ quick-checks, classify-changes ]
+    name: Test and Lint
+    if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
     runs-on: sm-standard-4
     timeout-minutes: 90
     env:
@@ -259,6 +252,9 @@ jobs:
           } > artifacts/test-and-lint/doctest-diagnostics.txt
           exit "${status}"
 
+      - name: Check offline enrollment E2E root boundary
+        run: ./scripts/check_offline_enrollment_e2e.sh
+
       - name: Upload test reports and diagnostics
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
@@ -293,35 +289,44 @@ jobs:
       - name: Run rebalance/decommission migration proofs
         run: ./scripts/check_migration_gate_count.sh
 
-  # The root boundary requires fresh CLI and integration-test builds. Give it
-  # its own time budget instead of sharing the workspace lint/test budget.
-  offline-enrollment-root-boundary:
-    name: Offline Enrollment Root Boundary
-    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
-    needs: [ quick-checks, classify-changes ]
-    runs-on: sm-standard-4
-    timeout-minutes: 90
-    env:
-      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      # Record the reason before this job completes as FAILURE. A separate
+      # dependent job cancels sibling lanes only after GitHub has preserved this
+      # required check's failure verdict.
+      - name: Annotate early-stop reason
+        if: >-
+          failure() && github.event_name == 'pull_request'
+          && github.event.pull_request.head.repo.full_name == github.repository
+        run: |
+          {
+            echo "## CI early-stop"
+            echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; a follow-up job will cancel sibling lanes to free runners."
+            echo "Sibling jobs showing **cancelled** were stopped by the early-stop follow-up, not by their own failure."
+          } >> "$GITHUB_STEP_SUMMARY"
+
+  # Preserve the required Test and Lint FAILURE verdict before stopping sibling
+  # lanes. Cancelling from inside test-and-lint changed its own conclusion to
+  # CANCELLED and hid the actionable failure in the PR checks UI.
+  cancel-after-test-and-lint-failure:
+    name: Cancel siblings after Test and Lint failure
+    if: >-
+      failure() && needs.test-and-lint.result == 'failure'
+      && github.event_name == 'pull_request'
+      && github.event.pull_request.head.repo.full_name == github.repository
+    needs: [ test-and-lint ]
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      actions: write
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          rust-version: stable
-          cache-shared-key: ci-dev
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
-
-      - name: Protect Connect test home
-        run: chmod go-w "$(realpath "$HOME")"
-
-      - name: Check offline enrollment E2E root boundary
-        run: ./scripts/check_offline_enrollment_e2e.sh
+      - name: Cancel remaining jobs
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          curl -fsS -X POST \
+            -H "Authorization: Bearer ${GH_TOKEN}" \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel"
 
   # Dedicated serial lane for the ILM / lifecycle integration tests. These tests
   # drive the object layer through process-global singletons (the GLOBAL_ENV
@@ -335,8 +340,8 @@ jobs:
   # See rustfs/backlog#1148 (ilm-1) and #1155.
   test-ilm-integration-serial:
     name: ILM Integration (serial)
-    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
-    needs: [ quick-checks, classify-changes ]
+    if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
     runs-on: sm-standard-4
     timeout-minutes: 90
     env:
@@ -403,8 +408,8 @@ jobs:
 
   test-and-lint-rio-v2:
     name: Test and Lint (rio-v2)
-    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
-    needs: [ quick-checks, classify-changes ]
+    if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
     runs-on: sm-standard-4
     timeout-minutes: 90
     env:
@@ -444,8 +449,8 @@ jobs:
 
   connect-short-credential-boundary:
     name: Connect Short Credential Boundary
-    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
-    needs: [ quick-checks, classify-changes ]
+    if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
     runs-on: sm-standard-4
     timeout-minutes: 60
     env:
@@ -502,8 +507,8 @@ jobs:
 
   test-and-lint-protocols:
     name: "Test and Lint (${{ matrix.features.name }})"
-    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
-    needs: [ quick-checks, classify-changes ]
+    if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
     runs-on: sm-standard-4
     timeout-minutes: 90
     strategy:
@@ -556,8 +561,8 @@ jobs:
 
   build-rustfs-debug-binary:
     name: Build RustFS Debug Binary
-    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
-    needs: [ quick-checks, classify-changes ]
+    if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
     runs-on: sm-standard-4
     timeout-minutes: 30
     env:
@@ -679,8 +684,8 @@ jobs:
     # job had neither, so each closed/merged PR really ran the whole io_uring
     # suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
     # 30662728539) and kept the cancellation run in progress for minutes.
-    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
-    needs: [ quick-checks, classify-changes ]
+    if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
     # GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
     # a container, applies no seccomp filter that would block io_uring_setup — so
     # the probe succeeds and the tests exercise the real UringBackend/FdCache/
@@ -997,7 +1002,6 @@ jobs:
       # debug binary; each test spawns its own rustfs server on a random port.
       - name: Run e2e full suite
         env:
-          CARGO_BIN_EXE_rustfs: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
           RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
           RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
           RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR: ${{ runner.temp }}/rustfs-startup-cas-evidence
@@ -1209,49 +1213,12 @@ jobs:
           if-no-files-found: ignore
           retention-days: 3
 
-  required-checks:
-    name: Test and Lint
-    if: always() && (github.event_name != 'pull_request' || github.event.action != 'closed')
-    needs:
-      - classify-changes
-      - typos
-      - quick-checks
-      - test-and-lint
-      - offline-enrollment-root-boundary
-      - test-ilm-integration-serial
-      - test-and-lint-rio-v2
-      - connect-short-credential-boundary
-      - test-and-lint-protocols
-      - build-rustfs-debug-binary
-      - uring-integration
-      - e2e-tests
-      - s3-implemented-tests
-      - s3-lifecycle-behavior-tests
-      - build-rustfs-debug-binary-rio-v2
-      - e2e-tests-rio-v2
-      - e2e-full
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-      - name: Require the expected result of every CI lane
-        env:
-          CI_NEEDS: ${{ toJSON(needs) }}
-        shell: bash
-        run: python3 scripts/ci_gate.py verify
-
   alert-on-failure:
     name: Alert on scheduled failure
     needs:
-      - classify-changes
-      - connect-short-credential-boundary
-      - required-checks
       - typos
       - quick-checks
       - test-and-lint
-      - offline-enrollment-root-boundary
       - test-ilm-integration-serial
       - test-and-lint-rio-v2
       - test-and-lint-protocols

.github/workflows/e2e-replication-nightly.yml

@@ -156,7 +156,6 @@ jobs:
 
       - name: Run cluster fault e2e nightly suite
         env:
-          CARGO_BIN_EXE_rustfs: ${{ github.workspace }}/target/debug/rustfs
           RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
         run: cargo nextest run --profile e2e-nightly -p e2e_test
 

.github/workflows/nightly-gnu.yml

@@ -19,27 +19,17 @@ on:
     - cron: "7 0 * * *"
       timezone: "Asia/Shanghai"
   workflow_dispatch:
-    inputs:
-      branch:
-        description: 'Branch/ref to build and publish as the nightly (empty = scheduled source, see NIGHTLY_BUILD_REF)'
-        required: false
-        default: ''
 
 permissions:
   contents: read
 
-# Scheduled builds follow the NIGHTLY_BRANCH repo variable so the channel can
-# be pointed at e.g. `release` for the GA cycle and back to `main` afterwards
-# without touching this file. Manual runs take the `branch` input, falling
-# back to the branch the run was dispatched from.
 concurrency:
-  group: nightly-gnu-build-${{ github.event_name }}-${{ github.event_name == 'schedule' && (vars.NIGHTLY_BRANCH || 'main') || (inputs.branch || github.ref_name) }}
+  group: nightly-gnu-build-main-${{ github.event_name }}
   cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' }}
 
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: 1
-  NIGHTLY_BUILD_REF: ${{ github.event_name == 'schedule' && (vars.NIGHTLY_BRANCH || 'main') || (inputs.branch || github.ref_name) }}
 
 jobs:
   build:
@@ -53,7 +43,6 @@ jobs:
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: ${{ env.NIGHTLY_BUILD_REF }}
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -163,104 +152,13 @@ jobs:
 
           fakeroot dpkg-deb --build "${PKG_DIR}"
           ls -lh "${DEB_FILE}"
-          echo "deb_date=${DEB_DATE}" >> "${GITHUB_OUTPUT}"
           echo "deb_file=${DEB_FILE}" >> "${GITHUB_OUTPUT}"
 
-      # Same packaging scheme as .github/workflows/package.yml (fpm), but from
-      # the locally built nightly binary instead of a release artifact, with a
-      # date-based version that mirrors the DEB.
-      - name: Build RPM package
-        id: rpm
-        shell: bash
-        env:
-          DEB_DATE: ${{ steps.deb.outputs.deb_date }}
-        run: |
-          set -euo pipefail
-
-          if ! command -v fpm >/dev/null 2>&1; then
-            SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-            ${SUDO} apt-get update -qq && ${SUDO} apt-get install -y -qq ruby ruby-dev build-essential rpm >/dev/null
-            ${SUDO} gem install fpm --no-document >/dev/null
-          fi
-
-          RPM_FILE="rustfs-nightly-${DEB_DATE}.rpm"
-          RPM_VERSION="0"
-          RPM_RELEASE="0.nightly.${DEB_DATE//-/.}"
-
-          echo "Building RPM: ${RPM_FILE} (version ${RPM_VERSION}-${RPM_RELEASE})"
-
-          # fpm wants the config file to exist before packaging.
-          mkdir -p ./tmp-pkg/etc/default
-          cat > ./tmp-pkg/etc/default/rustfs << 'ENVEOF'
-          # RustFS Environment Configuration
-          # See https://rustfs.com/docs/ for more information
-          # RUSTFS_VOLUMES=""
-          # RUSTFS_ROOT_USER=""
-          # RUSTFS_ROOT_PASSWORD=""
-          ENVEOF
-
-          fpm -s dir -t rpm \
-            --name rustfs \
-            --version "$RPM_VERSION" \
-            --iteration "$RPM_RELEASE" \
-            --architecture x86_64 \
-            --package "$RPM_FILE" \
-            --depends "glibc >= 2.31" \
-            --maintainer "RustFS Team <support@rustfs.com>" \
-            --description "High-performance distributed object storage" \
-            --url "https://rustfs.com" \
-            --license "Apache-2.0" \
-            --after-install <(cat << 'POSTINST'
-          #!/bin/bash
-          set -e
-          if ! getent passwd rustfs > /dev/null 2>&1; then
-              useradd -r -s /bin/false -d /opt/rustfs rustfs
-          fi
-          mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
-          chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
-          if [ -d /run/systemd/system ]; then
-              systemctl daemon-reload
-          fi
-          POSTINST
-            ) \
-            --before-remove <(cat << 'PRERM'
-          #!/bin/bash
-          set -e
-          if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
-              systemctl stop rustfs
-          fi
-          PRERM
-            ) \
-            --after-remove <(cat << 'POSTRM'
-          #!/bin/bash
-          set -e
-          if [ -d /run/systemd/system ]; then
-              systemctl daemon-reload
-          fi
-          POSTRM
-            ) \
-            --config-files /etc/default/rustfs \
-            "rustfs-nightly-${DEB_DATE}/usr/bin/rustfs=/usr/bin/rustfs" \
-            ./tmp-pkg/etc/default/rustfs=/etc/default/rustfs \
-            deploy/build/rustfs.service=/lib/systemd/system/rustfs.service \
-            LICENSE=/usr/share/doc/rustfs/LICENSE \
-            README.md=/usr/share/doc/rustfs/README.md
-
-          [[ -f "$RPM_FILE" ]] || { echo "RPM build failed"; exit 1; }
-          rpm -qpl "$RPM_FILE" | grep -Fx '/usr/bin/rustfs' >/dev/null
-          stat --printf='%n %s bytes\n' "$RPM_FILE"
-          echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
-
       - name: Upload DEB artifact
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
           name: ${{ steps.deb.outputs.deb_file }}
           path: ${{ steps.deb.outputs.deb_file }}
-      - name: Upload RPM artifact
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: ${{ steps.rpm.outputs.rpm_file }}
-          path: ${{ steps.rpm.outputs.rpm_file }}
           if-no-files-found: error
 
       # Persist the nightly deb on Cloudflare R2 (same channel as package.yml)
@@ -289,10 +187,11 @@ jobs:
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
-          # The candidate manifest must describe the tree that was actually
-          # built. With a ref override (NIGHTLY_BRANCH / dispatch input) that
-          # is not necessarily GITHUB_SHA, so always advertise HEAD.
           SOURCE_SHA="$(git rev-parse HEAD)"
+          if [[ "${SOURCE_SHA}" != "${GITHUB_SHA}" ]]; then
+            echo "Checkout SHA does not match the nightly build run" >&2
+            exit 1
+          fi
           DEB_SHA256="$(sha256sum "${DEB_FILE}" | cut -d ' ' -f 1)"
           CANDIDATE_KEY="artifacts/rustfs/packages/nightly/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${DEB_SHA256}/rustfs.deb"
           CANDIDATE_URL="https://dl.rustfs.com/${CANDIDATE_KEY}"
@@ -348,20 +247,6 @@ jobs:
           path: ${{ steps.publish.outputs.candidate_file }}
           if-no-files-found: error
 
-      # Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
-      # engineers can download and install the nightly directly. The branch is
-      # a single-commit orphan rewritten on every build, which keeps the repo
-      # small while the latest files stay reachable at stable raw URLs.
-      # Publish the deb/rpm pair as assets of the rolling `nightly` release on
-      # rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
-      - name: Publish packages to auto-testing release assets
-        env:
-          ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
-          RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
-          DEB_DATE: ${{ steps.deb.outputs.deb_date }}
-          BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
-        run: bash scripts/release/publish_nightly_assets.sh
   # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
   #
   # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
@@ -399,7 +284,6 @@ jobs:
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: ${{ env.NIGHTLY_BUILD_REF }}
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -488,7 +372,6 @@ jobs:
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: ${{ env.NIGHTLY_BUILD_REF }}
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup

.github/workflows/rustfs-kms-test.yml

@@ -4,9 +4,9 @@ on:
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test. Leave empty for nightly.'
+        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
         required: false
-        default: ''
+        default: '1.0.0-rc.4-preview.1'
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false

.github/workflows/rustfs-replication-test.yml

@@ -18,9 +18,9 @@ on:
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test. Leave empty for nightly.'
+        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
         required: false
-        default: ''
+        default: '1.0.0-rc.4-preview.1'
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false

.github/workflows/rustfs-s3-compat-test.yml

@@ -4,9 +4,9 @@ on:
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test. Leave empty for nightly.'
+        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
         required: false
-        default: ''
+        default: '1.0.0-rc.4-preview.1'
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false

.github/workflows/rustfs-security-test.yml

@@ -18,9 +18,9 @@ on:
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test. Leave empty for nightly.'
+        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
         required: false
-        default: ''
+        default: '1.0.0-rc.4-preview.1'
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false

.github/workflows/rustfs-storage-test.yml

@@ -4,9 +4,9 @@ on:
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test. Leave empty for nightly.'
+        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
         required: false
-        default: ''
+        default: '1.0.0-rc.4-preview.1'
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false

.github/workflows/rustfs-tier-test.yml

@@ -4,9 +4,9 @@ on:
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test. Leave empty for nightly.'
+        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
         required: false
-        default: ''
+        default: '1.0.0-rc.4-preview.1'
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false

.github/workflows/windows-filesystem.yml

@@ -16,7 +16,7 @@ name: Windows Filesystem Tests
 
 on:
   push:
-    branches: [ main, release ]
+    branches: [ main ]
     paths:
       - "crates/ecstore/src/disk/**"
       - "crates/ecstore/src/store/init_format.rs"
@@ -26,7 +26,7 @@ on:
       - ".github/actions/setup/**"
       - ".github/workflows/windows-filesystem.yml"
   pull_request:
-    branches: [ main, release ]
+    branches: [ main ]
     paths:
       - "crates/ecstore/src/disk/**"
       - "crates/ecstore/src/store/init_format.rs"

Cargo.toml

@@ -73,7 +73,7 @@ edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
 rust-version = "1.98.0"
-version = "1.0.0-rc.6"
+version = "1.0.0-rc.5"
 homepage = "https://rustfs.com"
 description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
 keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
@@ -90,62 +90,62 @@ redundant_clone = "warn"
 
 [workspace.dependencies]
 # RustFS Internal Crates
-rustfs = { path = "./rustfs", version = "1.0.0-rc.6" }
-rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.6" }
-rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.6" }
-rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.0-rc.6" }
-rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.6" }
-rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.6" }
-rustfs-common = { path = "crates/common", version = "1.0.0-rc.6" }
-rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.6" }
-rustfs-config = { path = "./crates/config", version = "1.0.0-rc.6" }
-rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.6" }
-rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.6" }
-rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.6" }
-rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.6" }
-rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.6" }
-rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.6" }
-rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.6" }
-rustfs-license = { path = "crates/license", version = "1.0.0-rc.6" }
-rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.6" }
-rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.6" }
-rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.6" }
-rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.6" }
-rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.6" }
-rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.6" }
-rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.6" }
-rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.6" }
-rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.6", default-features = false }
-rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.6" }
-rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.6" }
-rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.6" }
-rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.6" }
-rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.6" }
-rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.6" }
-rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.6" }
-rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.6" }
-rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.6" }
-rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.6" }
-rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.6" }
-rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.6" }
-rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.6" }
-rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.6" }
-rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.6" }
-rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.6" }
-rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.6" }
-rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.6" }
-rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.6" }
-rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.6" }
-rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.6" }
-rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.6" }
-rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.6" }
-rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.6" }
+rustfs = { path = "./rustfs", version = "1.0.0-rc.5" }
+rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.5" }
+rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.5" }
+rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.0-rc.5" }
+rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.5" }
+rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.5" }
+rustfs-common = { path = "crates/common", version = "1.0.0-rc.5" }
+rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.5" }
+rustfs-config = { path = "./crates/config", version = "1.0.0-rc.5" }
+rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.5" }
+rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.5" }
+rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.5" }
+rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.5" }
+rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.5" }
+rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.5" }
+rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.5" }
+rustfs-license = { path = "crates/license", version = "1.0.0-rc.5" }
+rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.5" }
+rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.5" }
+rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.5" }
+rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.5" }
+rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.5" }
+rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.5" }
+rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.5" }
+rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.5" }
+rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.5", default-features = false }
+rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.5" }
+rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.5" }
+rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.5" }
+rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.5" }
+rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.5" }
+rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.5" }
+rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.5" }
+rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.5" }
+rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.5" }
+rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.5" }
+rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.5" }
+rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.5" }
+rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.5" }
+rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.5" }
+rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.5" }
+rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.5" }
+rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.5" }
+rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.5" }
+rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.5" }
+rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.5" }
+rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.5" }
+rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.5" }
+rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.5" }
+rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.5" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
 mysql_async = { default-features = false, version = "0.37.1" }
-async-compression = { version = "0.4.46" }
+async-compression = { version = "0.4.44" }
 async-recursion = "1.1.1"
 async-trait = "0.1.92"
 async-nats = { version = "0.50.0", default-features = false }
@@ -156,7 +156,7 @@ futures-lite = "2.6.1"
 futures-util = "0.3.34"
 pollster = "1.0.1"
 pulsar = { default-features = false, version = "6.9.0" }
-lapin = { default-features = false, version = "4.11.0" }
+lapin = { default-features = false, version = "4.10.0" }
 hyper = { version = "1.11.1" }
 hyper-rustls = { default-features = false, version = "0.27.9" }
 hyper-util = { version = "0.1.20" }
@@ -164,7 +164,7 @@ http = "1.5.0"
 http-body = "1.1.0"
 http-body-util = "0.1.5"
 minlz = "1.2.3"
-reqwest = "0.13.5"
+reqwest = "0.13.4"
 rustfs-kafka-async = { version = "1.3.1" }
 socket2 = { version = "0.6.5" }
 tokio = { version = "1.53.1" }
@@ -211,7 +211,7 @@ openidconnect = { default-features = false, version = "4.0" }
 pbkdf2 = "0.13.0"
 p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
 rsa = { version = "=0.10.0-rc.18" }
-rustls = { default-features = false, version = "0.23.44" }
+rustls = { default-features = false, version = "0.23.43" }
 rustls-native-certs = "0.8"
 rustls-pki-types = "1.15.1"
 x509-parser = "0.18.1"
@@ -245,7 +245,7 @@ atomic_enum = "0.3.0"
 aws-config = { version = "1.12.0" }
 aws-credential-types = { version = "1.3.0" }
 aws-sdk-kms = { default-features = false, version = "1.118.0" }
-aws-sdk-s3 = { default-features = false, version = "1.146.0" }
+aws-sdk-s3 = { default-features = false, version = "1.145.0" }
 aws-sdk-sts = { default-features = false, version = "1.114.0" }
 aws-smithy-async = { version = "1.3.0" }
 aws-smithy-http-client = { default-features = false, version = "1.4.0" }
@@ -297,7 +297,7 @@ percent-encoding = "2.3.2"
 # Server-side QR rendering for TOTP enrollment, so neither the console nor the
 # CLI needs its own QR encoder. No default features: the image/render backends
 # pull in an image stack this only needs SVG and text output from.
-qrcode-rs = { version = "2.1.0", default-features = false, features = ["std", "svg"] }
+qrcode-rs = { version = "2.0.0", default-features = false, features = ["std", "svg"] }
 pin-project-lite = "0.2.17"
 pretty_assertions = "1.4.1"
 rand = { version = "0.10.2" }
@@ -335,7 +335,7 @@ tracing-subscriber = { version = "0.3.23" }
 transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
-uuid = { version = "1.26.1" }
+uuid = { version = "1.26.0" }
 vaultrs = { version = "0.8.0" }
 tar = "0.4.46"
 walkdir = "2.5.0"
@@ -362,10 +362,10 @@ pyroscope = { version = "2.1.1" }
 # FTP and SFTP
 libunftp = { version = "0.23.0" }
 unftp-core = "0.1.0"
-suppaftp = { version = "12.0.0" }
+suppaftp = { version = "11.0.0" }
 rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
-russh = { version = "0.63.3" }
-russh-sftp = "3.0.0"
+russh = { version = "0.63.2" }
+russh-sftp = "2.4.0"
 
 # WebDAV
 dav-server = "0.11.0"
@@ -373,7 +373,7 @@ dav-server = "0.11.0"
 # Performance Analysis and Memory Profiling
 rustfs-mimalloc = { version = "0.5.3" }
 # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
-hotpath = { version = "0.25.1", default-features = false }
+hotpath = { version = "=0.25.0", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.48" }
 

README.md

@@ -141,7 +141,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
 # Using specific version
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.6
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.5
 ```
 
 If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
@@ -211,10 +211,7 @@ For developers who want to build RustFS Docker images from source with multi-arc
 
 ```bash
 # Build multi-architecture images locally
-./docker-buildx.sh
-
-# Build a single-platform image locally
-./docker-buildx.sh -p linux/amd64
+./docker-buildx.sh --build-arg RELEASE=latest
 
 # Build and push to registry
 ./docker-buildx.sh --push

README_ZH.md

@@ -138,7 +138,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
 # 使用指定版本运行
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.6
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.5
 ```
 
 如果您通过绑定挂载启用 TLS 证书目录，也请用同样方式准备该目录：
@@ -167,10 +167,7 @@ docker compose -f docker-compose-simple.yml up -d
 
 ```bash
 # 在本地构建多架构镜像
-./docker-buildx.sh
-
-# 在本地构建单平台镜像
-./docker-buildx.sh -p linux/amd64
+./docker-buildx.sh --build-arg RELEASE=latest
 
 # 构建并推送到仓库
 ./docker-buildx.sh --push

crates/config/README.md

@@ -62,37 +62,6 @@ Current guidance:
 - `RUSTFS_CORS_ALLOWED_ORIGINS` defaults to empty, so the S3 endpoint emits no generic CORS headers unless configured. Set `*` for wildcard origins without credentials, or a comma-separated allow-list for credentialed explicit origins.
 - `RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS` defaults to `*` for the console service.
 
-## Console URL prefix
-
-`RUSTFS_CONSOLE_PREFIX` changes the embedded console URL prefix. The default is
-`/rustfs/console`. For example, `RUSTFS_CONSOLE_PREFIX=/console` serves the UI at
-`http://localhost:9001/console/`. Nested prefixes such as `/management/console`
-are supported; one trailing slash is removed. Restart the server after changing it.
-
-The prefix must be a non-root absolute path of at most 256 bytes, with nonempty
-segments containing only ASCII letters, digits, `-`, `_`, `.`, or `~`. Dot
-segments, encoded characters, and overlaps with reserved admin, RPC, health,
-profiling, browser entry, and icon routes are rejected at startup. `/` is not supported.
-Choose a prefix that does not collide with S3 bucket paths.
-
-The console routes, embedded frontend asset URLs, browser redirects, and OIDC
-console redirects use this prefix. Admin API paths and the identity provider's
-`/rustfs/admin/v3/oidc/callback/...` URL remain unchanged. `RUSTFS_CONSOLE_ADDRESS`
-continues to control only the listening address and port. The server adapts bundled
-console asset references from their build-time base path to the runtime prefix.
-
-OEM builds can set `RUSTFS_CONSOLE_BASE_PATH` when compiling RustFS to embed a
-different default, such as `/nuofans/console`. Build the bundled console with the
-same `NEXT_PUBLIC_BASE_PATH`. An unset or empty build variable retains
-`/rustfs/console`. The build path must satisfy the validation rules above and must
-not have a trailing slash.
-
-At startup, `RUSTFS_CONSOLE_PREFIX` takes precedence over the compiled default.
-Changing `RUSTFS_CONSOLE_BASE_PATH` when starting an existing binary has no effect;
-rebuild both components to change the embedded default. If a runtime prefix is
-configured, asset adaptation uses the compiled base path as its source, including
-when restoring `/rustfs/console` for a custom OEM build.
-
 ## Browser redirect environment variables
 
 - `RUSTFS_BROWSER_REDIRECT_URL` sets the externally reachable browser origin used for OIDC callback, console success redirect, and logout fallback URLs. Configure it to the public scheme and authority without a path, for example `https://console.example.com`. In load-balancer deployments, keep OIDC authorize and callback requests on the same backend node because the in-flight OIDC `state` is local to the RustFS node.

crates/config/src/constants/app.rs

@@ -213,11 +213,6 @@ pub const ENV_RUSTFS_CONSOLE_ENABLE: &str = "RUSTFS_CONSOLE_ENABLE";
 /// Environment variable for console server address.
 pub const ENV_RUSTFS_CONSOLE_ADDRESS: &str = "RUSTFS_CONSOLE_ADDRESS";
 
-/// URL path prefix for the embedded console, read once at server startup.
-pub const ENV_RUSTFS_CONSOLE_PREFIX: &str = "RUSTFS_CONSOLE_PREFIX";
-/// Default embedded console URL path prefix.
-pub const DEFAULT_CONSOLE_PREFIX: &str = "/rustfs/console";
-
 /// Public browser entrypoint used to build OIDC callback and console redirects.
 ///
 /// This should be the externally reachable scheme and authority, without a path.

crates/config/src/constants/body_limits.rs

@@ -64,14 +64,6 @@ pub const MAX_HEAL_REQUEST_SIZE: usize = 1024 * 1024; // 1 MB
 /// memory exhaustion from malicious or misconfigured remote services.
 pub const MAX_S3_CLIENT_RESPONSE_SIZE: usize = 10 * 1024 * 1024; // 10 MB
 
-/// Maximum body size accepted by a single `PutObject` or `UploadPart` request (5 GiB).
-/// Used for: the s3s streaming-body limit and the request-header admission check.
-/// Rationale: matches the AWS S3 single-PUT / single-part ceiling. Larger objects
-/// must use multipart upload. The header check rejects an oversize
-/// `Content-Length` before any body byte is read so the client gets
-/// `EntityTooLarge` immediately instead of streaming 5 GiB into a mid-stream failure.
-pub const MAX_SINGLE_PUT_OBJECT_SIZE: u64 = 5 * 1024 * 1024 * 1024; // 5 GiB
-
 /// Maximum size for OIDC provider response bodies (1 MB)
 /// Used for: discovery documents, JWKS documents and token endpoint responses
 /// Rationale: a hostile or compromised identity provider must not be able to exhaust

crates/data-usage/src/data_usage.rs

@@ -593,20 +593,6 @@ pub struct DataUsageSnapshotIdentity {
     pub scanner_epoch: Option<u64>,
 }
 
-#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
-pub struct DataUsageSegmentInvalidationProof {
-    #[serde(default)]
-    pub process_epoch: String,
-    #[serde(default)]
-    pub generation_start: u64,
-    #[serde(default)]
-    pub generation_end: u64,
-    #[serde(default)]
-    pub producer_identity_coverage_complete: bool,
-    #[serde(default)]
-    pub cold_zero_walk_oracle: bool,
-}
-
 #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
 pub struct DataUsageSnapshotSetState {
     pub pool_index: u64,
@@ -621,8 +607,6 @@ pub struct DataUsageSnapshotSetState {
     pub complete: bool,
     #[serde(default)]
     pub tombstone: bool,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub segment_invalidation_proof: Option<DataUsageSegmentInvalidationProof>,
 }
 
 impl DataUsageInfo {
@@ -3089,7 +3073,6 @@ mod tests {
             scan_plan_digest: Some([1; 32]),
             complete: false,
             tombstone: false,
-            segment_invalidation_proof: None,
         }];
         assert!(observed_data_usage_is_newer(&partial, &authoritative));
     }
@@ -3112,7 +3095,6 @@ mod tests {
                     scan_plan_digest: Some([1; 32]),
                     complete: true,
                     tombstone: false,
-                    segment_invalidation_proof: None,
                 },
                 DataUsageSnapshotSetState {
                     pool_index: 1,
@@ -3122,7 +3104,6 @@ mod tests {
                     scan_plan_digest: Some([2; 32]),
                     complete: false,
                     tombstone: false,
-                    segment_invalidation_proof: None,
                 },
             ],
             ..Default::default()
@@ -3132,42 +3113,6 @@ mod tests {
         assert!(partial.is_valid_partial_snapshot());
     }
 
-    #[test]
-    fn set_state_segment_invalidation_proof_is_additive() {
-        #[derive(Deserialize)]
-        struct LegacySetState {
-            pool_index: u64,
-            set_index: u64,
-            complete: bool,
-        }
-
-        let proof = DataUsageSegmentInvalidationProof {
-            process_epoch: "scanner-process".to_string(),
-            generation_start: 3,
-            generation_end: 5,
-            producer_identity_coverage_complete: true,
-            cold_zero_walk_oracle: true,
-        };
-        let state = DataUsageSnapshotSetState {
-            pool_index: 1,
-            set_index: 2,
-            scanner_cycle: Some(9),
-            scanner_epoch: Some(4),
-            scan_plan_digest: Some([7; 32]),
-            complete: true,
-            tombstone: false,
-            segment_invalidation_proof: Some(proof.clone()),
-        };
-        let encoded = rmp_serde::to_vec_named(&state).expect("set state should encode with additive proof");
-        let legacy: LegacySetState = rmp_serde::from_slice(&encoded).expect("legacy readers should ignore proof metadata");
-        assert_eq!(legacy.pool_index, 1);
-        assert_eq!(legacy.set_index, 2);
-        assert!(legacy.complete);
-
-        let decoded: DataUsageSnapshotSetState = rmp_serde::from_slice(&encoded).expect("new readers should restore proof");
-        assert_eq!(decoded.segment_invalidation_proof, Some(proof));
-    }
-
     #[test]
     fn completeness_marker_requires_a_snapshot_timestamp() {
         let untimestamped = DataUsageInfo {

crates/e2e_test/README.md

@@ -30,21 +30,12 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
 
-The external-tool `storage_metric_ownership_test` validates the OTLP/Collector/Prometheus path, including a rolling upgrade and node failures. See the [storage metrics guide](../../docs/operations/storage-metrics.md) for its required binaries and focused command.
-
 ## How to run
 
 All commands assume repo root. `cargo test` triggers an on-demand build of the
 `rustfs` binary from [`src/common.rs`](src/common.rs) (`rustfs_binary_path`) on
 first use — the first invocation is slow, later ones reuse the binary.
 
-Root-heal interruption scenarios use a test-only commit barrier. Prebuild with `e2e-test-hooks` and pin that binary so concurrent cases do not replace it through on-demand builds:
-
-```bash
-cargo build -p rustfs --bin rustfs --features e2e-test-hooks
-CARGO_BIN_EXE_rustfs="$PWD/target/debug/rustfs" cargo nextest run -p e2e_test -E 'test(heal_erasure_disk_rebuild_test)'
-```
-
 ```bash
 # Whole crate (default = ignored tests skipped)
 cargo nextest run -p e2e_test

crates/e2e_test/src/cluster_concurrency_test.rs

@@ -268,7 +268,6 @@ async fn test_bucket_cors_write_is_visible_on_peer_before_response() -> Result<(
     let rule = CorsRule::builder()
         .allowed_methods("GET")
         .allowed_origins("https://example.com")
-        .allowed_headers("*")
         .build()?;
     let configuration = CorsConfiguration::builder().cors_rules(rule).build()?;
 
@@ -289,60 +288,6 @@ async fn test_bucket_cors_write_is_visible_on_peer_before_response() -> Result<(
     assert_eq!(rules[0].allowed_methods(), ["GET"]);
     assert_eq!(rules[0].allowed_origins(), ["https://example.com"]);
 
-    let http = reqwest::Client::builder().no_proxy().build()?;
-    let url = format!("http://{}/{}", cluster.nodes[1].address, BUCKET_METADATA_RELOAD_BUCKET);
-    let without_headers = http
-        .request(reqwest::Method::OPTIONS, &url)
-        .header("Origin", "https://example.com")
-        .header("Access-Control-Request-Method", "GET")
-        .send()
-        .await?;
-    assert!(without_headers.status().is_success());
-    assert!(!without_headers.headers().contains_key("access-control-allow-headers"));
-    assert!(
-        without_headers
-            .headers()
-            .get("vary")
-            .and_then(|value| value.to_str().ok())
-            .is_some_and(|value| value.contains("Access-Control-Request-Headers")),
-        "a cached header-free preflight must not suppress a later requested header grant"
-    );
-    let preflight = http
-        .request(reqwest::Method::OPTIONS, &url)
-        .header("Origin", "https://example.com")
-        .header("Access-Control-Request-Method", "GET")
-        .header("Access-Control-Request-Headers", "X-Another-Header, x-could-be-anything")
-        .send()
-        .await?;
-    assert!(preflight.status().is_success(), "peer preflight should succeed: {preflight:?}");
-    assert_eq!(
-        preflight
-            .headers()
-            .get("access-control-allow-headers")
-            .and_then(|value| value.to_str().ok()),
-        Some("x-another-header,x-could-be-anything"),
-        "a wildcard rule must return only the headers requested by this preflight"
-    );
-    assert!(
-        preflight
-            .headers()
-            .get("vary")
-            .and_then(|value| value.to_str().ok())
-            .is_some_and(|value| value.contains("Access-Control-Request-Headers")),
-        "preflight caches must distinguish the requested header list"
-    );
-    let denied = http
-        .request(reqwest::Method::OPTIONS, &url)
-        .header("Origin", "https://disallowed.example.com")
-        .header("Access-Control-Request-Method", "GET")
-        .header("Access-Control-Request-Headers", "x-another-header")
-        .send()
-        .await?;
-    assert!(
-        !denied.headers().contains_key("access-control-allow-headers"),
-        "a rejected origin must not receive the requested header grant"
-    );
-
     writer
         .delete_bucket_cors()
         .bucket(BUCKET_METADATA_RELOAD_BUCKET)

crates/e2e_test/src/delete_authorization_test.rs

@@ -1,642 +0,0 @@
-// Copyright 2026 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Standard S3 deletion permissions and the explicit recursive-delete extension.
-
-use crate::common::{
-    AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user,
-    init_logging,
-};
-use aws_sdk_s3::Client;
-use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
-use aws_sdk_s3::operation::delete_object::{DeleteObjectError, DeleteObjectOutput};
-use aws_sdk_s3::primitives::ByteStream;
-use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration};
-use futures::{StreamExt, TryStreamExt, stream};
-use serde_json::{Value, json};
-use std::collections::BTreeSet;
-use std::error::Error;
-use uuid::Uuid;
-
-type TestResult<T = ()> = Result<T, Box<dyn Error + Send + Sync>>;
-type VersionSnapshot = BTreeSet<(String, String, bool)>;
-
-async fn set_policy(env: &RustFSTestEnvironment, name: &str, policy: &Value) -> TestResult {
-    admin_add_canned_policy_via(
-        AdminTransport::Signed,
-        &env.url,
-        &env.access_key,
-        &env.secret_key,
-        name,
-        &policy.to_string(),
-    )
-    .await
-}
-
-async fn policy_user(env: &RustFSTestEnvironment, policy_name: &str, policy: Option<Value>) -> TestResult<Client> {
-    let username = Uuid::new_v4().simple().to_string();
-    let secret = Uuid::new_v4().simple().to_string();
-    admin_create_user(env, &username, &secret).await?;
-    if let Some(policy) = policy {
-        set_policy(env, policy_name, &policy).await?;
-    }
-    admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, policy_name, &username)
-        .await?;
-    Ok(env.create_s3_client_with_credentials(&username, &secret))
-}
-
-async fn versioning(client: &Client, bucket: &str, status: BucketVersioningStatus) -> TestResult {
-    client
-        .put_bucket_versioning()
-        .bucket(bucket)
-        .versioning_configuration(VersioningConfiguration::builder().status(status).build())
-        .send()
-        .await?;
-    Ok(())
-}
-
-async fn put(client: &Client, bucket: &str, key: &str) -> TestResult<String> {
-    let result = client
-        .put_object()
-        .bucket(bucket)
-        .key(key)
-        .body(ByteStream::from_static(b"delete authorization fixture"))
-        .send()
-        .await?;
-    Ok(result.version_id().unwrap_or("null").to_string())
-}
-
-async fn versions(client: &Client, bucket: &str, prefix: &str) -> TestResult<VersionSnapshot> {
-    let mut result = BTreeSet::new();
-    let mut markers = (None, None);
-    loop {
-        let page = client
-            .list_object_versions()
-            .bucket(bucket)
-            .prefix(prefix)
-            .set_key_marker(markers.0.clone())
-            .set_version_id_marker(markers.1.clone())
-            .send()
-            .await?;
-        for version in page.versions() {
-            result.insert((
-                version.key().ok_or("listed version missing key")?.to_string(),
-                version.version_id().ok_or("listed version missing ID")?.to_string(),
-                false,
-            ));
-        }
-        for marker in page.delete_markers() {
-            result.insert((
-                marker.key().ok_or("listed delete marker missing key")?.to_string(),
-                marker.version_id().ok_or("listed delete marker missing ID")?.to_string(),
-                true,
-            ));
-        }
-        if page.is_truncated() != Some(true) {
-            return Ok(result);
-        }
-        let next = (
-            Some(
-                page.next_key_marker()
-                    .ok_or("truncated versions page missing next key marker")?
-                    .to_string(),
-            ),
-            page.next_version_id_marker().map(str::to_string),
-        );
-        assert_ne!(markers, next, "ListObjectVersions pagination must advance");
-        markers = next;
-    }
-}
-
-async fn force_delete(client: &Client, bucket: &str, prefix: &str) -> Result<DeleteObjectOutput, SdkError<DeleteObjectError>> {
-    client
-        .delete_object()
-        .bucket(bucket)
-        .key(prefix)
-        .customize()
-        .mutate_request(|request| {
-            request.headers_mut().insert("x-rustfs-force-delete", "true");
-        })
-        .send()
-        .await
-}
-
-async fn replica_force_delete(
-    client: &Client,
-    bucket: &str,
-    prefix: &str,
-) -> Result<DeleteObjectOutput, SdkError<DeleteObjectError>> {
-    client
-        .delete_object()
-        .bucket(bucket)
-        .key(prefix)
-        .customize()
-        .mutate_request(|request| {
-            request.headers_mut().insert("x-rustfs-force-delete", "true");
-            request.headers_mut().insert("x-amz-replication-status", "REPLICA");
-        })
-        .send()
-        .await
-}
-
-fn assert_denied<T, E>(result: Result<T, SdkError<E>>)
-where
-    T: std::fmt::Debug,
-    E: ProvideErrorMetadata + std::fmt::Debug,
-{
-    let error = result.expect_err("request must be denied by its S3 permission");
-    assert_eq!(
-        error.as_service_error().and_then(ProvideErrorMetadata::code),
-        Some("AccessDenied"),
-        "expected an S3 authorization denial, got {error:?}"
-    );
-}
-
-#[tokio::test]
-async fn sdk_version_deletion_requires_only_delete_object_version() -> TestResult {
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-    let root = env.create_s3_client();
-    let bucket = "delete-version-permissions";
-    root.create_bucket().bucket(bucket).send().await?;
-    put(&root, bucket, "single-null.txt").await?;
-    put(&root, bucket, "batch-null.txt").await?;
-    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
-    let old = put(&root, bucket, "single.txt").await?;
-    let current = put(&root, bucket, "single.txt").await?;
-    let batch_version = put(&root, bucket, "batch.txt").await?;
-    let ordinary_version = put(&root, bucket, "ordinary.txt").await?;
-    let user = policy_user(
-        &env,
-        "version-deleter",
-        Some(json!({"Version":"2012-10-17","Statement":[
-            {"Effect":"Allow","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/*")},
-            {"Effect":"Deny","Action":"s3:DeleteObject","Resource":format!("arn:aws:s3:::{bucket}/*")}
-        ]})),
-    )
-    .await?;
-
-    user.delete_object()
-        .bucket(bucket)
-        .key("single.txt")
-        .version_id(&old)
-        .send()
-        .await?;
-    user.delete_object()
-        .bucket(bucket)
-        .key("single-null.txt")
-        .version_id("null")
-        .send()
-        .await?;
-    assert_denied(user.delete_object().bucket(bucket).key("ordinary.txt").send().await);
-
-    let batch = user
-        .delete_objects()
-        .bucket(bucket)
-        .delete(
-            Delete::builder()
-                .objects(
-                    ObjectIdentifier::builder()
-                        .key("batch.txt")
-                        .version_id(&batch_version)
-                        .build()?,
-                )
-                .objects(ObjectIdentifier::builder().key("batch-null.txt").version_id("null").build()?)
-                .objects(ObjectIdentifier::builder().key("ordinary.txt").build()?)
-                .build()?,
-        )
-        .send()
-        .await?;
-    assert_eq!(batch.deleted().len(), 2, "both explicit version items must succeed");
-    assert_eq!(batch.errors().len(), 1, "only the unversioned item must be denied");
-    assert_eq!(batch.errors()[0].key(), Some("ordinary.txt"));
-    assert_eq!(batch.errors()[0].code(), Some("AccessDenied"));
-    assert_eq!(
-        versions(&root, bucket, "").await?,
-        BTreeSet::from([
-            ("single.txt".into(), current, false),
-            ("ordinary.txt".into(), ordinary_version, false)
-        ]),
-        "version-only deletion must preserve the current single-object version and denied object"
-    );
-    Ok(())
-}
-
-#[tokio::test]
-async fn sdk_list_bucket_and_list_bucket_versions_permissions_are_independent() -> TestResult {
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-    let root = env.create_s3_client();
-    let bucket = "list-version-permissions";
-    root.create_bucket().bucket(bucket).send().await?;
-    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
-    put(&root, bucket, "visible.txt").await?;
-    for (action, name) in [
-        ("s3:ListBucket", "object-lister"),
-        ("s3:ListBucketVersions", "version-lister"),
-    ] {
-        let user = policy_user(
-            &env,
-            name,
-            Some(json!({"Version":"2012-10-17","Statement":[
-                {"Effect":"Allow","Action":action,"Resource":format!("arn:aws:s3:::{bucket}")}
-            ]})),
-        )
-        .await?;
-        if action == "s3:ListBucket" {
-            assert_eq!(user.list_objects_v2().bucket(bucket).send().await?.contents().len(), 1);
-            assert_denied(user.list_object_versions().bucket(bucket).send().await);
-        } else {
-            assert_eq!(user.list_object_versions().bucket(bucket).send().await?.versions().len(), 1);
-            assert_denied(user.list_objects_v2().bucket(bucket).send().await);
-        }
-    }
-    Ok(())
-}
-
-#[tokio::test]
-async fn console_admin_force_delete_removes_prefix_versions_and_delete_markers() -> TestResult {
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-    let root = env.create_s3_client();
-    let bucket = "force-console-admin";
-    root.create_bucket().bucket(bucket).send().await?;
-    put(&root, bucket, "folder/null.txt").await?;
-    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
-    for key in ["folder/a.txt", "folder/deep/b.txt", "single.txt"] {
-        put(&root, bucket, key).await?;
-        put(&root, bucket, key).await?;
-        root.delete_object().bucket(bucket).key(key).send().await?;
-    }
-    put(&root, bucket, "keep.txt").await?;
-    put(&root, bucket, "folder-sibling/keep.txt").await?;
-    let keep = versions(&root, bucket, "keep.txt").await?;
-    let sibling = versions(&root, bucket, "folder-sibling/").await?;
-    let user = policy_user(&env, "consoleAdmin", None).await?;
-
-    force_delete(&user, bucket, "folder/").await?;
-    assert!(
-        versions(&root, bucket, "folder/").await?.is_empty(),
-        "force prefix deletion must remove null versions and markers"
-    );
-    assert_eq!(versions(&root, bucket, "folder-sibling/").await?, sibling);
-    assert_eq!(
-        versions(&root, bucket, "single.txt").await?.len(),
-        3,
-        "the separate key must survive folder deletion"
-    );
-    force_delete(&user, bucket, "single.txt").await?;
-    assert!(
-        versions(&root, bucket, "single.txt").await?.is_empty(),
-        "explicit force deletion must remove every version of the selected key"
-    );
-    assert_eq!(versions(&root, bucket, "keep.txt").await?, keep);
-    Ok(())
-}
-
-#[tokio::test]
-async fn force_delete_authorizes_only_its_path_scope_without_list_permissions() -> TestResult {
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-    let root = env.create_s3_client();
-    let bucket = "force-delete-only";
-    root.create_bucket().bucket(bucket).send().await?;
-    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
-    put(&root, bucket, "selected.txt").await?;
-    put(&root, bucket, "selected.txt/child.txt").await?;
-    put(&root, bucket, "selected.txt-sibling").await?;
-    let sibling = versions(&root, bucket, "selected.txt-sibling").await?;
-    let user = policy_user(
-        &env,
-        "delete-only",
-        Some(json!({"Version":"2012-10-17","Statement":[
-            {"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":[
-                format!("arn:aws:s3:::{bucket}/selected.txt"), format!("arn:aws:s3:::{bucket}/selected.txt/*")
-            ]},
-            {"Effect":"Deny","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/selected.txt-sibling")}
-        ]})),
-    )
-    .await?;
-    assert_denied(user.list_objects_v2().bucket(bucket).send().await);
-    assert_denied(user.list_object_versions().bucket(bucket).send().await);
-    force_delete(&user, bucket, "selected.txt").await?;
-    assert_eq!(
-        versions(&root, bucket, "selected.txt").await?,
-        sibling,
-        "force deletion must remove the selected path and descendants without authorizing or deleting its similarly prefixed sibling"
-    );
-    Ok(())
-}
-
-#[tokio::test]
-async fn force_directory_delete_cannot_remove_an_unauthorized_colliding_parent() -> TestResult {
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-    let root = env.create_s3_client();
-    let bucket = "force-directory-collision";
-    root.create_bucket().bucket(bucket).send().await?;
-    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
-    let protected_parent_version = put(&root, bucket, "collision.txt").await?;
-    for key in ["collision.txt/child", "collision.txt-sibling"] {
-        put(&root, bucket, key).await?;
-    }
-    put(&root, bucket, "collision.txt").await?;
-    root.delete_object().bucket(bucket).key("collision.txt").send().await?;
-    let user = policy_user(
-        &env,
-        "parent-denier",
-        Some(json!({"Version":"2012-10-17","Statement":[
-            {"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")},
-            {"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/collision.txt"),
-             "Condition":{"StringEquals":{"s3:VersionId":protected_parent_version}}}
-        ]})),
-    )
-    .await?;
-    let mut expected = versions(&root, bucket, "").await?;
-    expected.retain(|(key, _, _)| key != "collision.txt/child");
-    force_delete(&user, bucket, "collision.txt/").await?;
-    assert_eq!(
-        versions(&root, bucket, "").await?,
-        expected,
-        "folder deletion must preserve the denied parent's historical versions and delete marker, plus its sibling"
-    );
-    Ok(())
-}
-
-#[tokio::test]
-async fn force_unversioned_directory_requires_only_delete_object() -> TestResult {
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-    let root = env.create_s3_client();
-    let bucket = "force-unversioned-permissions";
-    root.create_bucket().bucket(bucket).send().await?;
-    for key in ["folder/", "folder/child.txt", "outside.txt"] {
-        put(&root, bucket, key).await?;
-    }
-    let outside = versions(&root, bucket, "outside.txt").await?;
-    let user = policy_user(
-        &env,
-        "unversioned-deleter",
-        Some(json!({"Version":"2012-10-17","Statement":[
-            {"Effect":"Allow","Action":"s3:DeleteObject","Resource":format!("arn:aws:s3:::{bucket}/*")},
-            {"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/*")}
-        ]})),
-    )
-    .await?;
-    force_delete(&user, bucket, "folder/").await?;
-    assert_eq!(
-        versions(&root, bucket, "").await?,
-        outside,
-        "unversioned force deletion, including a synthetic nil directory marker, must use DeleteObject permission"
-    );
-    Ok(())
-}
-
-#[tokio::test]
-async fn force_delete_denied_child_preserves_every_object_despite_bucket_allow() -> TestResult {
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-    let root = env.create_s3_client();
-    let bucket = "force-child-denial";
-    root.create_bucket().bucket(bucket).send().await?;
-    for key in ["folder/a-allowed.txt", "folder/z-denied.txt"] {
-        put(&root, bucket, key).await?;
-    }
-    let user = policy_user(
-        &env,
-        "child-denier",
-        Some(json!({"Version":"2012-10-17","Statement":[
-            {"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion","s3:ReplicateDelete"],"Resource":format!("arn:aws:s3:::{bucket}/*")},
-            {"Effect":"Deny","Action":["s3:DeleteObject","s3:DeleteObjectVersion","s3:ReplicateDelete"],"Resource":format!("arn:aws:s3:::{bucket}/folder/z-denied.txt")}
-        ]})),
-    )
-    .await?;
-    root.put_bucket_policy()
-        .bucket(bucket)
-        .policy(json!({"Version":"2012-10-17","Statement":[
-            {"Effect":"Allow","Principal":"*","Action":["s3:DeleteObject","s3:DeleteObjectVersion","s3:ReplicateDelete"],"Resource":format!("arn:aws:s3:::{bucket}/*")}
-        ]}).to_string())
-        .send().await?;
-    let before = versions(&root, bucket, "folder/").await?;
-    assert_denied(force_delete(&user, bucket, "folder/").await);
-    assert_eq!(
-        versions(&root, bucket, "folder/").await?,
-        before,
-        "a denied descendant must prevent every mutation in the force scope"
-    );
-    assert_denied(replica_force_delete(&user, bucket, "folder/").await);
-    assert_eq!(
-        versions(&root, bucket, "folder/").await?,
-        before,
-        "the REPLICA header must not bypass a descendant's ReplicateDelete denial"
-    );
-
-    root.delete_bucket_policy().bucket(bucket).send().await?;
-    let replica_user = policy_user(
-        &env,
-        "replica-deleter",
-        Some(json!({"Version":"2012-10-17","Statement":[
-            {"Effect":"Allow","Action":"s3:DeleteObject","Resource":format!("arn:aws:s3:::{bucket}/*")},
-            {"Effect":"Allow","Action":"s3:ReplicateDelete","Resource":format!("arn:aws:s3:::{bucket}/*")}
-        ]})),
-    )
-    .await?;
-    replica_force_delete(&replica_user, bucket, "folder/").await?;
-    assert!(
-        versions(&root, bucket, "folder/").await?.is_empty(),
-        "an authorized replica force request must check ReplicateDelete for its descendants"
-    );
-    Ok(())
-}
-
-#[tokio::test]
-async fn force_delete_denied_historical_version_preserves_versions_and_markers() -> TestResult {
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-    let root = env.create_s3_client();
-    let bucket = "force-version-denial";
-    root.create_bucket().bucket(bucket).send().await?;
-    put(&root, bucket, "folder/null.txt").await?;
-    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
-    let protected_version = put(&root, bucket, "folder/versioned.txt").await?;
-    put(&root, bucket, "folder/versioned.txt").await?;
-    let marker = root.delete_object().bucket(bucket).key("folder/versioned.txt").send().await?;
-    let marker_version = marker
-        .version_id()
-        .ok_or("versioned delete must return a marker version ID")?;
-    put(&root, bucket, "folder/a-allowed.txt").await?;
-    let policy = |version: &str| {
-        json!({"Version":"2012-10-17","Statement":[
-            {"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")},
-            {"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/folder/*"),
-             "Condition":{"StringEquals":{"s3:VersionId":version}}}
-        ]})
-    };
-    let user = policy_user(&env, "version-denier", Some(policy(&protected_version))).await?;
-    let before = versions(&root, bucket, "folder/").await?;
-    for version in [protected_version.as_str(), "null", marker_version] {
-        set_policy(&env, "version-denier", &policy(version)).await?;
-        assert_denied(force_delete(&user, bucket, "folder/").await);
-        assert_eq!(
-            versions(&root, bucket, "folder/").await?,
-            before,
-            "denial of a historical, null, or delete-marker version must prevent recursive deletion"
-        );
-    }
-    Ok(())
-}
-
-#[tokio::test]
-async fn sdk_ordinary_deletion_preserves_versions_and_directory_children() -> TestResult {
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-    let root = env.create_s3_client();
-    let user = policy_user(
-        &env,
-        "ordinary-deleter",
-        Some(json!({"Version":"2012-10-17","Statement":[
-            {"Effect":"Allow","Action":"s3:DeleteObject","Resource":"arn:aws:s3:::*/*"},
-            {"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":"arn:aws:s3:::*/*"}
-        ]})),
-    )
-    .await?;
-    for state in ["unversioned", "enabled", "suspended"] {
-        let bucket = format!("ordinary-directory-{state}");
-        root.create_bucket().bucket(&bucket).send().await?;
-        if state != "unversioned" {
-            versioning(&root, &bucket, BucketVersioningStatus::Enabled).await?;
-        }
-        let historical = put(&root, &bucket, "object.txt").await?;
-        if state == "suspended" {
-            versioning(&root, &bucket, BucketVersioningStatus::Suspended).await?;
-            put(&root, &bucket, "object.txt").await?;
-        }
-        put(&root, &bucket, "folder/").await?;
-        put(&root, &bucket, "folder/child.txt").await?;
-        let child_before = versions(&root, &bucket, "folder/child.txt").await?;
-        user.delete_object().bucket(&bucket).key("folder/").send().await?;
-        assert_eq!(
-            versions(&root, &bucket, "folder/").await?,
-            child_before,
-            "ordinary {state} directory-key deletion must remove only its synthetic marker and preserve children"
-        );
-        let deleted = user.delete_object().bucket(&bucket).key("object.txt").send().await?;
-        let object_versions = versions(&root, &bucket, "object.txt").await?;
-        if state == "unversioned" {
-            assert!(object_versions.is_empty());
-        } else {
-            assert_eq!(deleted.delete_marker(), Some(true));
-            assert_eq!(
-                object_versions.len(),
-                2,
-                "ordinary {state} deletion must retain its historical data version"
-            );
-            assert!(object_versions.contains(&("object.txt".into(), historical, false)));
-            if state == "suspended" {
-                assert!(object_versions.contains(&("object.txt".into(), "null".into(), true)));
-            }
-        }
-    }
-    Ok(())
-}
-
-#[tokio::test]
-async fn sdk_delete_objects_force_header_keeps_explicit_item_scope() -> TestResult {
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-    let root = env.create_s3_client();
-    let bucket = "batch-force-explicit-scope";
-    root.create_bucket().bucket(bucket).send().await?;
-    put(&root, bucket, "folder/").await?;
-    put(&root, bucket, "folder/child.txt").await?;
-    let child = versions(&root, bucket, "folder/child.txt").await?;
-    let user = policy_user(&env, "consoleAdmin", None).await?;
-    let result = user
-        .delete_objects()
-        .bucket(bucket)
-        .delete(
-            Delete::builder()
-                .objects(ObjectIdentifier::builder().key("folder/").build()?)
-                .build()?,
-        )
-        .customize()
-        .mutate_request(|request| {
-            request.headers_mut().insert("x-rustfs-force-delete", "true");
-        })
-        .send()
-        .await?;
-    assert!(result.errors().is_empty());
-    assert_eq!(result.deleted().len(), 1);
-    assert_eq!(
-        versions(&root, bucket, "folder/").await?,
-        child,
-        "batch deletion must remove only the explicit directory marker even with the force header"
-    );
-    Ok(())
-}
-
-#[tokio::test]
-async fn force_delete_checks_every_version_page_before_mutation() -> TestResult {
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-    let root = env.create_s3_client();
-    let bucket = "force-delete-pagination";
-    root.create_bucket().bucket(bucket).send().await?;
-    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
-    stream::iter(0..1000)
-        .map(|index| {
-            let root = &root;
-            async move { put(root, bucket, &format!("folder/{index:04}.txt")).await.map(|_| ()) }
-        })
-        .buffer_unordered(16)
-        .try_collect::<Vec<_>>()
-        .await?;
-    put(&root, bucket, "folder/z-denied.txt").await?;
-    let allow = json!({"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")});
-    let user = policy_user(
-        &env,
-        "paged-deleter",
-        Some(json!({"Version":"2012-10-17","Statement":[allow.clone(),
-            {"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/folder/z-denied.txt")}
-        ]})),
-    )
-    .await?;
-    let before = versions(&root, bucket, "folder/").await?;
-    assert_eq!(before.len(), 1001, "the denied key must be beyond one default versions page");
-    assert_denied(force_delete(&user, bucket, "folder/").await);
-    assert_eq!(
-        versions(&root, bucket, "folder/").await?,
-        before,
-        "a denial on the second page must preserve the first page too"
-    );
-
-    set_policy(&env, "paged-deleter", &json!({"Version":"2012-10-17","Statement":[allow]})).await?;
-    force_delete(&user, bucket, "folder/").await?;
-    assert!(
-        versions(&root, bucket, "folder/").await?.is_empty(),
-        "authorized recursive deletion must cover all pages"
-    );
-    Ok(())
-}

crates/e2e_test/src/distributed/harness.rs

@@ -962,6 +962,27 @@ pub(crate) async fn wait_for_rebalance_active(
     }
 }
 
+pub(crate) async fn wait_for_rebalance_running_with_progress(
+    cluster: &RustFSTestClusterEnvironment,
+    expected_id: &str,
+    timeout: Duration,
+) -> TestResult {
+    let deadline = Instant::now() + timeout;
+    loop {
+        let status = rebalance_status_json(cluster).await?;
+        if rebalance_running_with_progress(&status, expected_id)? {
+            return Ok(());
+        }
+        if Instant::now() >= deadline {
+            return Err(format!(
+                "rebalance did not become active with non-zero progress within {timeout:?}; last status: {status}"
+            )
+            .into());
+        }
+        sleep(Duration::from_millis(100)).await;
+    }
+}
+
 pub(crate) async fn wait_for_rebalance_complete(
     cluster: &RustFSTestClusterEnvironment,
     expected_id: &str,

crates/e2e_test/src/distributed/heal_test.rs

@@ -26,7 +26,6 @@ use std::collections::{BTreeMap, HashSet};
 use std::io::{Read, Write};
 use std::path::{Path, PathBuf};
 use std::time::Duration;
-use tokio::time::{Instant, sleep};
 
 const EC84_NODE_COUNT: usize = 3;
 const EC84_DRIVES_PER_NODE: usize = 4;
@@ -34,8 +33,6 @@ const EC84_DATA_BLOCKS: usize = 8;
 const EC84_PARITY_BLOCKS: usize = 4;
 const EC84_TARGET_DRIVE_RESTART_CASE: &str = "ec84-target-drive-restart";
 const EC84_TARGET_DRIVE_RESTART_ORACLE: &str = "ec84-target-drive-restart.json";
-const EC84_HEAL_CONTROL_READY_TIMEOUT: Duration = Duration::from_secs(45);
-const EC84_HEAL_CONTROL_RETRY_DELAY: Duration = Duration::from_millis(250);
 
 #[derive(Clone)]
 struct ExpectedShard {
@@ -214,29 +211,6 @@ fn assert_replaced_drive_empty(drive: &Path, bucket: &str, keys: &[String]) -> T
     Ok(())
 }
 
-fn is_cluster_heal_coordination_unavailable(error: &(dyn std::error::Error + Send + Sync)) -> bool {
-    let message = error.to_string();
-    message.contains("500 Internal Server Error") && message.contains("cluster heal coordination unavailable")
-}
-
-async fn start_ec84_root_heal_when_control_ready(
-    heal_url: &str,
-    heal_body: &str,
-    access_key: &str,
-    secret_key: &str,
-) -> TestResult {
-    let deadline = Instant::now() + EC84_HEAL_CONTROL_READY_TIMEOUT;
-    loop {
-        match signed_admin_post(heal_url, Some(heal_body), access_key, secret_key).await {
-            Ok(_) => return Ok(()),
-            Err(error) if is_cluster_heal_coordination_unavailable(error.as_ref()) && Instant::now() < deadline => {
-                sleep(EC84_HEAL_CONTROL_RETRY_DELAY).await;
-            }
-            Err(error) => return Err(error),
-        }
-    }
-}
-
 async fn put_large_inventory(client: &Client, bucket: &str) -> TestResult<Vec<ExpectedShard>> {
     let mut expected = Vec::new();
     for index in 0..4 {
@@ -330,7 +304,7 @@ async fn three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_res
     let heal_body =
         r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
     let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[0].url);
-    start_ec84_root_heal_when_control_ready(&heal_url, heal_body, &dist.cluster.access_key, &dist.cluster.secret_key).await?;
+    signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?;
 
     wait_until(
         Duration::from_secs(120),
@@ -391,23 +365,3 @@ async fn three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_res
 
     Ok(())
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn cluster_heal_coordination_retry_is_exact() {
-        let retryable: Box<dyn std::error::Error + Send + Sync> =
-            "admin POST failed: 500 Internal Server Error cluster heal coordination unavailable".into();
-        assert!(is_cluster_heal_coordination_unavailable(retryable.as_ref()));
-
-        let other_internal: Box<dyn std::error::Error + Send + Sync> =
-            "admin POST failed: 500 Internal Server Error unrelated".into();
-        assert!(!is_cluster_heal_coordination_unavailable(other_internal.as_ref()));
-
-        let wrong_status: Box<dyn std::error::Error + Send + Sync> =
-            "admin POST failed: 503 Service Unavailable cluster heal coordination unavailable".into();
-        assert!(!is_cluster_heal_coordination_unavailable(wrong_status.as_ref()));
-    }
-}

crates/e2e_test/src/distributed/mod.rs

@@ -28,7 +28,6 @@ mod harness;
 mod heal_test;
 mod object_lock_test;
 mod observability_test;
-mod replication_delete_marker_test;
 mod replication_quota_test;
 mod s3_basic_test;
 mod s3_during_data_movement_test;

crates/e2e_test/src/distributed/replication_delete_marker_test.rs

@@ -1,137 +0,0 @@
-// Copyright 2026 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Functional REP-105 (rustfs/backlog#2195 item 4): a delete marker created
-//! on a multi-node source cluster must replicate to the bucket-replication
-//! target. Objects converged in seconds while delete markers did not arrive
-//! within 180 s on the shared 3-node functional environment; the single-node
-//! e2e never saw it.
-
-use super::harness::{
-    DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, set_remote_target, unique_bucket,
-    wait_for_replicated_bytes, wait_until,
-};
-use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestEnvironment, init_logging, replication_fast_env, signed_request};
-use crate::replication_extension_test::LOOPBACK_REPLICATION_TARGET_ENV;
-use aws_sdk_s3::Client;
-use http::{Method, StatusCode};
-use std::time::Duration;
-
-async fn target_has_delete_marker(client: &Client, bucket: &str, key: &str) -> TestResult<bool> {
-    let versions = client.list_object_versions().bucket(bucket).prefix(key).send().await?;
-    Ok(versions.delete_markers().iter().any(|marker| marker.key() == Some(key)))
-}
-
-async fn delete_marker_replicates(
-    source: &DistCluster,
-    source_bucket: &str,
-    target_client: &Client,
-    target_bucket: &str,
-) -> TestResult {
-    let key = "delete-marker/object.bin";
-    let body = b"delete marker replication payload".to_vec();
-    // Write through one node, delete through another: behind a load
-    // balancer consecutive requests land on different nodes.
-    put_object(&source.client(1)?, source_bucket, key, body.clone()).await?;
-    wait_for_replicated_bytes(target_client, target_bucket, key, &body, Duration::from_secs(60)).await?;
-
-    let delete = source
-        .client(2)?
-        .delete_object()
-        .bucket(source_bucket)
-        .key(key)
-        .send()
-        .await?;
-    assert_eq!(
-        delete.delete_marker(),
-        Some(true),
-        "a versioned DELETE without versionId must create a marker"
-    );
-
-    wait_until(
-        Duration::from_secs(90),
-        || async { target_has_delete_marker(target_client, target_bucket, key).await },
-        "delete marker replicated to the target bucket",
-    )
-    .await
-}
-
-#[tokio::test]
-async fn four_node_bucket_replication_replicates_delete_marker_to_peer_cluster() -> TestResult {
-    init_logging();
-    let (source, target) = DistCluster::start_replication_pair().await?;
-    let source_bucket = unique_bucket("dm-src");
-    let target_bucket = unique_bucket("dm-dst");
-    source.create_bucket(&source_bucket).await?;
-    target.create_bucket(&target_bucket).await?;
-    enable_versioning(&source.client(0)?, &source_bucket).await?;
-    enable_versioning(&target.client(0)?, &target_bucket).await?;
-
-    let arn = set_remote_target(&source.cluster, &source_bucket, &target.cluster, &target_bucket).await?;
-    put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
-
-    delete_marker_replicates(&source, &source_bucket, &target.client(0)?, &target_bucket).await
-}
-
-/// The functional environment replicates from a 3-node site to a single-node
-/// target; keep that shape as its own case.
-#[tokio::test]
-async fn four_node_bucket_replication_replicates_delete_marker_to_single_node_target() -> TestResult {
-    init_logging();
-    let mut extra: Vec<(&str, &str)> = replication_fast_env();
-    extra.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
-    extra.extend_from_slice(FAST_DATA_USAGE_SCANNER_ENV);
-    let source = DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &extra).await?;
-    let mut target = RustFSTestEnvironment::new().await?;
-    target.start_rustfs_server_without_cleanup(vec![]).await?;
-
-    let source_bucket = unique_bucket("dm-src");
-    let target_bucket = unique_bucket("dm-dst");
-    source.create_bucket(&source_bucket).await?;
-    let target_client = target.create_s3_client();
-    target_client.create_bucket().bucket(&target_bucket).send().await?;
-    enable_versioning(&source.client(0)?, &source_bucket).await?;
-    enable_versioning(&target_client, &target_bucket).await?;
-
-    let body = serde_json::json!({
-        "endpoint": target.address,
-        "credentials": { "accessKey": target.access_key, "secretKey": target.secret_key },
-        "targetbucket": target_bucket,
-        "secure": false,
-        "type": "replication"
-    });
-    let url = format!(
-        "{}/rustfs/admin/v3/set-remote-target?bucket={}",
-        source.cluster.nodes[0].url,
-        urlencoding::encode(&source_bucket)
-    );
-    let response = signed_request(
-        Method::PUT,
-        &url,
-        &source.cluster.access_key,
-        &source.cluster.secret_key,
-        Some(body.to_string().into_bytes()),
-        Some("application/json"),
-    )
-    .await?;
-    if response.status() != StatusCode::OK {
-        let status = response.status();
-        let body = response.text().await.unwrap_or_default();
-        return Err(format!("set remote target failed: {status} {body}").into());
-    }
-    let arn: String = serde_json::from_slice(&response.bytes().await?)?;
-    put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
-
-    delete_marker_replicates(&source, &source_bucket, &target_client, &target_bucket).await
-}

crates/e2e_test/src/distributed/s3_during_data_movement_test.rs

@@ -14,9 +14,9 @@
 
 use super::harness::{
     DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
-    decommission_status_json, put_inventory_retrying, rebalance_active, rebalance_status_json, retrying_get_equals, retrying_put,
-    start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
-    wait_for_decommission_running_with_progress, wait_for_rebalance_active, wait_for_rebalance_complete,
+    decommission_status_json, put_inventory_retrying, rebalance_running_with_progress, rebalance_status_json,
+    retrying_get_equals, retrying_put, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
+    wait_for_decommission_running_with_progress, wait_for_rebalance_complete, wait_for_rebalance_running_with_progress,
 };
 use crate::common::init_logging;
 use std::time::Duration;
@@ -67,10 +67,7 @@ async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResu
     assert_inventory(&live, &bucket, &inventory).await?;
 
     let rebalance_id = start_rebalance(&dist.cluster).await?;
-    // The status API reads persisted progress, whose first periodic save is
-    // after 30 seconds. A shorter run can remain at zero until completion.
-    // Require Started around the S3 operations and nonzero progress at completion.
-    wait_for_rebalance_active(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
+    wait_for_rebalance_running_with_progress(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
     retrying_put(
         &live,
         &bucket,
@@ -87,26 +84,11 @@ async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResu
         Duration::from_secs(30),
     )
     .await?;
-    let listed = live.list_objects_v2().bucket(&bucket).send().await?;
-    assert!(
-        listed
-            .contents()
-            .iter()
-            .any(|object| object.key() == Some("during-rebalance.bin")),
-        "list during rebalance missed the newly written key"
-    );
     let status = rebalance_status_json(&dist.cluster).await?;
-    if !rebalance_active(&status, &rebalance_id)? {
+    if !rebalance_running_with_progress(&status, &rebalance_id)? {
         return Err(format!("rebalance did not remain active across the S3 operations: {status}").into());
     }
     wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
-    let after = dist.client(1)?;
-    assert_inventory(&after, &bucket, &inventory).await?;
-    for (key, body) in [
-        ("during-decommission.bin", b"written-while-decommissioning".as_slice()),
-        ("during-rebalance.bin", b"written-while-rebalancing".as_slice()),
-    ] {
-        retrying_get_equals(&after, &bucket, key, body, Duration::from_secs(30)).await?;
-    }
+    assert_inventory(&dist.client(1)?, &bucket, &inventory).await?;
     Ok(())
 }

crates/e2e_test/src/distributed/site_replication_test.rs

@@ -126,165 +126,3 @@ async fn four_node_site_replication_replicates_object_to_peer_site() -> TestResu
     wait_for_replicated_bytes(&site_a.client(3)?, &bucket, reverse_key, &reverse_body, Duration::from_secs(60)).await?;
     Ok(())
 }
-
-async fn node_admin(
-    cluster: &crate::common::RustFSTestClusterEnvironment,
-    node_idx: usize,
-    method: Method,
-    path_and_query: &str,
-    body: Option<String>,
-) -> TestResult<(StatusCode, String)> {
-    crate::common::admin_request(
-        &cluster.nodes[node_idx].url,
-        method,
-        path_and_query,
-        body,
-        &cluster.access_key,
-        &cluster.secret_key,
-    )
-    .await
-}
-
-/// Pair two clusters through site A's first node and wait until both report
-/// the two-site topology as enabled.
-async fn pair_sites(site_a: &DistCluster, site_b: &DistCluster) -> TestResult {
-    let sites = vec![
-        PeerSite {
-            name: "site-a".to_string(),
-            endpoint: site_a.cluster.nodes[0].url.clone(),
-            access_key: site_a.cluster.access_key.clone(),
-            secret_key: site_a.cluster.secret_key.clone(),
-            ..Default::default()
-        },
-        PeerSite {
-            name: "site-b".to_string(),
-            endpoint: site_b.cluster.nodes[0].url.clone(),
-            access_key: site_b.cluster.access_key.clone(),
-            secret_key: site_b.cluster.secret_key.clone(),
-            ..Default::default()
-        },
-    ];
-    let add_status = site_replication_add(&site_a.cluster, &sites).await?;
-    assert!(
-        add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(),
-        "site replication add reported failure: {add_status:?}"
-    );
-    wait_for_site_replication_enabled(&site_a.cluster).await?;
-    wait_for_site_replication_enabled(&site_b.cluster).await?;
-    Ok(())
-}
-
-async fn list_users_contains(
-    cluster: &crate::common::RustFSTestClusterEnvironment,
-    node_idx: usize,
-    access_key: &str,
-) -> TestResult<bool> {
-    let (status, body) = node_admin(cluster, node_idx, Method::GET, "/rustfs/admin/v3/list-users", None).await?;
-    if !status.is_success() {
-        return Err(format!("list-users on node {node_idx} failed: {status} {body}").into());
-    }
-    let users: serde_json::Value = serde_json::from_str(&body)?;
-    Ok(users.get(access_key).is_some())
-}
-
-/// backlog#2367 A-7 / functional SITE-102: an IAM change handled by a node
-/// other than the one that ran `site-replication/add` must still reach the
-/// peer site. Behind a load balancer every admin call may land on a
-/// different node, so the coordinator node is not special.
-#[tokio::test]
-async fn four_node_site_replication_converges_iam_user_created_on_a_non_coordinator_node() -> TestResult {
-    init_logging();
-    let (site_a, site_b) = DistCluster::start_replication_pair().await?;
-    pair_sites(&site_a, &site_b).await?;
-
-    let user = format!("siteuser-{}", &uuid::Uuid::new_v4().simple().to_string()[..8]);
-    let body = serde_json::json!({ "secretKey": "siteuser-secret-key-1234", "status": "enabled" }).to_string();
-    let (status, response) = node_admin(
-        &site_a.cluster,
-        1,
-        Method::PUT,
-        &format!("/rustfs/admin/v3/add-user?accessKey={user}"),
-        Some(body),
-    )
-    .await?;
-    assert!(status.is_success(), "add-user on site A node 1 failed: {status} {response}");
-
-    let site_b_cluster = &site_b.cluster;
-    let user_ref = user.as_str();
-    wait_until(
-        Duration::from_secs(90),
-        || async move { list_users_contains(site_b_cluster, 0, user_ref).await },
-        "user created on site A node 1 visible on site B",
-    )
-    .await?;
-    assert!(
-        list_users_contains(&site_a.cluster, 2, &user).await?,
-        "the user must be visible on every site A node"
-    );
-    Ok(())
-}
-
-/// backlog#2367 A-5 / functional SITE-105: a resync started right after
-/// pairing must not report buckets as failed. The bucket carrying an
-/// operator-configured bucket-replication target to the peer (the shape the
-/// functional suite leaves behind) and a plain versioned bucket are both
-/// wired by the pairing itself.
-#[tokio::test]
-async fn four_node_site_replication_resync_start_right_after_pairing_reports_no_failed_bucket() -> TestResult {
-    init_logging();
-    let (site_a, site_b) = DistCluster::start_replication_pair().await?;
-
-    let pre_src = unique_bucket("pre-src");
-    let pre_dst = unique_bucket("pre-dst");
-    let plain = unique_bucket("plain");
-    site_a.create_bucket(&pre_src).await?;
-    site_b.create_bucket(&pre_dst).await?;
-    site_a.create_bucket(&plain).await?;
-    enable_versioning(&site_a.client(0)?, &pre_src).await?;
-    enable_versioning(&site_b.client(0)?, &pre_dst).await?;
-    enable_versioning(&site_a.client(0)?, &plain).await?;
-    let arn = super::harness::set_remote_target(&site_a.cluster, &pre_src, &site_b.cluster, &pre_dst).await?;
-    super::harness::put_bucket_replication(&site_a.cluster, &pre_src, &arn).await?;
-
-    pair_sites(&site_a, &site_b).await?;
-
-    let (status, info) = node_admin(&site_a.cluster, 1, Method::GET, "/rustfs/admin/v3/site-replication/info", None).await?;
-    assert!(status.is_success(), "site-replication/info failed: {status} {info}");
-    let info: serde_json::Value = serde_json::from_str(&info)?;
-    let peer = info["sites"]
-        .as_array()
-        .and_then(|sites| sites.iter().find(|site| site["name"] == "site-b"))
-        .cloned()
-        .ok_or_else(|| format!("site-b peer missing from info: {info}"))?;
-
-    // Through a non-coordinator node, like a load-balanced admin call.
-    let (status, response) = node_admin(
-        &site_a.cluster,
-        1,
-        Method::PUT,
-        "/rustfs/admin/v3/site-replication/resync/op?operation=start",
-        Some(peer.to_string()),
-    )
-    .await?;
-    assert!(status.is_success(), "resync start failed: {status} {response}");
-    let resync: rustfs_madmin::SRResyncOpStatus = serde_json::from_str(&response)?;
-    let failed: Vec<String> = resync
-        .buckets
-        .iter()
-        .filter(|bucket| bucket.status == "failed")
-        .map(|bucket| format!("{}: {}", bucket.bucket, bucket.err_detail))
-        .collect();
-    assert!(
-        failed.is_empty(),
-        "resync right after pairing reported failed buckets: {failed:?} (status={}, detail={})",
-        resync.status,
-        resync.err_detail
-    );
-    assert!(
-        resync.buckets.iter().any(|bucket| bucket.bucket == pre_src)
-            && resync.buckets.iter().any(|bucket| bucket.bucket == plain),
-        "both buckets must be part of the resync: {:?}",
-        resync.buckets
-    );
-    Ok(())
-}

crates/e2e_test/src/kms/kms_fault_recovery_test.rs

@@ -109,10 +109,10 @@ async fn test_kms_key_directory_unavailable() -> Result<(), Box<dyn std::error::
         .await;
 
     let unavailable_error = put_result2.expect_err("a missing Local KMS key directory must reject encrypted writes");
-    assert_eq!(unavailable_error.raw_response().map(|response| response.status().as_u16()), Some(503));
+    assert_eq!(unavailable_error.raw_response().map(|response| response.status().as_u16()), Some(500));
     assert_eq!(
         unavailable_error.as_service_error().and_then(ProvideErrorMetadata::code),
-        Some("ServiceUnavailable")
+        Some("InternalError")
     );
     let unavailable_absence = s3_client
         .get_object()

crates/e2e_test/src/lib.rs

@@ -12,9 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#[cfg(test)]
-mod storage_metric_ownership_test;
-
 mod reliant;
 mod storage_api;
 
@@ -179,9 +176,6 @@ mod compression_test;
 #[cfg(test)]
 mod delete_objects_versioning_test;
 
-#[cfg(test)]
-mod delete_authorization_test;
-
 // Regression test for signed DELETE Object?versionId requests without Content-Length.
 #[cfg(test)]
 mod delete_object_no_content_length_test;

crates/e2e_test/src/multipart_auth_test.rs

@@ -15,7 +15,6 @@
 //! Regression coverage for anonymous access on multipart control APIs.
 
 use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
-use crate::kms::common::LocalKMSTestEnvironment;
 use async_compression::tokio::write::{BzEncoder, Lz4Encoder, XzEncoder};
 use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
 use aws_sdk_s3::operation::head_object::HeadObjectOutput;
@@ -1466,10 +1465,10 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_s3() -> Result<(), B
 async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
-    let mut kms_env = LocalKMSTestEnvironment::new().await?;
-    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    kms_env.wait_for_kms_ready().await?;
-    let env = &kms_env.base_env;
+    let mut env = RustFSTestEnvironment::new().await?;
+    let master_key = local_sse_master_key_value();
+    env.start_rustfs_server_with_env(vec![], &[(LOCAL_SSE_MASTER_KEY_ENV, master_key.as_str())])
+        .await?;
 
     let bucket = "anon-post-default-sse-kms";
     let object_key = "post-default-sse-kms-object.txt";
@@ -1485,7 +1484,7 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(),
                 .apply_server_side_encryption_by_default(
                     ServerSideEncryptionByDefault::builder()
                         .sse_algorithm(ServerSideEncryption::AwsKms)
-                        .kms_master_key_id(default_key_id)
+                        .kms_master_key_id("test-key")
                         .build()
                         .expect("default encryption rule should build"),
                 )

crates/e2e_test/src/namespace_lock_quorum_test.rs

@@ -17,7 +17,6 @@ use aws_sdk_s3::Client;
 use aws_sdk_s3::error::SdkError;
 use bytes::Bytes;
 use std::sync::Arc;
-use std::time::{Duration, Instant};
 use tokio::sync::Barrier;
 use tracing::{info, warn};
 
@@ -26,307 +25,6 @@ const KEY: &str = "thumb/79/concurrent-overwrite.jpg";
 
 type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
 
-async fn assert_degraded_cluster_publication_guard_errors_are_retryable() -> TestResult {
-    let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
-    cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2");
-    cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
-    cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
-    cluster.set_env("RUST_LOG", "warn");
-    cluster.start().await?;
-    cluster.create_test_bucket(BUCKET).await?;
-    let clients: Vec<_> = cluster
-        .create_all_clients()?
-        .into_iter()
-        .map(|client| {
-            Client::from_conf(
-                client
-                    .config()
-                    .to_builder()
-                    .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
-                    .build(),
-            )
-        })
-        .collect();
-
-    for alive in (1..=4).rev() {
-        if alive < 4 {
-            cluster.stop_node(alive)?;
-        }
-        for (node, client) in clients.iter().take(alive).enumerate() {
-            let key = format!("publication-put-{alive}-{node}");
-            let put = client
-                .put_object()
-                .bucket(BUCKET)
-                .key(&key)
-                .body(Bytes::from_static(b"publication guard regression").into())
-                .send()
-                .await;
-            if alive >= 3 {
-                put?;
-            } else {
-                let err = put.expect_err("PUT must reject writes without a write quorum");
-                assert_eq!(
-                    err.raw_response().map(|response| response.status().as_u16()),
-                    Some(503),
-                    "PUT with {alive} nodes alive, requested through node {node}: {err:?}"
-                );
-                assert_eq!(
-                    err.as_service_error().and_then(|error| error.meta().code()),
-                    Some("ServiceUnavailable"),
-                    "PUT with {alive} nodes alive, requested through node {node}: {err:?}"
-                );
-            }
-
-            let multipart_key = format!("publication-multipart-{alive}-{node}");
-            let multipart = client
-                .create_multipart_upload()
-                .bucket(BUCKET)
-                .key(&multipart_key)
-                .send()
-                .await;
-            if alive >= 3 {
-                let upload = multipart?;
-                let upload_id = upload
-                    .upload_id()
-                    .expect("successful multipart initialization must return an upload ID");
-                client
-                    .abort_multipart_upload()
-                    .bucket(BUCKET)
-                    .key(&multipart_key)
-                    .upload_id(upload_id)
-                    .send()
-                    .await?;
-            } else {
-                let err = multipart.expect_err("multipart initialization must reject writes without a write quorum");
-                assert_eq!(
-                    err.raw_response().map(|response| response.status().as_u16()),
-                    Some(503),
-                    "CreateMultipartUpload with {alive} nodes alive, requested through node {node}: {err:?}"
-                );
-                assert_eq!(
-                    err.as_service_error().and_then(|error| error.meta().code()),
-                    Some("ServiceUnavailable"),
-                    "CreateMultipartUpload with {alive} nodes alive, requested through node {node}: {err:?}"
-                );
-            }
-        }
-    }
-
-    cluster.stop();
-    cluster.start().await?;
-    for client in &clients {
-        for alive in [1, 3, 4] {
-            for node in 0..alive {
-                let key = format!("publication-put-{alive}-{node}");
-                let get = client.get_object().bucket(BUCKET).key(key).send().await;
-                if alive >= 3 {
-                    assert_eq!(
-                        get?.body.collect().await?.into_bytes().as_ref(),
-                        b"publication guard regression",
-                        "acknowledged writes must survive restart"
-                    );
-                } else {
-                    let err = get.expect_err("a rejected publication guard must not publish an object");
-                    assert_eq!(err.as_service_error().and_then(|error| error.meta().code()), Some("NoSuchKey"));
-                }
-            }
-        }
-        let uploads = client.list_multipart_uploads().bucket(BUCKET).send().await?;
-        assert!(
-            uploads
-                .uploads()
-                .iter()
-                .all(|upload| upload.key() != Some("publication-multipart-1-0")),
-            "a rejected publication guard must not publish a multipart upload"
-        );
-    }
-    Ok(())
-}
-
-async fn assert_quorum_object_body(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
-    let body = client
-        .get_object()
-        .bucket(bucket)
-        .key(key)
-        .send()
-        .await?
-        .body
-        .collect()
-        .await?
-        .into_bytes();
-    assert_eq!(body.as_ref(), expected, "quorum read returned incorrect contents for {key}");
-    Ok(())
-}
-
-async fn wait_for_quorum_read_admission(clients: &[Client], bucket: &str) -> TestResult {
-    // SIGKILL can orphan a granted lease. Wait for shared metadata-lock
-    // admission before asserting the stable quorum boundary; cold bodies
-    // remain unread throughout this readiness probe.
-    let deadline =
-        tokio::time::Instant::now() + rustfs_lock::fast_lock::DEFAULT_LOCK_TIMEOUT + std::time::Duration::from_secs(15);
-    loop {
-        let mut ready = true;
-        for client in clients {
-            for key in ["warm-small", "warm-large"] {
-                match client.head_object().bucket(bucket).key(key).send().await {
-                    Ok(_) => {}
-                    Err(error) if error.raw_response().is_some_and(|response| response.status().as_u16() == 503) => {
-                        ready = false;
-                        break;
-                    }
-                    Err(error) => return Err(error.into()),
-                }
-            }
-            if !ready {
-                break;
-            }
-        }
-        if ready {
-            return Ok(());
-        }
-        if tokio::time::Instant::now() >= deadline {
-            return Err(format!("read quorum did not become available after lease convergence for {bucket}").into());
-        }
-        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
-    }
-}
-
-#[tokio::test]
-async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult {
-    crate::common::init_logging();
-
-    for (node_count, parity) in [(4, 2), (6, 3), (6, 2)] {
-        let read_quorum = node_count - parity;
-        let write_quorum = read_quorum + usize::from(read_quorum == parity);
-        let mut cluster = RustFSTestClusterEnvironment::new(node_count).await?;
-        cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{parity}"));
-        // Wait for every seed fanout before removing any physical shard.
-        cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
-        cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
-        cluster.set_env("RUST_LOG", "warn,rustfs_lock=debug");
-        cluster.start().await?;
-
-        let clients = cluster
-            .create_all_clients()?
-            .into_iter()
-            .map(|client| {
-                Client::from_conf(
-                    client
-                        .config()
-                        .to_builder()
-                        .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
-                        .build(),
-                )
-            })
-            .collect::<Vec<_>>();
-        let bucket = format!("read-quorum-{node_count}-{parity}");
-        clients[0].create_bucket().bucket(&bucket).send().await?;
-        let small = b"read quorum is derived from the erasure layout".to_vec();
-        let large = (0..1_048_576)
-            .map(|index| u8::try_from(index % 251).expect("bounded payload byte"))
-            .collect::<Vec<_>>();
-        for (key, body) in [
-            ("warm-small", &small),
-            ("warm-large", &large),
-            ("cold-small", &small),
-            ("cold-large", &large),
-            ("below-quorum", &large),
-        ] {
-            clients[node_count - 1]
-                .put_object()
-                .bucket(&bucket)
-                .key(key)
-                .body(Bytes::copy_from_slice(body).into())
-                .send()
-                .await?;
-        }
-        for node in &cluster.nodes {
-            for key in ["warm-small", "warm-large", "cold-small", "cold-large", "below-quorum"] {
-                let census =
-                    crate::chaos::census_object_version_on_disk(std::path::Path::new(&node.data_dir), &bucket, key, None)?;
-                assert!(census.is_complete(), "seed shard must be complete before fault injection: {census:?}");
-                assert_eq!(census.data_blocks, Some(read_quorum));
-                assert_eq!(census.parity_blocks, Some(parity));
-            }
-        }
-        for client in &clients {
-            assert_quorum_object_body(client, &bucket, "warm-small", &small).await?;
-            assert_quorum_object_body(client, &bucket, "warm-large", &large).await?;
-        }
-
-        for offline_node in (read_quorum..node_count).rev() {
-            cluster.stop_node(offline_node)?;
-            wait_for_quorum_read_admission(&clients[..offline_node], &bucket).await?;
-            for client in clients.iter().take(offline_node) {
-                client.head_bucket().bucket(&bucket).send().await?;
-                assert_quorum_object_body(client, &bucket, "warm-large", &large).await?;
-            }
-        }
-
-        // Exercise more than the five-second positive bucket-validation TTL.
-        // Every sample must succeed; polling must not hide a transient failure.
-        let validation_deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(6);
-        loop {
-            for client in clients.iter().take(read_quorum) {
-                assert_quorum_object_body(client, &bucket, "warm-small", &small).await?;
-                assert_quorum_object_body(client, &bucket, "warm-large", &large).await?;
-                let listing = client.list_objects_v2().bucket(&bucket).send().await?;
-                for key in ["warm-small", "warm-large", "cold-small", "cold-large", "below-quorum"] {
-                    assert!(listing.contents().iter().any(|entry| entry.key() == Some(key)), "listing omitted {key}");
-                }
-            }
-            if tokio::time::Instant::now() >= validation_deadline {
-                break;
-            }
-            tokio::time::sleep(std::time::Duration::from_millis(250)).await;
-        }
-        for client in clients.iter().take(read_quorum) {
-            assert_quorum_object_body(client, &bucket, "cold-small", &small).await?;
-            assert_quorum_object_body(client, &bucket, "cold-large", &large).await?;
-        }
-
-        let write = clients[0]
-            .put_object()
-            .bucket(&bucket)
-            .key("quorum-write")
-            .body(Bytes::copy_from_slice(&small).into())
-            .send()
-            .await;
-        if read_quorum >= write_quorum {
-            write?;
-        } else {
-            let error = write.expect_err("a read quorum must not authorize a write that needs more votes");
-            assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
-        }
-
-        cluster.stop_node(read_quorum - 1)?;
-        for client in clients.iter().take(read_quorum - 1) {
-            match client.get_object().bucket(&bucket).key("below-quorum").send().await {
-                Ok(response) => assert!(
-                    response.body.collect().await.is_err(),
-                    "fewer than {read_quorum} valid fragments must not reconstruct an uncached object"
-                ),
-                Err(error) => assert_eq!(
-                    error.as_service_error().and_then(|error| error.meta().code()),
-                    Some("ServiceUnavailable"),
-                    "a quorum loss must not be mistaken for a missing object"
-                ),
-            }
-        }
-
-        for node in 0..read_quorum - 1 {
-            cluster.stop_node(node)?;
-        }
-        cluster.start().await?;
-        for client in &clients {
-            assert_quorum_object_body(client, &bucket, "warm-large", &large).await?;
-            assert_quorum_object_body(client, &bucket, "below-quorum", &large).await?;
-        }
-    }
-
-    Ok(())
-}
-
 async fn put_object(client: Client, payload: Vec<u8>, writer_id: usize) -> Result<(), String> {
     client
         .put_object()
@@ -360,9 +58,6 @@ async fn test_concurrent_cluster_overwrites_do_not_fail_namespace_lock_quorum()
     // Keep the regression focused on false quorum-loss errors, not ordinary lock
     // wait exhaustion under a heavily contended same-key overwrite workload.
     cluster.set_env("RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT", "20");
-    cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2");
-    cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
-    cluster.set_env("RUSTFS_HEALTH_MINIMAL_RESPONSE_ENABLE", "false");
     cluster.start().await?;
     cluster.create_test_bucket(BUCKET).await?;
 
@@ -421,181 +116,6 @@ async fn test_concurrent_cluster_overwrites_do_not_fail_namespace_lock_quorum()
     );
 
     clients[0].delete_object().bucket(BUCKET).key(KEY).send().await?;
-    assert_node_readiness_tracks_quorum(&mut cluster).await?;
-    Ok(())
-}
-
-async fn assert_node_readiness_tracks_quorum(cluster: &mut RustFSTestClusterEnvironment) -> TestResult {
-    let clients: Vec<_> = cluster
-        .create_all_clients()?
-        .into_iter()
-        .map(|client| {
-            Client::from_conf(
-                client
-                    .config()
-                    .to_builder()
-                    .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
-                    .build(),
-            )
-        })
-        .collect();
-    let http = reqwest::Client::builder()
-        .no_proxy()
-        .timeout(Duration::from_secs(3))
-        .build()?;
-    let seed_key = "readiness-seed";
-    let seed_body = b"readiness quorum regression";
-    clients[0]
-        .put_object()
-        .bucket(BUCKET)
-        .key(seed_key)
-        .body(Bytes::from_static(seed_body).into())
-        .send()
-        .await?;
-
-    for (phase, survivors) in [4, 3, 2, 1, 4].into_iter().enumerate() {
-        if phase == 4 {
-            cluster.stop();
-            cluster.start().await?;
-        } else if survivors < 4 {
-            cluster.stop_node(survivors)?;
-        }
-        let write_ready = survivors >= 3;
-        let read_quorum = survivors >= 2;
-        let expected_status = if write_ready { 200 } else { 503 };
-        for (idx, client) in clients.iter().enumerate().take(survivors) {
-            let url = &cluster.nodes[idx].url;
-            let deadline = Instant::now() + Duration::from_secs(30);
-            // Poll health before issuing S3 I/O: idle remote disk handles must
-            // not remain evidence of quorum after their host becomes unreachable.
-            let payload = loop {
-                let response = http.get(format!("{url}/health/ready")).send().await?;
-                let status = response.status().as_u16();
-                let payload: serde_json::Value = response.json().await?;
-                if status == expected_status
-                    && payload["ready"] == write_ready
-                    && payload["details"]["storage"]["ready"] == write_ready
-                    && payload["details"]["storage"]["readQuorum"] == read_quorum
-                    && payload["details"]["storage"]["writeQuorum"] == write_ready
-                    && payload["details"]["poolMetadata"]["ready"] == true
-                    && payload["details"]["iam"]["ready"] == true
-                    && payload["details"]["lock"]["ready"] == write_ready
-                {
-                    break payload;
-                }
-                assert!(Instant::now() < deadline, "node {idx}, survivors={survivors}: HTTP {status}, {payload}");
-                tokio::time::sleep(Duration::from_millis(200)).await;
-            };
-            assert_eq!(payload["details"]["storage"]["readinessScope"], "write_quorum_and_pool_metadata");
-            assert_eq!(payload["details"]["storage"]["source"], "local_runtime");
-            assert_eq!(
-                payload["details"]["storage"]["status"],
-                if write_ready { "connected" } else { "disconnected" }
-            );
-            if !write_ready {
-                assert!(
-                    payload["degradedReasons"]
-                        .as_array()
-                        .expect("degraded reasons")
-                        .iter()
-                        .any(|reason| reason == "storage_and_lock_unavailable")
-                );
-            }
-
-            for path in ["/health/ready", "/minio/health/ready"] {
-                let head = http.head(format!("{url}{path}")).send().await?;
-                assert_eq!(head.status().as_u16(), expected_status, "HEAD {path}, survivors={survivors}");
-                assert!(head.bytes().await?.is_empty());
-                let response = http.get(format!("{url}{path}")).send().await?;
-                assert_eq!(response.status().as_u16(), expected_status);
-                let body: serde_json::Value = response.json().await?;
-                assert_eq!(body["details"]["storage"], payload["details"]["storage"]);
-                assert_eq!(body["details"]["poolMetadata"], payload["details"]["poolMetadata"]);
-            }
-            let live = http.get(format!("{url}/health/live")).send().await?;
-            assert_eq!(live.status().as_u16(), 200);
-            assert!(live.json::<serde_json::Value>().await?.get("details").is_none());
-            for (path, storage_ready, scope) in [
-                ("/minio/health/cluster", write_ready, "write_quorum_and_pool_metadata"),
-                ("/minio/health/cluster/read", read_quorum, "read_quorum"),
-            ] {
-                let deadline = Instant::now() + Duration::from_secs(30);
-                // Cluster read/write reports have independent caches; allow
-                // each observation to expire before comparing stable states.
-                let body = loop {
-                    let response = http.get(format!("{url}{path}")).send().await?;
-                    let status = response.status().as_u16();
-                    let body: serde_json::Value = response.json().await?;
-                    if status == expected_status
-                        && body["details"]["storage"]["ready"] == storage_ready
-                        && body["details"]["lock"]["ready"] == write_ready
-                    {
-                        break body;
-                    }
-                    assert!(Instant::now() < deadline, "{path}, survivors={survivors}: HTTP {status}, {body}");
-                    tokio::time::sleep(Duration::from_millis(200)).await;
-                };
-                assert_eq!(body["details"]["storage"]["readinessScope"], scope);
-            }
-
-            let put = client
-                .put_object()
-                .bucket(BUCKET)
-                .key(format!("readiness-phase-{phase}-node-{idx}"))
-                .body(Bytes::from_static(seed_body).into())
-                .send()
-                .await;
-            let put_status = if write_ready {
-                put.expect("a ready node must accept the PUT");
-                200
-            } else {
-                let error = put.expect_err("subquorum node must reject PUT");
-                assert!(
-                    error.raw_response().is_some_and(|response| response.status().as_u16() >= 500),
-                    "unexpected PUT failure: {error:?}"
-                );
-                error.raw_response().expect("PUT error response").status().as_u16()
-            };
-            let get = client.get_object().bucket(BUCKET).key(seed_key).send().await;
-            let get_status = match get {
-                Ok(object) => {
-                    assert_eq!(object.body.collect().await?.into_bytes().as_ref(), seed_body);
-                    200
-                }
-                Err(error) => {
-                    assert!(!write_ready, "GET must succeed on a ready cluster: {error:?}");
-                    let status = error
-                        .raw_response()
-                        .expect("GET should have an HTTP response")
-                        .status()
-                        .as_u16();
-                    assert!(status >= 500, "unexpected GET failure: {error:?}");
-                    status
-                }
-            };
-            let list = client.list_objects_v2().bucket(BUCKET).send().await;
-            let list_status = match list {
-                Ok(result) => {
-                    assert!(result.contents().iter().any(|object| object.key() == Some(seed_key)));
-                    200
-                }
-                Err(error) => {
-                    assert!(!write_ready, "listing must succeed on a ready cluster: {error:?}");
-                    let status = error
-                        .raw_response()
-                        .expect("LIST should have an HTTP response")
-                        .status()
-                        .as_u16();
-                    assert!(status >= 500, "unexpected listing failure: {error:?}");
-                    status
-                }
-            };
-            eprintln!(
-                "readiness matrix: survivors={survivors}, node={idx}, ready={write_ready}, read_quorum={read_quorum}, PUT={put_status}, GET={get_status}, LIST={list_status}"
-            );
-        }
-    }
-    cluster.stop();
     Ok(())
 }
 
@@ -605,8 +125,6 @@ async fn assert_node_readiness_tracks_quorum(cluster: &mut RustFSTestClusterEnvi
 /// Before the fix, `map_namespace_lock_error` wrapped lock timeout/conflict errors as
 /// `StorageError::other(...)` → `StorageError::Io(...)`, which fell through to
 /// `S3ErrorCode::InternalError` (500) in the error mapping.
-/// Also checks PUT and multipart initialization when node failures prevent
-/// acquiring a table publication guard.
 #[tokio::test]
 async fn test_concurrent_put_same_key_never_returns_500() -> TestResult {
     crate::common::init_logging();
@@ -709,6 +227,5 @@ async fn test_concurrent_put_same_key_never_returns_500() -> TestResult {
     );
 
     clients[0].delete_object().bucket(BUCKET).key(KEY).send().await?;
-    cluster.stop();
-    assert_degraded_cluster_publication_guard_errors_are_retryable().await
+    Ok(())
 }

crates/e2e_test/src/on_demand_migration/interaction_test.rs

@@ -1137,12 +1137,7 @@ async fn test_odm_admin_config_is_redacted_and_status_counts_match_the_source()
         let miss = env.raw_get(bucket, miss_key).await?;
         assert_eq!(miss.status, 404, "{}", String::from_utf8_lossy(&miss.body));
     }
-    let (listed, _, _) = tokio::try_join!(
-        env.wait_local_listed(bucket, hit_key, SETTLE),
-        env.wait_for_status_counter(bucket, "/counters/pulled_objects_total/inline", 1, SETTLE),
-        env.wait_for_status_counter(bucket, "/counters/pulled_bytes_total", body.len() as u64, SETTLE),
-    )?;
-    assert!(listed);
+    assert!(env.wait_local_listed(bucket, hit_key, SETTLE).await?);
 
     let status = env.status_json(bucket).await?;
     assert_eq!(status.pointer("/configured").and_then(Value::as_bool), Some(true), "{status}");

crates/e2e_test/src/replication_extension_test.rs

@@ -6383,18 +6383,6 @@ async fn test_site_replication_edit_and_status_peer_state_real_three_node() -> R
     let relayed_key = "after-edit-from-relay.txt";
     let relayed_payload = b"site replication after endpoint edit from relay".to_vec();
 
-    // The first joining receiver owns data before the third site has the
-    // shared account. Initial probes and backfill must wait for every join.
-    target_client.create_bucket().bucket(bucket).send().await?;
-    enable_bucket_versioning(&target_env, bucket).await?;
-    target_client
-        .put_object()
-        .bucket(bucket)
-        .key(baseline_key)
-        .body(ByteStream::from(baseline_payload.clone()))
-        .send()
-        .await?;
-
     let add_status = site_replication_add(
         &source_env,
         &[
@@ -6422,10 +6410,7 @@ async fn test_site_replication_edit_and_status_peer_state_real_three_node() -> R
         ],
     )
     .await?;
-    assert!(
-        add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(),
-        "unexpected site add result: {add_status:?}"
-    );
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
 
     let source_info = wait_for_site_replication_enabled(&source_env, 3).await?;
     let _target_info = wait_for_site_replication_enabled(&target_env, 3).await?;
@@ -6436,11 +6421,19 @@ async fn test_site_replication_edit_and_status_peer_state_real_three_node() -> R
         .find(|peer| peer.endpoint == target_env.url)
         .ok_or("target peer missing from source site replication info")?;
 
-    for client in [&source_client, &relay_client] {
-        wait_for_bucket_on_target(client, bucket).await?;
-        let backfilled = wait_for_object_on_target(client, bucket, baseline_key).await?;
-        assert_eq!(backfilled, baseline_payload);
-    }
+    source_client.create_bucket().bucket(bucket).send().await?;
+    enable_bucket_versioning(&source_env, bucket).await?;
+    wait_for_bucket_on_target(&target_client, bucket).await?;
+    wait_for_bucket_on_target(&relay_client, bucket).await?;
+    source_client
+        .put_object()
+        .bucket(bucket)
+        .key(baseline_key)
+        .body(ByteStream::from(baseline_payload.clone()))
+        .send()
+        .await?;
+    let replicated_baseline = wait_for_object_on_target(&target_client, bucket, baseline_key).await?;
+    assert_eq!(replicated_baseline, baseline_payload);
 
     let old_target_address = target_env.address.clone();
     let new_target_port = RustFSTestEnvironment::find_available_port().await?;

crates/e2e_test/src/storage_metric_ownership_test.rs

@@ -1,335 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Native OTLP -> Collector -> Prometheus contract, including a rolling upgrade.
-
-use crate::common::RustFSTestClusterEnvironment;
-use aws_sdk_s3::primitives::ByteStream;
-use serde_json::Value;
-use std::fs::{self, File};
-use std::net::TcpListener;
-use std::path::{Path, PathBuf};
-use std::process::{Child, Command, Stdio};
-use std::time::{Duration, Instant};
-
-type TestResult<T = ()> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
-
-struct ToolProcess(Child);
-impl Drop for ToolProcess {
-    fn drop(&mut self) {
-        let _ = self.0.kill();
-        let _ = self.0.wait();
-    }
-}
-
-fn required_binary(name: &str) -> TestResult<PathBuf> {
-    let path = PathBuf::from(std::env::var(name).map_err(|_| format!("{name} must name a pinned executable"))?);
-    if !path.is_file() {
-        return Err(format!("{name} does not name a file: {}", path.display()).into());
-    }
-    Ok(path)
-}
-
-fn free_port() -> TestResult<u16> {
-    Ok(TcpListener::bind("127.0.0.1:0")?.local_addr()?.port())
-}
-
-fn start_tool(binary: &Path, args: &[String], log: &Path) -> TestResult<ToolProcess> {
-    let log = File::create(log)?;
-    Ok(ToolProcess(
-        Command::new(binary)
-            .args(args)
-            .env("NO_PROXY", "127.0.0.1,localhost")
-            .env_remove("HTTP_PROXY")
-            .env_remove("HTTPS_PROXY")
-            .stdout(Stdio::from(log.try_clone()?))
-            .stderr(Stdio::from(log))
-            .spawn()?,
-    ))
-}
-
-async fn query(client: &reqwest::Client, base: &str, expression: &str) -> TestResult<Value> {
-    let mut url = reqwest::Url::parse(&format!("{base}/api/v1/query"))?;
-    url.query_pairs_mut().append_pair("query", expression);
-    let result: Value = client.get(url).send().await?.error_for_status()?.json().await?;
-    if result["status"] != "success" {
-        return Err(format!("PromQL failed: {result}").into());
-    }
-    Ok(result["data"]["result"].clone())
-}
-
-async fn await_count(client: &reqwest::Client, base: &str, selector: &str, expected: u64) -> TestResult {
-    let deadline = Instant::now() + Duration::from_secs(120);
-    loop {
-        let result = query(client, base, &format!("count({selector}) or vector(0)")).await;
-        if let Ok(rows) = &result
-            && rows[0]["value"][1].as_str().and_then(|value| value.parse::<u64>().ok()) == Some(expected)
-        {
-            println!("PASS count={expected}: {selector}");
-            return Ok(());
-        }
-        if Instant::now() >= deadline {
-            return Err(format!("expected {expected} for {selector}; last result: {result:?}").into());
-        }
-        tokio::time::sleep(Duration::from_millis(250)).await;
-    }
-}
-
-async fn validate_dashboard_queries(client: &reqwest::Client, base: &str, observer: &str) -> TestResult {
-    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../.docker/observability/grafana/dashboards/rustfs.json");
-    let dashboard: Value = serde_json::from_str(&fs::read_to_string(path)?)?;
-    for name in ["storage_cluster", "storage_observer"] {
-        let variable = dashboard["templating"]["list"]
-            .as_array()
-            .ok_or("dashboard variables")?
-            .iter()
-            .find(|variable| variable["name"] == name)
-            .ok_or("storage selection variable")?;
-        assert_eq!(variable["multi"], false, "storage views must select one {name}");
-        assert_eq!(variable["includeAll"], false, "storage views must select one {name}");
-    }
-    let mut pending = dashboard["panels"]
-        .as_array()
-        .ok_or("dashboard panels")?
-        .iter()
-        .collect::<Vec<_>>();
-    let mut checked = 0;
-    while let Some(panel) = pending.pop() {
-        if let Some(children) = panel["panels"].as_array() {
-            pending.extend(children);
-        }
-        for target in panel["targets"].as_array().into_iter().flatten() {
-            let Some(expression) = target["expr"].as_str() else { continue };
-            if !expression.contains("rustfs:storage:current") {
-                continue;
-            }
-            if expression.contains("collection_scope=\"cluster\"") {
-                assert!(
-                    expression.contains("observer=\"$storage_observer\""),
-                    "global views must select one observer: {expression}"
-                );
-            }
-            let mut expression = expression.to_string();
-            for (name, value) in [
-                ("$__rate_interval", "5m"),
-                ("$storage_cluster", "metrics-e2e"),
-                ("$storage_observer", observer),
-                ("$drive_api", ".*"),
-                ("$server", ".*"),
-                ("$drive", ".*"),
-                ("$job", "rustfs"),
-            ] {
-                expression = expression.replace(name, value);
-            }
-            query(client, base, &expression).await?;
-            checked += 1;
-        }
-    }
-    assert!(checked > 0, "the storage dashboard queries must be exercised");
-    println!("PASS: {checked} storage dashboard queries against the live pipeline");
-    Ok(())
-}
-
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-#[ignore = "external tools: pinned Collector, Prometheus, previous release and current RustFS binaries"]
-async fn storage_metric_ownership_pipeline() -> TestResult {
-    let baseline = required_binary("RUSTFS_METRICS_BASELINE_BINARY")?;
-    let current = required_binary("CARGO_BIN_EXE_rustfs")?;
-    let collector = required_binary("RUSTFS_OTELCOL_BINARY")?;
-    let prometheus = required_binary("RUSTFS_PROMETHEUS_BINARY")?;
-    let temp = tempfile::Builder::new().prefix("rustfs-storage-metrics-").tempdir()?;
-    let work = if let Ok(path) = std::env::var("RUSTFS_METRICS_E2E_ARTIFACTS") {
-        let path = PathBuf::from(path);
-        fs::create_dir_all(&path)?;
-        tempfile::Builder::new().prefix("storage-run-").tempdir_in(path)?.keep()
-    } else {
-        temp.path().to_path_buf()
-    };
-    println!("Metrics pipeline logs: {}", work.display());
-    let otlp = free_port()?;
-    let scrape = free_port()?;
-    let prom = free_port()?;
-    let collector_config = work.join("collector.yaml");
-    fs::write(
-        &collector_config,
-        format!(
-            r#"receivers:
-  otlp:
-    protocols:
-      http:
-        endpoint: 127.0.0.1:{otlp}
-exporters:
-  prometheus:
-    endpoint: 127.0.0.1:{scrape}
-    send_timestamps: true
-    metric_expiration: 5m
-    resource_to_telemetry_conversion:
-      enabled: true
-service:
-  telemetry:
-    metrics:
-      level: none
-    logs:
-      level: warn
-  pipelines:
-    metrics:
-      receivers: [otlp]
-      exporters: [prometheus]
-"#
-        ),
-    )?;
-    let rules = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-        .join("../../.docker/observability/prometheus-rules/rustfs-storage.yml")
-        .canonicalize()?;
-    let prom_config = work.join("prometheus.yaml");
-    fs::write(
-        &prom_config,
-        format!(
-            r#"global:
-  scrape_interval: 1s
-  evaluation_interval: 1s
-rule_files:
-  - '{}'
-scrape_configs:
-  - job_name: rustfs
-    static_configs:
-      - targets: ['127.0.0.1:{scrape}']
-"#,
-            rules.display()
-        ),
-    )?;
-    // Use the shipped expressions, with only the test evaluation interval shortened.
-    let test_rules = work.join("storage-rules.yaml");
-    fs::write(&test_rules, fs::read_to_string(&rules)?.replace("interval: 15s", "interval: 1s"))?;
-    fs::write(
-        &prom_config,
-        fs::read_to_string(&prom_config)?.replace(&rules.display().to_string(), &test_rules.display().to_string()),
-    )?;
-    let _collector = start_tool(
-        &collector,
-        &[format!("--config={}", collector_config.display())],
-        &work.join("collector.log"),
-    )?;
-    let _prometheus = start_tool(
-        &prometheus,
-        &[
-            format!("--config.file={}", prom_config.display()),
-            format!("--web.listen-address=127.0.0.1:{prom}"),
-            format!("--storage.tsdb.path={}", work.join("prometheus-data").display()),
-        ],
-        &work.join("prometheus.log"),
-    )?;
-    let http = reqwest::Client::builder()
-        .no_proxy()
-        .timeout(Duration::from_secs(5))
-        .build()?;
-    let prom_url = format!("http://127.0.0.1:{prom}");
-    await_count(&http, &prom_url, "up{job=\"rustfs\"} == 1", 1).await?;
-
-    let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
-    cluster.set_env("NO_PROXY", "127.0.0.1,localhost");
-    cluster.set_env("RUSTFS_OBS_METRIC_ENDPOINT", format!("http://127.0.0.1:{otlp}/v1/metrics"));
-    cluster.set_env("OTEL_RESOURCE_ATTRIBUTES", "rustfs.cluster.id=metrics-e2e");
-    cluster.set_env("RUSTFS_OBS_METER_INTERVAL", "2");
-    cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "true");
-    cluster.set_env("RUSTFS_OBS_LOGS_EXPORT_ENABLED", "false");
-    cluster.set_env("RUSTFS_OBS_TRACES_EXPORT_ENABLED", "false");
-    cluster.set_env("RUSTFS_METRICS_NODE_INTERVAL", "2");
-    cluster.set_env("RUSTFS_METRICS_CLUSTER_INTERVAL", "5");
-    for index in 0..4 {
-        // Four localhost processes otherwise share the startup resource IP.
-        // Distinct test host IDs model the four hosts in a distributed deployment.
-        cluster.set_node_env(
-            index,
-            "OTEL_RESOURCE_ATTRIBUTES",
-            format!("rustfs.cluster.id=metrics-e2e,host.id=metrics-node-{index}"),
-        )?;
-        cluster.set_node_capture_log_path(index, work.join(format!("node-{index}.log")).display().to_string())?;
-    }
-    cluster.start_with_binary(&baseline).await?;
-    // Reproduce the original four observers x four global drives before fixing it.
-    await_count(&http, &prom_url, "rustfs_system_drive_total_bytes{collection_scope=\"\",drive!=\"\"}", 16).await?;
-    let local = "rustfs:storage:current{source_metric=\"rustfs_system_drive_total_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"metrics-e2e\"}";
-    for index in 0..4 {
-        cluster.stop_node_gracefully(index).await?;
-        cluster.start_node_from_binary(index, &current).await?;
-        await_count(&http, &prom_url, local, u64::try_from(index + 1)?).await?;
-    }
-    let rows = query(&http, &prom_url, local).await?;
-    for row in rows.as_array().ok_or("expected a metric vector")? {
-        assert_eq!(
-            row["metric"]["observer"], row["metric"]["server"],
-            "a node must only export its own detailed drives"
-        );
-    }
-    let observer = cluster.nodes[0].address.clone();
-    let inventory = format!(
-        "rustfs:storage:current{{source_metric=\"rustfs_cluster_drive_present\",collection_scope=\"cluster\",rustfs_cluster_id=\"metrics-e2e\",observer=\"{observer}\"}}"
-    );
-    await_count(&http, &prom_url, &inventory, 4).await?;
-    cluster.create_test_bucket("metrics-ownership").await?;
-    let client = cluster.create_s3_client(0)?;
-    for index in 0..8 {
-        client
-            .put_object()
-            .bucket("metrics-ownership")
-            .key(format!("object-{index}"))
-            .body(ByteStream::from(vec![7_u8; 4096]))
-            .send()
-            .await?;
-    }
-    await_count(
-        &http,
-        &prom_url,
-        "count by (server) (rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\"})",
-        4,
-    ).await?;
-    let counters = query(
-        &http,
-        &prom_url,
-        "rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\"}",
-    )
-    .await?;
-    assert!(
-        !counters.as_array().ok_or("expected counters")?.is_empty(),
-        "exercise actual storage counters"
-    );
-    for row in counters.as_array().ok_or("expected counters")? {
-        assert_eq!(row["metric"]["observer"], row["metric"]["server"]);
-        assert!(
-            !row["metric"]["disk_id"].as_str().unwrap_or_default().is_empty(),
-            "counters must carry physical disk identity"
-        );
-    }
-    validate_dashboard_queries(&http, &prom_url, &observer).await?;
-    for index in (1..4).rev() {
-        cluster.stop_node(index)?;
-        // The Collector stays alive; cached samples must not keep stopped owners fresh.
-        await_count(&http, &prom_url, local, u64::try_from(index)?).await?;
-        await_count(&http, &prom_url, &inventory, 4).await?;
-        let unavailable = format!(
-            "rustfs:storage:current{{source_metric=\"rustfs_cluster_drive_runtime_state\",collection_scope=\"cluster\",rustfs_cluster_id=\"metrics-e2e\",observer=\"{observer}\",state=~\"offline|unknown|suspect\"}} == 1"
-        );
-        await_count(&http, &prom_url, &unavailable, u64::try_from(4 - index)?).await?;
-    }
-    cluster.stop();
-    await_count(&http, &prom_url, local, 0).await?;
-    cluster.start_with_binary(&current).await?;
-    await_count(&http, &prom_url, local, 4).await?;
-    await_count(&http, &prom_url, &inventory, 4).await?;
-    let restored = client.get_object().bucket("metrics-ownership").key("object-0").send().await?;
-    assert_eq!(restored.body.collect().await?.into_bytes().as_ref(), vec![7_u8; 4096].as_slice());
-    println!("PASS: baseline duplication; four rolling upgrades; owner identity; counters; 4 -> 3 -> 2 -> 1 -> 0 -> 4 recovery");
-    Ok(())
-}

crates/ecstore/src/api/mod.rs

@@ -38,16 +38,6 @@ pub mod bucket {
     }
 
     pub mod lifecycle {
-        pub mod legacy_transition_state_reconcile {
-            pub use crate::bucket::lifecycle::legacy_transition_state_reconcile::{
-                LegacyTransitionStateCopyRepresentation, LegacyTransitionStateMetadataAlias, LegacyTransitionStateReconcileError,
-                LegacyTransitionStateReconcileOutcome, LegacyTransitionStateReconcileReadiness,
-                LegacyTransitionStateReconcileRequest, LegacyTransitionStateReconcileResponse,
-                LegacyTransitionStateReconcileSelector, LegacyTransitionStateSetRepresentation, LegacyTransitionStateSource,
-                LegacyTransitionStateTarget,
-            };
-        }
-
         pub mod bucket_lifecycle_audit {
             pub use crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
         }
@@ -202,9 +192,7 @@ pub mod bucket {
     }
 
     pub mod migration {
-        pub use crate::bucket::migration::{
-            LegacyBlobDecryptFn, migration_startup_error, try_migrate_bucket_metadata, try_migrate_iam_config,
-        };
+        pub use crate::bucket::migration::{LegacyBlobDecryptFn, try_migrate_bucket_metadata, try_migrate_iam_config};
     }
 
     pub mod object_lock {
@@ -272,19 +260,18 @@ pub mod bucket {
             ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
             ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
             ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
-            ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, ScannerDirtyUsageMutationObserver,
-            ScannerDirtyUsageMutationSource, TargetReplicationResyncStatus, VersionPurgeStatusType, XferStats,
-            assign_site_replication_rule_priorities, commit_force_delete_intent, complete_force_delete_intent,
-            delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
-            get_global_replication_stats, get_proxy_targets, init_background_replication,
+            ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
+            VersionPurgeStatusType, XferStats, assign_site_replication_rule_priorities, commit_force_delete_intent,
+            complete_force_delete_intent, delete_replication_state_from_config, delete_replication_version_id,
+            get_global_replication_pool, get_global_replication_stats, get_proxy_targets, init_background_replication,
             invalid_replication_config_status_field, is_site_replication_role, is_site_replication_rule,
             merge_incoming_replication_config, merge_user_replication_config, persist_force_delete_intent,
             read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map,
             replication_target_arn_deployment_id, replication_target_arns, resync_start_conflict_id,
-            set_scanner_dirty_usage_mutation_observer, should_remove_replication_target, should_schedule_delete_replication,
-            should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
-            site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
-            validate_replication_config_target_arns, version_purge_status_to_filemeta,
+            should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info,
+            should_use_existing_delete_replication_source, site_replication_rule_deployment_id,
+            unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
+            version_purge_status_to_filemeta,
         };
     }
 
@@ -379,14 +366,6 @@ pub mod config {
 }
 
 pub mod data_usage {
-    #[cfg(feature = "test-util")]
-    pub use crate::data_movement::SourceCleanupDeleteBarrier;
-    #[cfg(feature = "test-util")]
-    pub use crate::data_movement::scanner_backlog::test_util::NativeScannerPauseBacklogWriteFault;
-    pub use crate::data_movement::scanner_backlog::{
-        MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementPlan, ScannerPauseBacklogRetirementPlanner,
-        ScannerPauseBacklogRetirementReplica, register_scanner_pause_backlog_retirement_planner,
-    };
     pub use crate::data_usage::{
         DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
         init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -966,11 +966,6 @@ async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel:
     if let Some(err) = first_error {
         return Err(err);
     }
-    runtime_sources::notify_scanner_dirty_usage_mutation(
-        &oi.bucket,
-        &oi.name,
-        runtime_sources::ScannerDirtyUsageMutationSource::TierExpiration,
-    );
     Ok(true)
 }
 
@@ -1220,7 +1215,7 @@ impl ExpiryState {
 
         while state.tasks_tx.len() < n {
             let (tx, rx) = mpsc::channel(EXPIRY_WORKER_QUEUE_CAPACITY);
-            let api = Arc::downgrade(&api);
+            let api = api.clone();
             let rx = Arc::new(tokio::sync::Mutex::new(rx));
             let stats = Arc::clone(&state.stats);
             let recovery_notify = Arc::clone(&state.recovery_notify);
@@ -1248,18 +1243,14 @@ impl ExpiryState {
 
     async fn worker(
         rx: &mut Receiver<Option<ExpiryOpType>>,
-        api: Weak<ECStore>,
+        api: Arc<ECStore>,
         stats: Arc<ExpiryStats>,
         recovery_notify: Arc<Notify>,
     ) {
-        let Some(initial_api) = api.upgrade() else {
-            return;
-        };
-        let cancel_token = initial_api.ctx.background_cancel_token().unwrap_or_else(|| {
+        let cancel_token = api.ctx.background_cancel_token().unwrap_or_else(|| {
             static FALLBACK: std::sync::OnceLock<tokio_util::sync::CancellationToken> = std::sync::OnceLock::new();
             FALLBACK.get_or_init(tokio_util::sync::CancellationToken::new).clone()
         });
-        drop(initial_api);
 
         loop {
             select! {
@@ -1288,9 +1279,6 @@ impl ExpiryState {
                     let v = v.expect("received None after None check");
                     stats.decrement_pending_tasks();
                     let _active_task = ExpiryActiveTask::begin(Arc::clone(&stats));
-                    let Some(api) = api.upgrade() else {
-                        return;
-                    };
                     if v.as_any().is::<ExpiryTask>() {
                         let v = v.as_any().downcast_ref::<ExpiryTask>().expect("ExpiryTask downcast failed");
                         //debug!("lifecycle expiry worker received task: {:?}", v.obj_info);
@@ -4761,11 +4749,6 @@ async fn expire_transitioned_object_with_lock_lost_signal(
                 // Drop any cached restored-copy body so it does not sit resident
                 // until TTL after the copy is expired (ODC-26).
                 crate::object_api::notify_object_mutation(&oi.bucket, &oi.name).await;
-                runtime_sources::notify_scanner_dirty_usage_mutation(
-                    &oi.bucket,
-                    &oi.name,
-                    runtime_sources::ScannerDirtyUsageMutationSource::TierExpiration,
-                );
                 //audit_log_lifecycle(*oi, ILMExpiry, tags, traceFn);
                 Ok(dobj)
             }
@@ -4801,11 +4784,6 @@ async fn expire_transitioned_object_with_lock_lost_signal(
     // The transitioned version is gone; evict any cached body for this object
     // so it does not linger until TTL (ODC-26).
     crate::object_api::notify_object_mutation(&oi.bucket, &oi.name).await;
-    runtime_sources::notify_scanner_dirty_usage_mutation(
-        &oi.bucket,
-        &oi.name,
-        runtime_sources::ScannerDirtyUsageMutationSource::TierExpiration,
-    );
 
     //audit_log_lifecycle(oi, ILMExpiry, tags);
 
@@ -7766,9 +7744,8 @@ mod tests {
         let (tx, mut rx) = tokio::sync::mpsc::channel(2);
         let worker_stats = Arc::clone(&stats);
         let worker_notify = Arc::clone(&recovery_notify);
-        let worker_store = Arc::downgrade(&ecstore);
         let worker = tokio::spawn(async move {
-            ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await;
+            ExpiryState::worker(&mut rx, ecstore, worker_stats, worker_notify).await;
         });
         let oi = ObjectInfo {
             bucket: "bucket".to_string(),
@@ -7869,9 +7846,8 @@ mod tests {
         let (tx, mut rx) = tokio::sync::mpsc::channel(2);
         let worker_stats = Arc::clone(&stats);
         let worker_notify = Arc::clone(&recovery_notify);
-        let worker_store = Arc::downgrade(&ecstore);
         let worker = tokio::spawn(async move {
-            ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await;
+            ExpiryState::worker(&mut rx, ecstore, worker_stats, worker_notify).await;
         });
 
         stats.increment_pending_tasks();
@@ -8040,7 +8016,7 @@ mod tests {
         let (tx, mut rx) = tokio::sync::mpsc::channel(2);
         let worker_stats = Arc::clone(&stats);
         let worker_notify = Arc::clone(&recovery_notify);
-        let worker_store = Arc::downgrade(&ecstore);
+        let worker_store = Arc::clone(&ecstore);
         let worker = tokio::spawn(async move {
             ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await;
         });
@@ -8127,7 +8103,7 @@ mod tests {
         let (tx, mut rx) = tokio::sync::mpsc::channel(2);
         let worker_stats = Arc::clone(&stats);
         let worker_notify = Arc::clone(&recovery_notify);
-        let worker_store = Arc::downgrade(&ecstore);
+        let worker_store = Arc::clone(&ecstore);
         let worker = tokio::spawn(async move {
             ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await;
         });
@@ -8234,7 +8210,7 @@ mod tests {
         let (tx, mut rx) = tokio::sync::mpsc::channel(2);
         let worker_stats = Arc::clone(&stats);
         let worker_notify = Arc::clone(&recovery_notify);
-        let worker_store = Arc::downgrade(&ecstore);
+        let worker_store = Arc::clone(&ecstore);
         let worker = tokio::spawn(async move {
             ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await;
         });
@@ -8288,9 +8264,8 @@ mod tests {
         let (tx, mut rx) = tokio::sync::mpsc::channel(2);
         let worker_stats = Arc::clone(&stats);
         let worker_notify = Arc::clone(&recovery_notify);
-        let worker_store = Arc::downgrade(&ecstore);
         let worker = tokio::spawn(async move {
-            ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await;
+            ExpiryState::worker(&mut rx, ecstore, worker_stats, worker_notify).await;
         });
         let oi = ObjectInfo {
             bucket: format!("missing-bucket-{}", Uuid::new_v4()),

crates/ecstore/src/bucket/lifecycle/mod.rs

@@ -18,7 +18,6 @@ mod config_boundary;
 pub mod core;
 mod durable_namespace;
 pub mod evaluator;
-pub mod legacy_transition_state_reconcile;
 pub mod manual_transition_job;
 mod metadata_boundary;
 pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs, get_lifecycle_config};

crates/ecstore/src/bucket/lifecycle/runtime_boundary.rs

@@ -20,7 +20,6 @@ use tokio_util::sync::CancellationToken;
 
 use crate::bucket::lifecycle::bucket_lifecycle_ops::{ExpiryState, TransitionState};
 use crate::runtime::sources;
-pub(crate) use crate::runtime::sources::ScannerDirtyUsageMutationSource;
 use crate::services::tier::tier::TierConfigMgr;
 use crate::store::ECStore;
 
@@ -55,7 +54,3 @@ pub(crate) fn deployment_id() -> Option<String> {
 pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<BucketLifecycleConfiguration> {
     sources::bucket_lifecycle_config(bucket).await
 }
-
-pub(crate) fn notify_scanner_dirty_usage_mutation(bucket: &str, object: &str, source: ScannerDirtyUsageMutationSource) {
-    sources::notify_scanner_dirty_usage_mutation(bucket, object, source);
-}

crates/ecstore/src/bucket/metadata_sys.rs

@@ -248,9 +248,10 @@ fn validate_authoritative_object_lock_config(config: &ObjectLockConfiguration) -
 }
 
 pub async fn init_bucket_metadata_sys(api: Arc<ECStore>, buckets: Vec<String>) {
-    // The metadata system is inherently per-store, so it lives on the store's
-    // own instance context (backlog#1052 S3). It resolves the store through a
-    // weak handle so the context cache cannot keep the store and disks alive.
+    // The metadata system is inherently per-store (it holds the store handle
+    // and that store's bucket cache), so it lives on the store's own instance
+    // context (backlog#1052 S3) — a second instance initializes its own cell
+    // instead of panicking on the process-global one.
     let instance_ctx = api.ctx.clone();
     let is_dist_erasure = instance_ctx.is_dist_erasure().await;
 
@@ -316,22 +317,18 @@ fn start_refresh_buckets_metadata_loop(sys: Arc<RwLock<BucketMetadataSys>>) {
         warn!("bucket metadata refresh loop skipped because background cancellation token is not initialized");
         return;
     };
-    let sys = Arc::downgrade(&sys);
 
     tokio::spawn(async move {
         refresh_buckets_metadata_loop(sys, cancel_token).await;
     });
 }
 
-async fn refresh_buckets_metadata_loop(sys: Weak<RwLock<BucketMetadataSys>>, cancel_token: CancellationToken) {
+async fn refresh_buckets_metadata_loop(sys: Arc<RwLock<BucketMetadataSys>>, cancel_token: CancellationToken) {
     loop {
         if !wait_refresh_interval_or_cancel(&cancel_token, BUCKET_METADATA_REFRESH_INTERVAL).await {
             break;
         }
-        let Some(sys) = sys.upgrade() else {
-            break;
-        };
-        refresh_buckets_metadata_once(sys).await;
+        refresh_buckets_metadata_once(sys.clone()).await;
     }
 }
 
@@ -458,7 +455,7 @@ pub(crate) async fn object_store_in(ctx: &crate::runtime::instance::InstanceCont
 
 pub(crate) async fn object_store_if_initialized_in(ctx: &crate::runtime::instance::InstanceContext) -> Option<Arc<ECStore>> {
     let sys = ctx.bucket_metadata_sys().or_else(get_global_bucket_metadata_sys)?;
-    sys.read().await.object_store_if_live()
+    Some(sys.read().await.api.clone())
 }
 
 pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<Arc<BucketMetadata>> {
@@ -478,7 +475,7 @@ pub(crate) async fn get_config_from_disk_with_presence_in(
     bucket: &str,
 ) -> Result<(BucketMetadata, bool)> {
     let sys = bucket_metadata_sys_of(ctx)?;
-    let api = sys.read().await.object_store();
+    let api = sys.read().await.api.clone();
     load_bucket_metadata_parse_with_presence(api, bucket, true).await
 }
 
@@ -741,7 +738,7 @@ pub async fn acquire_scanner_bucket_incarnation_fence(
 ) -> Result<BucketMetadataMutationGuard> {
     super::utils::check_valid_bucket_name(bucket)?;
     let sys = get_bucket_metadata_sys()?;
-    if expected_owner_id.is_nil() || sys.read().await.object_store().id != expected_owner_id || expected_incarnation_id.is_nil() {
+    if expected_owner_id.is_nil() || sys.read().await.api.id != expected_owner_id || expected_incarnation_id.is_nil() {
         return Err(Error::other("scanner bucket incarnation owner does not match"));
     }
     acquire_config_write_guard_with_migration(sys, bucket, Some(expected_incarnation_id), false).await
@@ -754,7 +751,7 @@ async fn acquire_config_write_guard_with_migration(
     migrate: bool,
 ) -> Result<BucketMetadataMutationGuard> {
     let metadata_sys = sys.read().await.clone();
-    let lifecycle_guard = metadata_sys.object_store().acquire_bucket_lifecycle_read_lock(bucket).await?;
+    let lifecycle_guard = metadata_sys.api.acquire_bucket_lifecycle_read_lock(bucket).await?;
 
     // Legacy buckets are migrated while the lifecycle fence prevents a
     // same-name replacement. The second read under the write transaction is
@@ -785,7 +782,7 @@ async fn acquire_config_write_guard_with_migration(
             "bucket config existence transaction validation",
             async {
                 match metadata_sys
-                    .object_store()
+                    .api
                     .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
                     .await
                 {
@@ -805,7 +802,7 @@ async fn acquire_config_write_guard_with_migration(
             Some(&transaction_guard),
             bucket,
             "bucket config incarnation transaction validation",
-            load_bucket_incarnation(metadata_sys.object_store(), bucket),
+            load_bucket_incarnation(metadata_sys.api.clone(), bucket),
         ),
     )
     .await?
@@ -1464,7 +1461,7 @@ pub struct BucketMetadataSys {
     /// Physically missing names are TTL-bounded to limit memory under bogus
     /// name floods while avoiding repeated namespace and erasure reads.
     missing_buckets: moka::future::Cache<String, ()>,
-    api: Weak<ECStore>,
+    api: Arc<ECStore>,
 }
 
 impl BucketMetadataSys {
@@ -1492,17 +1489,12 @@ impl BucketMetadataSys {
                 .max_capacity(MISSING_BUCKET_MAX_ENTRIES)
                 .time_to_live(MISSING_BUCKET_TTL)
                 .build(),
-            api: Arc::downgrade(&api),
+            api,
         }
     }
 
     pub(crate) fn object_store(&self) -> Arc<ECStore> {
-        self.object_store_if_live()
-            .expect("bucket metadata object store should still be live")
-    }
-
-    fn object_store_if_live(&self) -> Option<Arc<ECStore>> {
-        self.api.upgrade()
+        self.api.clone()
     }
 
     fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
@@ -1557,7 +1549,7 @@ impl BucketMetadataSys {
     ) -> Result<bool> {
         await_bucket_namespace_operation(Some(namespace_guard), bucket, operation, async {
             match self
-                .object_store()
+                .api
                 .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
                 .await
             {
@@ -1574,7 +1566,7 @@ impl BucketMetadataSys {
     }
     async fn init_internal(&self, buckets: Vec<String>) -> Result<()> {
         let count = self
-            .object_store()
+            .api
             .pools
             .iter()
             .map(|pool| pool.disk_set.len())
@@ -1606,7 +1598,7 @@ impl BucketMetadataSys {
         let mut futures = Vec::new();
 
         for bucket in buckets.iter() {
-            let api = self.object_store();
+            let api = self.api.clone();
             let bucket = bucket.clone();
             futures.push(async move {
                 sleep(Duration::from_millis(30)).await;
@@ -1652,9 +1644,7 @@ impl BucketMetadataSys {
             let bucket = bucket.clone();
             futures.push(async move {
                 sleep(Duration::from_millis(30)).await;
-                let Some(api) = sys.read().await.object_store_if_live() else {
-                    return Ok(());
-                };
+                let api = sys.read().await.api.clone();
                 let namespace_lock = api.new_ns_lock(&bucket, &bucket).await?;
                 let namespace_guard = namespace_lock
                     .get_read_lock(crate::set_disk::get_lock_acquire_timeout())
@@ -1691,13 +1681,9 @@ impl BucketMetadataSys {
         expected: Option<&Arc<BucketMetadata>>,
         namespace_guard: &rustfs_lock::NamespaceLockGuard,
     ) -> Result<()> {
-        if !await_bucket_namespace_operation(
-            Some(namespace_guard),
-            bucket,
-            "bucket metadata heal existence check",
-            self.object_store().bucket_exists_for_heal(bucket),
-        )
-        .await?
+        if !self
+            .bucket_exists(bucket, namespace_guard, "bucket metadata existence check")
+            .await?
         {
             if matches!(mode, MetadataLoadMode::Refresh) {
                 let _publish_guard = self
@@ -1719,7 +1705,7 @@ impl BucketMetadataSys {
             Some(namespace_guard),
             bucket,
             "bucket metadata heal",
-            self.object_store().heal_bucket(
+            self.api.heal_bucket(
                 bucket,
                 &HealOpts {
                     recreate: true,
@@ -1733,7 +1719,7 @@ impl BucketMetadataSys {
             Some(namespace_guard),
             bucket,
             "bucket metadata load",
-            load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
+            load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true),
         )
         .await?;
         match mode {
@@ -1913,7 +1899,7 @@ impl BucketMetadataSys {
         // (backlog#1052 S7). Reading from the ambient handle instead made the
         // read and the write of a single read-modify-write able to target
         // different instances.
-        let mut bm = Box::pin(Self::load_bucket_metadata_for_update(self.object_store(), bucket, parse)).await?;
+        let mut bm = Box::pin(Self::load_bucket_metadata_for_update(self.api.clone(), bucket, parse)).await?;
         if !bm.bucket_incarnation_sidecar || bm.bucket_incarnation_id != expected_incarnation_id {
             return Err(Error::BucketNotFound(bucket.to_string()));
         }
@@ -1952,7 +1938,7 @@ impl BucketMetadataSys {
     where
         F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
     {
-        let mut bm = Box::pin(Self::load_bucket_metadata_for_update(self.object_store(), bucket, true)).await?;
+        let mut bm = Box::pin(Self::load_bucket_metadata_for_update(self.api.clone(), bucket, true)).await?;
         if !bm.bucket_incarnation_sidecar || bm.bucket_incarnation_id != expected_incarnation_id {
             return Err(Error::BucketNotFound(bucket.to_string()));
         }
@@ -2003,7 +1989,7 @@ impl BucketMetadataSys {
     /// server's metadata never leaks into the ambient (first) instance.
     pub(crate) async fn persist_and_set(&self, bm: BucketMetadata) -> Result<()> {
         let mut bm = bm;
-        bm.save_with_store(self.object_store()).await?;
+        bm.save_with_store(self.api.clone()).await?;
 
         self.set(bm.name.clone(), Arc::new(bm)).await;
 
@@ -2011,8 +1997,8 @@ impl BucketMetadataSys {
     }
 
     async fn persist_new_and_set(&self, mut bm: BucketMetadata) -> Result<()> {
-        bm.save_with_store(self.object_store()).await?;
-        save_bucket_incarnation(self.object_store(), &bm.name, bm.bucket_incarnation_id).await?;
+        bm.save_with_store(self.api.clone()).await?;
+        save_bucket_incarnation(self.api.clone(), &bm.name, bm.bucket_incarnation_id).await?;
         bm.bucket_incarnation_sidecar = true;
         self.set(bm.name.clone(), Arc::new(bm)).await;
         Ok(())
@@ -2029,7 +2015,7 @@ impl BucketMetadataSys {
             return Err(Error::other("errInvalidArgument"));
         }
 
-        load_bucket_metadata(self.object_store(), bucket).await
+        load_bucket_metadata(self.api.clone(), bucket).await
     }
 
     /// Reload persisted metadata under the bucket namespace generation fence.
@@ -2043,7 +2029,7 @@ impl BucketMetadataSys {
             return Err(Error::other("errInvalidArgument"));
         }
 
-        let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?;
+        let namespace_lock = self.api.new_ns_lock(bucket, bucket).await?;
         let namespace_guard = namespace_lock
             .get_read_lock(crate::set_disk::get_lock_acquire_timeout())
             .await?;
@@ -2066,7 +2052,7 @@ impl BucketMetadataSys {
             Some(namespace_guard),
             bucket,
             "peer bucket metadata load",
-            load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
+            load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true),
         )
         .await?;
         if !persisted {
@@ -2111,11 +2097,11 @@ impl BucketMetadataSys {
             #[cfg(test)]
             self.lazy_disk_loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
 
-            let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
+            let lock = self.api.new_ns_lock(bucket, bucket).await?;
             let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
             #[cfg(test)]
             if self.lazy_load_lock_probe.load(std::sync::atomic::Ordering::Relaxed) {
-                let competing = self.object_store().new_ns_lock(bucket, bucket).await?;
+                let competing = self.api.new_ns_lock(bucket, bucket).await?;
                 assert!(
                     competing.get_write_lock(Duration::from_millis(20)).await.is_err(),
                     "lazy metadata IO must start while the bucket namespace read lock is held"
@@ -2125,7 +2111,7 @@ impl BucketMetadataSys {
                 Some(&guard),
                 bucket,
                 "lazy bucket metadata load",
-                Box::pin(load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true)),
+                Box::pin(load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true)),
             )
             .await?;
 
@@ -2137,7 +2123,7 @@ impl BucketMetadataSys {
                     bucket,
                     "lazy bucket metadata existence check",
                     Box::pin(async {
-                        self.object_store()
+                        self.api
                             .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
                             .await
                             .map(|_| ())
@@ -2344,13 +2330,13 @@ impl BucketMetadataSys {
 
     async fn get_bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Uuid> {
         let transaction_lock = self
-            .object_store()
+            .api
             .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
             .await?;
         let _transaction_guard = transaction_lock
             .get_read_lock(crate::set_disk::get_lock_acquire_timeout())
             .await?;
-        let incarnation_id = load_bucket_incarnation(self.object_store(), bucket).await?;
+        let incarnation_id = load_bucket_incarnation(self.api.clone(), bucket).await?;
         if _transaction_guard.is_lock_lost() {
             return Err(Error::other(format!("bucket incarnation metadata transaction lock was lost: {bucket}")));
         }
@@ -2386,7 +2372,7 @@ impl BucketMetadataSys {
 
     async fn migrate_legacy_metadata(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
         let transaction_lock = self
-            .object_store()
+            .api
             .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
             .await?;
         let _transaction_guard = transaction_lock
@@ -2411,7 +2397,7 @@ impl BucketMetadataSys {
             return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
         }
 
-        let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?;
+        let namespace_lock = self.api.new_ns_lock(bucket, bucket).await?;
         let namespace_guard = namespace_lock
             .get_read_lock(crate::set_disk::get_lock_acquire_timeout())
             .await?;
@@ -2421,7 +2407,7 @@ impl BucketMetadataSys {
             bucket,
             "legacy bucket metadata existence check",
             async {
-                self.object_store()
+                self.api
                     .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
                     .await
             },
@@ -2445,7 +2431,7 @@ impl BucketMetadataSys {
             Some(&namespace_guard),
             bucket,
             "legacy bucket metadata confirmation",
-            load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
+            load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true),
         )
         .await?;
         if persisted && !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() {
@@ -2467,20 +2453,20 @@ impl BucketMetadataSys {
             }
             #[cfg(test)]
             if self.legacy_migration_lock_probe.load(std::sync::atomic::Ordering::Relaxed) {
-                let competing = self.object_store().new_ns_lock(bucket, bucket).await?;
+                let competing = self.api.new_ns_lock(bucket, bucket).await?;
                 assert!(
                     competing.get_write_lock(Duration::from_millis(20)).await.is_err(),
                     "bucket delete/recreate must not cross the legacy metadata migration fence"
                 );
             }
-            save_bucket_incarnation(self.object_store(), bucket, metadata.bucket_incarnation_id).await?;
+            save_bucket_incarnation(self.api.clone(), bucket, metadata.bucket_incarnation_id).await?;
             metadata.bucket_incarnation_sidecar = true;
             if !persisted {
                 await_bucket_namespace_operation(
                     Some(&namespace_guard),
                     bucket,
                     "legacy bucket metadata migration",
-                    metadata.save_with_store(self.object_store()),
+                    metadata.save_with_store(self.api.clone()),
                 )
                 .await?;
             }
@@ -2516,7 +2502,7 @@ impl BucketMetadataSys {
             return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
         }
 
-        let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?;
+        let namespace_lock = self.api.new_ns_lock(bucket, bucket).await?;
         let namespace_guard = namespace_lock
             .get_read_lock(crate::set_disk::get_lock_acquire_timeout())
             .await?;
@@ -2525,11 +2511,8 @@ impl BucketMetadataSys {
             bucket,
             "bucket metadata snapshot existence check",
             async {
-                self.object_store()
-                    .get_bucket_info_from_sets_at_read_quorum(
-                        bucket,
-                        &crate::storage_api_contracts::bucket::BucketOptions::default(),
-                    )
+                self.api
+                    .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
                     .await
             },
         )
@@ -2544,7 +2527,7 @@ impl BucketMetadataSys {
             Some(&namespace_guard),
             bucket,
             "bucket metadata authoritative snapshot",
-            load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
+            load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true),
         )
         .await?;
         if persisted {
@@ -3708,7 +3691,7 @@ mod tests {
         let mut stale = BucketMetadata::new("recreated-bucket");
         stale.policy_config_json = b"old-generation".to_vec();
         let namespace_lock = sys
-            .object_store()
+            .api
             .new_ns_lock("recreated-bucket", "recreated-bucket")
             .await
             .expect("namespace lock should be created");

crates/ecstore/src/bucket/migration.rs

@@ -17,7 +17,7 @@
 use crate::bucket::metadata::BUCKET_METADATA_FILE;
 use crate::bucket::replication::ReplicationMigrationBridge;
 use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
-use crate::error::{Error, Result, is_err_strict_not_found, is_err_strict_volume_not_found};
+use crate::error::Error;
 use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
 use crate::storage_api_contracts::{
     bucket::{BucketOperations, BucketOptions},
@@ -33,7 +33,7 @@ use rustfs_utils::path::SLASH_SEPARATOR;
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 use time::OffsetDateTime;
-use tracing::{debug, info};
+use tracing::{debug, info, warn};
 
 /// IAM config prefix under meta bucket (e.g. config/iam/).
 const IAM_CONFIG_PREFIX: &str = "config/iam";
@@ -53,39 +53,6 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
 type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
 type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
 
-#[derive(Clone, Debug, thiserror::Error)]
-enum MigrationMetadataError {
-    #[error("empty legacy metadata: {0}")]
-    Empty(String),
-    #[error("incompatible legacy metadata: {0}")]
-    Incompatible(String),
-}
-
-impl From<MigrationMetadataError> for Error {
-    fn from(error: MigrationMetadataError) -> Self {
-        let message = match &error {
-            MigrationMetadataError::Empty(_) => "empty legacy metadata",
-            MigrationMetadataError::Incompatible(_) => "incompatible legacy metadata",
-        };
-        // Keep the record path in the typed source, not in the quorum grouping key.
-        Self::other_with_context(message, error)
-    }
-}
-
-/// Converts a migration failure at the startup boundary, rendering the safe
-/// record path while leaving storage-layer error grouping stable.
-pub fn migration_startup_error(error: Error) -> std::io::Error {
-    if let Error::Io(io_error) = &error
-        && let Some(metadata_error) = io_error
-            .get_ref()
-            .and_then(|context| context.source())
-            .and_then(|source| source.downcast_ref::<MigrationMetadataError>())
-    {
-        return std::io::Error::other(metadata_error.clone());
-    }
-    std::io::Error::other(error)
-}
-
 /// Callback used to decrypt an at-rest config blob during MinIO -> RustFS migration.
 ///
 /// MinIO encrypts IAM identity/service-account files and the server config at rest
@@ -244,7 +211,7 @@ fn normalize_bucket_meta_blob(path: &str, data: &[u8]) -> std::result::Result<Op
 /// Uses list_bucket (from disk volumes) to get bucket names, since list_objects_v2 on the legacy
 /// meta bucket may not work (legacy format differs from object layer expectations).
 /// Skips buckets that already exist in RustFS (idempotent).
-pub async fn try_migrate_bucket_metadata<S>(store: Arc<S>) -> Result<()>
+pub async fn try_migrate_bucket_metadata<S>(store: Arc<S>)
 where
     S: BucketOperations<Error = crate::error::Error>
         + ObjectIO<
@@ -264,18 +231,25 @@ where
             DeletedObject = DeletedObject,
         >,
 {
-    let buckets_list = store
+    let buckets_list = match store
         .list_bucket(&BucketOptions {
             no_metadata: true,
             ..Default::default()
         })
-        .await?;
+        .await
+    {
+        Ok(b) => b,
+        Err(e) => {
+            warn!("list buckets failed (skip migration): {e}");
+            return;
+        }
+    };
 
     let buckets: Vec<String> = buckets_list.into_iter().map(|b| b.name).collect();
 
     if buckets.is_empty() {
         debug!("No migrating bucket metadata found");
-        return Ok(());
+        return;
     }
 
     debug!("Found {} migrating bucket metadata, migrating...", buckets.len());
@@ -289,40 +263,26 @@ where
 
     for bucket in buckets {
         let meta_path = format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{bucket}{SLASH_SEPARATOR}{BUCKET_METADATA_FILE}");
-        migrate_one_if_missing(store.clone(), &opts, &h, &meta_path, &format!("bucket metadata: {bucket}")).await?;
+        migrate_one_if_missing(store.clone(), &opts, &h, &meta_path, &format!("bucket metadata: {bucket}")).await;
 
         let resync_path = format!(
             "{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{bucket}{SLASH_SEPARATOR}{REPLICATION_META_DIR}{SLASH_SEPARATOR}{RESYNC_META_FILE}"
         );
-        migrate_one_if_missing(store.clone(), &opts, &h, &resync_path, &format!("bucket replication resync: {bucket}")).await?;
-    }
-    Ok(())
-}
-
-async fn migration_target_exists<S: EcstoreObjectOperations>(store: &S, path: &str) -> Result<bool> {
-    match store
-        .get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
-        .await
-    {
-        Ok(_) => Ok(true),
-        Err(err) if is_err_strict_not_found(&err) || is_err_strict_volume_not_found(&err) => Ok(false),
-        Err(err) => Err(err),
+        migrate_one_if_missing(store.clone(), &opts, &h, &resync_path, &format!("bucket replication resync: {bucket}")).await;
     }
 }
 
-async fn migrate_one_if_missing<S>(
-    store: Arc<S>,
-    opts: &ObjectOptions,
-    headers: &HeaderMap,
-    path: &str,
-    label: &str,
-) -> Result<()>
+async fn migrate_one_if_missing<S>(store: Arc<S>, opts: &ObjectOptions, headers: &HeaderMap, path: &str, label: &str)
 where
     S: EcstoreObjectIO + EcstoreObjectOperations,
 {
-    if migration_target_exists(store.as_ref(), path).await? {
+    if store
+        .get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
+        .await
+        .is_ok()
+    {
         debug!("{label} already exists in RustFS, skip");
-        return Ok(());
+        return;
     }
 
     let mut rd = match store
@@ -330,31 +290,43 @@ where
         .await
     {
         Ok(r) => r,
-        // Ordinary RustFS deployments have no legacy bucket, and optional
-        // legacy settings (such as replication resync) may not exist.
-        Err(err) if is_err_strict_not_found(&err) || is_err_strict_volume_not_found(&err) => return Ok(()),
-        Err(err) => return Err(err),
+        Err(e) => {
+            debug!("read migrating {label}: {e}");
+            return;
+        }
     };
 
-    let data = rd.read_all().await?;
-    if data.is_empty() {
-        return Err(MigrationMetadataError::Empty(path.to_owned()).into());
-    }
-    let data = normalize_bucket_meta_blob(path, &data)
-        .map_err(|_| MigrationMetadataError::Incompatible(path.to_owned()))?
-        .unwrap_or(data);
+    let data = match rd.read_all().await {
+        Ok(d) if !d.is_empty() => d,
+        Ok(_) => return,
+        Err(e) => {
+            debug!("read migrating {label} body: {e}");
+            return;
+        }
+    };
+
+    let data = match normalize_bucket_meta_blob(path, &data) {
+        Ok(Some(normalized)) => normalized,
+        Ok(None) => data,
+        Err(e) => {
+            warn!("skip {label} migration due to incompatible format: {e}");
+            return;
+        }
+    };
 
     let mut put_data = PutObjReader::from_vec(data);
-    store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts).await?;
-    info!("Migrated {label}");
-    Ok(())
+    if let Err(e) = store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts).await {
+        warn!("write {label}: {e}");
+    } else {
+        info!("Migrated {label}");
+    }
 }
 
 /// Migrates IAM config from legacy meta bucket `config/iam/` to RustFS meta bucket.
 /// Lists all objects under the IAM prefix in the source, copies each to the target if not present.
 /// Skips objects that already exist in RustFS (idempotent).
-/// An absent legacy bucket is a no-op; migration errors prevent startup readiness.
-pub async fn try_migrate_iam_config<S>(store: Arc<S>, decrypt_fn: Option<LegacyBlobDecryptFn>) -> Result<()>
+/// If list_objects_v2 on the legacy bucket fails (e.g. format differs), migration is skipped.
+pub async fn try_migrate_iam_config<S>(store: Arc<S>, decrypt_fn: Option<LegacyBlobDecryptFn>)
 where
     S: ListOperations<
             Error = crate::error::Error,
@@ -394,36 +366,47 @@ where
     loop {
         let list_result = match store
             .clone()
-            .list_objects_v2(MIGRATING_META_BUCKET, &prefix, continuation.clone(), None, 500, false, None, false)
+            .list_objects_v2(MIGRATING_META_BUCKET, &prefix, continuation, None, 500, false, None, false)
             .await
         {
             Ok(r) => r,
-            Err(err) if is_err_strict_volume_not_found(&err) => return Ok(()),
-            Err(err) => return Err(err),
+            Err(e) => {
+                debug!("list IAM config from legacy bucket failed (skip migration): {e}");
+                return;
+            }
         };
 
         for obj in list_result.objects {
             let path = &obj.name;
-            // Unsupported records must not trigger target lookups, reads, or decryption.
-            if path != IAM_FORMAT_FILE_PATH
-                && !is_identity_path(path)
-                && !is_group_path(path)
-                && !is_policy_doc_path(path)
-                && !is_policy_mapping_path(path)
-            {
+            if path.is_empty() || path.ends_with('/') {
                 continue;
             }
-            if migration_target_exists(store.as_ref(), path).await? {
+            if store
+                .get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
+                .await
+                .is_ok()
+            {
                 debug!("IAM config already exists in RustFS, skip: {path}");
                 continue;
             }
-            let mut rd = store
+            let mut rd = match store
                 .get_object_reader(MIGRATING_META_BUCKET, path, None, h.clone(), &opts)
-                .await?;
-            let data = rd.read_all().await?;
-            if data.is_empty() {
-                return Err(MigrationMetadataError::Empty(path.to_owned()).into());
-            }
+                .await
+            {
+                Ok(r) => r,
+                Err(e) => {
+                    debug!("read migrating IAM config {path}: {e}");
+                    continue;
+                }
+            };
+            let data = match rd.read_all().await {
+                Ok(d) if !d.is_empty() => d,
+                Ok(_) => continue,
+                Err(e) => {
+                    debug!("read migrating IAM config {path} body: {e}");
+                    continue;
+                }
+            };
             // MinIO encrypts IAM identity/service-account files at rest. Decrypt
             // before normalizing; fall back to the raw bytes when no key applies
             // (plaintext blobs, or nothing to decrypt) so existing behavior holds.
@@ -437,17 +420,22 @@ where
                     debug!("skip unsupported IAM config path during migration: {path}");
                     continue;
                 }
-                // Parser errors may contain credential data. Report only the path.
-                Err(_) => return Err(MigrationMetadataError::Incompatible(path.to_owned()).into()),
+                Err(e) => {
+                    warn!("skip IAM config migration due to incompatible format, path: {path}, err: {e}");
+                    continue;
+                }
             };
             let mut put_data = PutObjReader::from_vec(data);
-            store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts).await?;
-            info!("Migrated IAM config: {path}");
-            total_migrated += 1;
+            if let Err(e) = store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts).await {
+                warn!("write IAM config {path}: {e}");
+            } else {
+                info!("Migrated IAM config: {path}");
+                total_migrated += 1;
+            }
         }
 
-        continuation = next_iam_migration_page(list_result.is_truncated, continuation, list_result.next_continuation_token)?;
-        if continuation.is_none() {
+        continuation = list_result.next_continuation_token.or(list_result.continuation_token);
+        if !list_result.is_truncated || continuation.is_none() {
             break;
         }
     }
@@ -455,74 +443,10 @@ where
     if total_migrated > 0 {
         info!("IAM migration complete: {} object(s) migrated", total_migrated);
     }
-    Ok(())
-}
-
-fn next_iam_migration_page(truncated: bool, previous: Option<String>, next: Option<String>) -> Result<Option<String>> {
-    if !truncated {
-        return Ok(None);
-    }
-    let next = next.filter(|token| !token.is_empty());
-    if next.is_none() || next == previous {
-        return Err(Error::other("legacy IAM migration listing did not advance"));
-    }
-    Ok(next)
 }
 
 #[cfg(test)]
 mod tests {
-    #[test]
-    fn migration_errors_group_by_cause_and_retain_typed_record_context() {
-        use super::{Error, MigrationMetadataError};
-
-        for (make_error, message) in [
-            (
-                MigrationMetadataError::Empty as fn(String) -> MigrationMetadataError,
-                "empty legacy metadata",
-            ),
-            (MigrationMetadataError::Incompatible, "incompatible legacy metadata"),
-        ] {
-            let first: Error = make_error("buckets/first/.metadata.bin".into()).into();
-            let second: Error = make_error("buckets/second/.metadata.bin".into()).into();
-            assert_eq!(first, second, "record paths must not fragment error grouping");
-            assert_eq!(first.clone(), second, "cloning must preserve error grouping");
-
-            let io_error = std::io::Error::from(first);
-            let detail = io_error
-                .get_ref()
-                .and_then(|context| context.source())
-                .expect("record context must remain in the error source");
-            assert!(detail.downcast_ref::<MigrationMetadataError>().is_some());
-            assert!(detail.to_string().contains("buckets/first/.metadata.bin"));
-
-            let startup_error = super::migration_startup_error(make_error("buckets/startup/.metadata.bin".into()).into());
-            assert!(
-                startup_error
-                    .get_ref()
-                    .is_some_and(|source| source.is::<MigrationMetadataError>())
-            );
-            assert_eq!(startup_error.to_string(), format!("{message}: buckets/startup/.metadata.bin"));
-        }
-        assert_ne!(
-            Error::from(MigrationMetadataError::Empty("record".into())),
-            Error::from(MigrationMetadataError::Incompatible("record".into())),
-            "different migration failures must remain distinguishable"
-        );
-    }
-
-    #[test]
-    fn truncated_iam_listing_cannot_report_completed_migration() {
-        use super::next_iam_migration_page;
-        assert_eq!(next_iam_migration_page(false, Some("old".into()), None).expect("final page"), None);
-        assert_eq!(
-            next_iam_migration_page(true, Some("old".into()), Some("next".into())).expect("advancing page"),
-            Some("next".into())
-        );
-        for next in [None, Some(String::new()), Some("old".into())] {
-            assert!(next_iam_migration_page(true, Some("old".into()), next).is_err());
-        }
-    }
-
     use super::{normalize_bucket_meta_blob, normalize_iam_config_blob};
     use crate::bucket::replication::{
         BucketReplicationResyncStatus, ReplicationMigrationBridge, ResyncStatusType, TargetReplicationResyncStatus,
@@ -735,13 +659,6 @@ mod tests {
             .collect();
         crate::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), existing).await;
 
-        super::try_migrate_bucket_metadata(ecstore.clone())
-            .await
-            .expect("fresh stores do not require a legacy metadata bucket");
-        super::try_migrate_iam_config(ecstore.clone(), None)
-            .await
-            .expect("fresh stores do not require a legacy IAM bucket");
-
         let meta_path = format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}interop{SLASH_SEPARATOR}{BUCKET_METADATA_FILE}");
         let put_opts = ObjectOptions::default();
 
@@ -763,31 +680,8 @@ mod tests {
             .await
             .expect("seed .minio.sys bucket metadata");
 
-        // A partial import must report failure, even if the main bucket
-        // metadata copied successfully before an incompatible resync record.
-        let resync_path = format!("{BUCKET_META_PREFIX}/interop/.replication/resync.bin");
-        ecstore
-            .put_object(
-                MIGRATING_META_BUCKET,
-                &resync_path,
-                &mut PutObjReader::from_vec(b"invalid resync metadata".to_vec()),
-                &put_opts,
-            )
-            .await
-            .expect("seed malformed legacy resync metadata");
-        assert!(
-            super::try_migrate_bucket_metadata(ecstore.clone()).await.is_err(),
-            "incompatible native metadata must not be reported as a completed migration"
-        );
-        ecstore
-            .delete_object(MIGRATING_META_BUCKET, &resync_path, ObjectOptions::default())
-            .await
-            .expect("remove invalid optional legacy resync record");
-
-        // Retry the real startup migration after repairing the source.
-        super::try_migrate_bucket_metadata(ecstore.clone())
-            .await
-            .expect("native bucket metadata migration completes");
+        // --- Run the real startup migration. ---
+        super::try_migrate_bucket_metadata(ecstore.clone()).await;
 
         // --- The migrated `.rustfs.sys` blob must carry every MinIO config, ---
         // byte-identical to the source (typed XML/JSON parsing of these fields is

crates/ecstore/src/bucket/replication/mod.rs

@@ -92,6 +92,3 @@ pub use replication_target_boundary::SsecPassthroughCapability;
 pub use replication_target_boundary::VersionIdentityCapability;
 pub use replication_target_boundary::{ObjectLockIntegrity, object_lock_put_integrity};
 pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
-pub use runtime_boundary::{
-    ScannerDirtyUsageMutationObserver, ScannerDirtyUsageMutationSource, set_scanner_dirty_usage_mutation_observer,
-};

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -3843,13 +3843,6 @@ async fn persist_replication_state_if_current<S: ReplicationStorage>(
     match storage.put_object_metadata(&roi.bucket, &roi.name, &write_opts).await {
         Ok(updated) => {
             *object_info = updated;
-            if mode == ReplicationStatusWritebackMode::Update {
-                runtime_sources::notify_scanner_dirty_usage_mutation(
-                    &roi.bucket,
-                    &roi.name,
-                    runtime_sources::ScannerDirtyUsageMutationSource::Replication,
-                );
-            }
             Ok(ReplicationStatePersistOutcome::Updated)
         }
         Err(Error::PreconditionFailed) => Ok(ReplicationStatePersistOutcome::Superseded),

crates/ecstore/src/bucket/replication/runtime_boundary.rs

@@ -19,9 +19,6 @@ use super::replication_pool::DynReplicationPool;
 use super::replication_state::ReplicationStats;
 use super::replication_storage_boundary::ReplicationObjectStore;
 use crate::runtime::sources;
-pub use crate::runtime::sources::{
-    ScannerDirtyUsageMutationObserver, ScannerDirtyUsageMutationSource, set_scanner_dirty_usage_mutation_observer,
-};
 
 pub(crate) fn object_store_handle() -> Option<Arc<ReplicationObjectStore>> {
     sources::object_store_handle()
@@ -46,7 +43,3 @@ pub(crate) fn replication_runtime_initialized() -> bool {
 pub(crate) fn bucket_monitor() -> Option<Arc<ReplicationBucketMonitor>> {
     sources::bucket_monitor()
 }
-
-pub(crate) fn notify_scanner_dirty_usage_mutation(bucket: &str, object: &str, source: ScannerDirtyUsageMutationSource) {
-    sources::notify_scanner_dirty_usage_mutation(bucket, object, source);
-}

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -51,10 +51,10 @@ use rustfs_protos::proto_gen::node_service::{
     LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ReplacementRecoveryStatusRequest,
     ScannerActivityRequest, ScannerActivityResponse, ScannerDirtyUsageSnapshotRequest, ScannerDirtyUsageSnapshotResponse,
     ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse,
-    ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse, ScannerScopedDirtyUsageEntry, ServerInfoRequest,
-    SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest,
-    TierDailyStatsRequest, TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse,
-    TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
+    ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageEntry, ServerInfoRequest, SignalServiceRequest,
+    SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, TierDailyStatsRequest,
+    TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse, TierMutationFailureClass,
+    TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
     tier_mutation_control_service_client::TierMutationControlServiceClient,
 };
 pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
@@ -289,30 +289,8 @@ fn scanner_scoped_dirty_usage_ack_payload(
     Ok(payload)
 }
 
-fn scanner_scoped_dirty_usage_ack_response_matches(
-    request: &ScannerScopedDirtyUsageAckRequest,
-    response: &ScannerScopedDirtyUsageAckResponse,
-) -> bool {
-    let cleared_within_request = u64::try_from(request.entries.len())
-        .is_ok_and(|entry_count| response.cleared <= entry_count && (!request.probe_only || response.cleared == 0));
-    response.protocol_version == rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
-        && response.owner_id == request.owner_id
-        && response.instance_id == request.instance_id
-        && response.max_entries == rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_ENTRIES
-        && response.max_request_bytes == rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
-        && cleared_within_request
-}
-
-fn scanner_scoped_dirty_usage_ack_reconciled(
-    activity: &ScannerPeerActivity,
-    expected_instance_id: &str,
-    expected_generation: u64,
-) -> bool {
-    activity.instance_id == expected_instance_id
-        && activity.dirty_usage_pending == Some(false)
-        && activity
-            .dirty_usage_generation
-            .is_some_and(|generation| generation >= expected_generation)
+fn scanner_scoped_dirty_usage_ack_reconciled(activity: &ScannerPeerActivity, expected_instance_id: &str) -> bool {
+    activity.instance_id == expected_instance_id && activity.dirty_usage_pending == Some(false)
 }
 
 fn scanner_instance_id_is_valid(instance_id: &str) -> bool {
@@ -2277,7 +2255,13 @@ impl PeerRestClient {
                     let body = canonical_scoped_dirty_usage_response(&canonical, &response)
                         .map_err(|_| Error::other("scoped dirty usage capability response is too large"))?;
                     verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
-                    if !scanner_scoped_dirty_usage_ack_response_matches(&payload, &response) {
+                    if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
+                        || response.owner_id != payload.owner_id
+                        || response.instance_id != payload.instance_id
+                        || response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
+                        || response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
+                        || response.cleared != 0
+                    {
                         return Err(Error::other("scoped dirty usage capability response does not match request"));
                     }
                     if !response.supported {
@@ -2298,7 +2282,6 @@ impl PeerRestClient {
         entries: Vec<ScannerScopedDirtyUsageAckEntry>,
     ) -> Result<ScannerPeerActivity> {
         use rustfs_protos::scoped_dirty_usage::*;
-        let expected_generation = entries.iter().map(|entry| entry.generation).max().unwrap_or(0);
         let payloads = scanner_scoped_dirty_usage_ack_payloads(owner_id, instance_id.clone(), false, entries)?;
         let ack_attempt = async {
             let mut client = super::client::scanner_control_time_out_client(
@@ -2314,7 +2297,13 @@ impl PeerRestClient {
                 let body = canonical_scoped_dirty_usage_response(&canonical, &response)
                     .map_err(|_| Error::other("scoped dirty usage acknowledgement response is too large"))?;
                 verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
-                if !scanner_scoped_dirty_usage_ack_response_matches(&payload, &response) || !response.supported {
+                if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
+                    || response.owner_id != payload.owner_id
+                    || response.instance_id != payload.instance_id
+                    || response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
+                    || response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
+                    || !response.supported
+                {
                     return Err(Error::other("scoped dirty usage acknowledgement response does not match request"));
                 }
             }
@@ -2346,9 +2335,7 @@ impl PeerRestClient {
                         .await;
                 }
                 match self.scanner_scoped_dirty_usage_activity_confirmation().await {
-                    Ok(activity) if scanner_scoped_dirty_usage_ack_reconciled(&activity, &instance_id, expected_generation) => {
-                        Ok(activity)
-                    }
+                    Ok(activity) if scanner_scoped_dirty_usage_ack_reconciled(&activity, &instance_id) => Ok(activity),
                     _ => Err(err),
                 }
             }
@@ -3131,104 +3118,32 @@ mod tests {
         }
     }
 
-    #[test]
-    fn scanner_scoped_dirty_usage_ack_response_bounds_cleared_entries_to_request() {
-        use rustfs_protos::scoped_dirty_usage::{
-            SCOPED_DIRTY_USAGE_MAX_ENTRIES, SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES, SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
-        };
-
-        let mut request = scanner_scoped_dirty_usage_ack_payload(
-            "33333333-3333-3333-3333-333333333333",
-            "0123456789abcdef0123456789abcdef",
-            false,
-            vec![
-                ScannerScopedDirtyUsageEntry {
-                    bucket: "archive".to_string(),
-                    bucket_incarnation: Uuid::from_u128(0x11111111111111111111111111111111).as_bytes().to_vec().into(),
-                    generation: 3,
-                },
-                ScannerScopedDirtyUsageEntry {
-                    bucket: "photos".to_string(),
-                    bucket_incarnation: Uuid::from_u128(0x22222222222222222222222222222222).as_bytes().to_vec().into(),
-                    generation: 7,
-                },
-            ],
-        )
-        .expect("two ordered entries should form a valid scoped ACK request");
-        let mut response = ScannerScopedDirtyUsageAckResponse {
-            protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
-            owner_id: request.owner_id.clone(),
-            instance_id: request.instance_id.clone(),
-            supported: true,
-            max_entries: SCOPED_DIRTY_USAGE_MAX_ENTRIES,
-            max_request_bytes: SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES,
-            cleared: 1,
-            response_proof: Bytes::new(),
-        };
-
-        assert!(scanner_scoped_dirty_usage_ack_response_matches(&request, &response));
-        response.cleared = 2;
-        assert!(scanner_scoped_dirty_usage_ack_response_matches(&request, &response));
-        response.cleared = 3;
-        assert!(
-            !scanner_scoped_dirty_usage_ack_response_matches(&request, &response),
-            "a peer cannot clear more entries than the signed request contains"
-        );
-
-        request.probe_only = true;
-        response.cleared = 1;
-        assert!(
-            !scanner_scoped_dirty_usage_ack_response_matches(&request, &response),
-            "a capability probe cannot report a mutation"
-        );
-        response.cleared = 0;
-        assert!(scanner_scoped_dirty_usage_ack_response_matches(&request, &response));
-    }
-
     #[test]
     fn scanner_scoped_dirty_usage_ack_reconciliation_requires_same_clean_instance() {
-        let activity = |instance_id: &str, generation, pending| ScannerPeerActivity {
+        let activity = |instance_id: &str, pending| ScannerPeerActivity {
             instance_id: instance_id.to_string(),
             namespace_generation: 1,
             maintenance_generation: 1,
             protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
             topology_digest: Some([1; 32]),
             data_movement_active: Some(false),
-            dirty_usage_generation: generation,
+            dirty_usage_generation: Some(9),
             dirty_usage_pending: pending,
             movement_generation: Some(1),
             publication_blocked: Some(false),
         };
 
         assert!(scanner_scoped_dirty_usage_ack_reconciled(
-            &activity("0123456789abcdef0123456789abcdef", Some(9), Some(false)),
-            "0123456789abcdef0123456789abcdef",
-            9
-        ));
-        assert!(scanner_scoped_dirty_usage_ack_reconciled(
-            &activity("0123456789abcdef0123456789abcdef", Some(10), Some(false)),
-            "0123456789abcdef0123456789abcdef",
-            9
+            &activity("0123456789abcdef0123456789abcdef", Some(false)),
+            "0123456789abcdef0123456789abcdef"
         ));
         assert!(!scanner_scoped_dirty_usage_ack_reconciled(
-            &activity("0123456789abcdef0123456789abcdef", Some(8), Some(false)),
-            "0123456789abcdef0123456789abcdef",
-            9
+            &activity("0123456789abcdef0123456789abcdef", Some(true)),
+            "0123456789abcdef0123456789abcdef"
         ));
         assert!(!scanner_scoped_dirty_usage_ack_reconciled(
-            &activity("0123456789abcdef0123456789abcdef", None, Some(false)),
-            "0123456789abcdef0123456789abcdef",
-            9
-        ));
-        assert!(!scanner_scoped_dirty_usage_ack_reconciled(
-            &activity("0123456789abcdef0123456789abcdef", Some(9), Some(true)),
-            "0123456789abcdef0123456789abcdef",
-            9
-        ));
-        assert!(!scanner_scoped_dirty_usage_ack_reconciled(
-            &activity("fedcba9876543210fedcba9876543210", Some(9), Some(false)),
-            "0123456789abcdef0123456789abcdef",
-            9
+            &activity("fedcba9876543210fedcba9876543210", Some(false)),
+            "0123456789abcdef0123456789abcdef"
         ));
     }
 

crates/ecstore/src/core/pools_test.rs

@@ -5256,35 +5256,16 @@ mod decommission_lock_order_tests {
 
     #[test]
     #[serial_test::serial]
-    fn data_movement_existing_replica_reconciles_capacity_and_cleans_source() {
-        data_movement_existing_replica_reconciles_capacity_case(false);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn data_movement_existing_replica_outside_reservation_uses_reserved_target() {
-        data_movement_existing_replica_reconciles_capacity_case(true);
-    }
-
-    fn data_movement_existing_replica_reconciles_capacity_case(existing_outside_reservation: bool) {
-        run_large_stack_current_thread_async_test("reserved-replica-reconcile", async move || {
+    fn scanner_backlog_native_replica_reconciles_capacity_and_cleans_source() {
+        run_large_stack_current_thread_async_test("scanner-backlog-reconcile", async || {
             let (_temp_dirs, store, other_store) =
                 test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await;
-            let object = "buckets/reserved-replica-routing.json";
+            let object = "buckets/.scanner-pause-backlog.json";
             let body = br#"{"schemaVersion":1,"generation":2}"#.to_vec();
             let old_body = br#"{"schemaVersion":1,"generation":1}"#.to_vec();
             let source_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(20);
-            let target_time = source_time;
-            let target_pool_index = if existing_outside_reservation { 1 } else { 2 };
-            let mut replicas = vec![(0, body.clone(), source_time), (target_pool_index, old_body, target_time)];
-            if existing_outside_reservation {
-                replicas.push((
-                    2,
-                    br#"{"schemaVersion":1,"generation":3}"#.to_vec(),
-                    source_time + time::Duration::seconds(10),
-                ));
-            }
-            for (pool_index, payload, mod_time) in replicas.iter().cloned() {
+            let target_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(10);
+            for (pool_index, payload, mod_time) in [(0, body.clone(), source_time), (2, old_body, target_time)] {
                 store.pools[pool_index]
                     .put_object(
                         RUSTFS_META_BUCKET,
@@ -5297,19 +5278,13 @@ mod decommission_lock_order_tests {
                         },
                     )
                     .await
-                    .expect("seed existing replicas with independent write times");
+                    .expect("seed native scanner replicas with independent write times");
             }
             let layout = DecommissionErasureLayout { data: 1, parity: 0 };
             let target_total = body.len() * 8;
             let capacities = vec![
                 DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len() * 2, body.len() * 2),
-                DecommissionPoolCapacityInfo::for_test(
-                    1,
-                    layout,
-                    if existing_outside_reservation { target_total } else { 0 },
-                    target_total,
-                    if existing_outside_reservation { 0 } else { target_total },
-                ),
+                DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
                 DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
             ];
             set_decommission_capacity_info_overrides_for_test(store.id, vec![capacities.clone()]);
@@ -5318,17 +5293,6 @@ mod decommission_lock_order_tests {
                 .await
                 .expect("activate the source reservation");
             let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
-            let reserved_snapshot = store.pool_meta.read().await.clone();
-            let reservation = reserved_snapshot.pools[0]
-                .decommission
-                .as_ref()
-                .and_then(|info| info.capacity_reservation.as_ref())
-                .expect("active source reservation");
-            assert_eq!(
-                reservation.targets.iter().map(|target| target.pool_index).collect::<Vec<_>>(),
-                vec![target_pool_index],
-                "the fixture must reserve exactly one target"
-            );
             let source_reader = store.pools[0]
                 .get_object_reader(
                     RUSTFS_META_BUCKET,
@@ -5350,91 +5314,12 @@ mod decommission_lock_order_tests {
                 RUSTFS_META_BUCKET.to_string(),
                 source_reader,
                 None,
-                "reserved_replica_conflict",
+                "scanner_backlog_conflict",
                 Some(owner),
             )
             .await
-            .expect_err("a different older existing record must retain its source and capacity intent");
+            .expect_err("a different older native ledger must retain its source and capacity intent");
             assert!(conflict.to_string().contains("Precondition failed"), "unexpected conflict: {conflict}");
-            let reserved_snapshot = store.pool_meta.read().await.clone();
-            let mut selection_opts = ObjectOptions {
-                data_movement: true,
-                src_pool_idx: 0,
-                ..Default::default()
-            };
-            assert_eq!(
-                store
-                    .select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &selection_opts, true)
-                    .await
-                    .expect("selection without a capacity owner retains existing-replica routing"),
-                2
-            );
-            owner.apply_to(&mut selection_opts);
-            for stale_owner in [
-                DecommissionCapacityOwner {
-                    owner_nonce: uuid::Uuid::new_v4(),
-                    ..owner
-                },
-                DecommissionCapacityOwner {
-                    generation: owner.generation + 1,
-                    ..owner
-                },
-            ] {
-                let mut stale_opts = selection_opts.clone();
-                stale_owner.apply_to(&mut stale_opts);
-                assert!(
-                    matches!(
-                        store
-                            .select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &stale_opts, true)
-                            .await,
-                        Err(crate::error::Error::DecommissionCapacityBlocked { .. })
-                    ),
-                    "a stale owner must not fall back to another target"
-                );
-            }
-            {
-                let mut meta = store.pool_meta.write().await;
-                meta.pools[0]
-                    .decommission
-                    .as_mut()
-                    .unwrap()
-                    .capacity_reservation
-                    .as_mut()
-                    .unwrap()
-                    .expires_at = time::OffsetDateTime::now_utc() - time::Duration::seconds(1);
-            }
-            assert!(
-                matches!(
-                    store
-                        .select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &selection_opts, true)
-                        .await,
-                    Err(crate::error::Error::DecommissionCapacityBlocked { .. })
-                ),
-                "an expired owner must not fall back to another target"
-            );
-            *store.pool_meta.write().await = reserved_snapshot.clone();
-            if !existing_outside_reservation {
-                {
-                    let mut meta = store.pool_meta.write().await;
-                    let target = &mut meta.pools[0]
-                        .decommission
-                        .as_mut()
-                        .unwrap()
-                        .capacity_reservation
-                        .as_mut()
-                        .unwrap()
-                        .targets[0];
-                    target.consumed_physical_bytes = target.reserved_physical_bytes;
-                }
-                assert_eq!(
-                    store
-                        .select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &selection_opts, true)
-                        .await
-                        .expect("an existing reserved replica can still be selected after capacity was consumed"),
-                    target_pool_index
-                );
-                *store.pool_meta.write().await = reserved_snapshot;
-            }
             let mut persisted = crate::core::pools::PoolMeta::default();
             persisted
                 .load_no_lock_from_replicas(store.pools.clone())
@@ -5451,24 +5336,11 @@ mod decommission_lock_order_tests {
                     .pending_target_physical_bytes,
                 body.len()
             );
-            for (pool_index, payload, mod_time) in &replicas {
-                let mut reader = store.pools[*pool_index]
-                    .get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
-                    .await
-                    .expect("a refused existing record replacement must preserve every replica");
-                assert_eq!(reader.object_info.mod_time, Some(*mod_time));
-                let mut actual = Vec::new();
-                reader
-                    .read_to_end(&mut actual)
-                    .await
-                    .expect("read the unchanged existing record");
-                assert_eq!(&actual, payload);
-            }
-            let previous = store.pools[target_pool_index]
+            let previous = store.pools[2]
                 .get_object_info(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
                 .await
-                .expect("read the existing writer's CAS revision");
-            let replacement = store.pools[target_pool_index]
+                .expect("read the native writer's CAS revision");
+            let replacement = store.pools[2]
                 .put_object(
                     RUSTFS_META_BUCKET,
                     object,
@@ -5484,7 +5356,7 @@ mod decommission_lock_order_tests {
                     },
                 )
                 .await
-                .expect("existing CAS converges the payload without a migration marker");
+                .expect("native scanner CAS converges the payload without a migration marker");
             assert!(!data_movement::is_owned_data_movement_target(&replacement));
             *other_store.pool_meta.write().await = persisted;
             set_decommission_capacity_info_overrides_for_test(other_store.id, vec![capacities]);
@@ -5502,7 +5374,7 @@ mod decommission_lock_order_tests {
             )
             .await
             .expect("replica conflict recovery must be bounded")
-            .expect("identical existing replica should finish migration on the reloaded node");
+            .expect("identical native replica should finish migration on the reloaded node");
             let mut reconciled = crate::core::pools::PoolMeta::default();
             reconciled
                 .load_no_lock_from_replicas(other_store.pools.clone())
@@ -5532,14 +5404,14 @@ mod decommission_lock_order_tests {
                 .await
                 .expect_err("the source should be cleaned only after equivalent-target capacity reconciliation");
             assert!(crate::error::is_err_object_not_found(&missing));
-            let mut target_reader = other_store.pools[target_pool_index]
+            let mut target_reader = other_store.pools[2]
                 .get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
                 .await
                 .expect("the surviving replica should remain readable");
             assert_eq!(
                 target_reader.object_info.mod_time,
                 Some(target_time),
-                "recovery must not overwrite the existing target"
+                "recovery must not overwrite the native target"
             );
             let mut actual = Vec::new();
             target_reader
@@ -5547,20 +5419,6 @@ mod decommission_lock_order_tests {
                 .await
                 .expect("read surviving ledger bytes");
             assert_eq!(actual, body);
-            if existing_outside_reservation {
-                let (_, outside_body, outside_time) = replicas.last().expect("unreserved existing replica");
-                let mut outside = other_store.pools[2]
-                    .get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
-                    .await
-                    .expect("migration must leave the unreserved existing replica intact");
-                assert_eq!(outside.object_info.mod_time, Some(*outside_time));
-                let mut actual = Vec::new();
-                outside
-                    .read_to_end(&mut actual)
-                    .await
-                    .expect("read the untouched unreserved replica");
-                assert_eq!(&actual, outside_body);
-            }
         });
     }
 

crates/ecstore/src/core/sets.rs

@@ -55,7 +55,7 @@ use rustfs_madmin::heal_commands::HealResultItem;
 use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
 use std::{
     collections::{HashMap, HashSet},
-    sync::{Arc, Weak},
+    sync::Arc,
 };
 use tokio::sync::RwLock;
 use tokio::sync::broadcast::{Receiver, Sender};
@@ -219,10 +219,7 @@ impl Sets {
 
         let mut disk_set = Vec::with_capacity(set_count);
 
-        let pool_lockers = runtime_sources::lock_registry()
-            .as_ref()
-            .map(|registry| registry.clients_for_endpoints(endpoints.endpoints.as_ref()))
-            .unwrap_or_default();
+        let lock_registry = runtime_sources::lock_registry();
 
         for i in 0..set_count {
             let mut set_drive = Vec::with_capacity(set_drive_count);
@@ -273,6 +270,10 @@ impl Sets {
                 }
             }
 
+            let lockers = lock_registry
+                .as_ref()
+                .map(|registry| registry.clients_for_endpoints(&set_endpoints))
+                .unwrap_or_default();
             let set_disks = SetDisks::new_with_instance_ctx(
                 runtime_sources::local_node_name().await,
                 Arc::new(RwLock::new(set_drive)),
@@ -282,7 +283,7 @@ impl Sets {
                 pool_idx,
                 set_endpoints,
                 fm.clone(),
-                pool_lockers.clone(),
+                lockers,
                 instance_ctx.clone(),
             )
             .await;
@@ -307,9 +308,10 @@ impl Sets {
             ctx: instance_ctx,
         });
 
+        let asets = sets.clone();
+
         let rx1 = rx.resubscribe();
-        let weak_sets = Arc::downgrade(&sets);
-        tokio::spawn(async move { Self::monitor_and_connect_endpoints_task(weak_sets, rx1).await });
+        tokio::spawn(async move { asets.monitor_and_connect_endpoints(rx1).await });
 
         Ok(sets)
     }
@@ -324,26 +326,12 @@ impl Sets {
         &self.ctx
     }
 
-    async fn monitor_and_connect_endpoints_task(sets: Weak<Sets>, mut rx: Receiver<()>) {
-        let startup_delay = tokio::time::sleep(Duration::from_secs(5));
-        tokio::pin!(startup_delay);
-
-        tokio::select! {
-            _ = &mut startup_delay => {}
-            _ = rx.recv() => {
-                warn!("monitor_and_connect_endpoints ctx cancelled");
-                return;
-            }
-        }
+    pub async fn monitor_and_connect_endpoints(&self, mut rx: Receiver<()>) {
+        tokio::time::sleep(Duration::from_secs(5)).await;
 
         info!("start monitor_and_connect_endpoints");
 
-        let Some(current) = sets.upgrade() else {
-            warn!("monitor_and_connect_endpoints exit");
-            return;
-        };
-        current.connect_disks().await;
-        drop(current);
+        self.connect_disks().await;
 
         // TODO(backlog): make monitor_and_connect interval configurable instead of hardcoded 15s
         let mut interval = tokio::time::interval(Duration::from_secs(15));
@@ -351,10 +339,7 @@ impl Sets {
             tokio::select! {
                _= interval.tick()=>{
                 // debug!("tick...");
-                let Some(current) = sets.upgrade() else {
-                    break;
-                };
-                current.connect_disks().await;
+                self.connect_disks().await;
 
                 interval.reset();
                },

crates/ecstore/src/data_movement/mod.rs

@@ -15,7 +15,6 @@
 // #730: data-movement migration keeps staged cleanup helpers until copy paths converge.
 
 pub(crate) mod backpressure;
-pub(crate) mod scanner_backlog;
 
 use crate::core::pools::{DecommissionCapacityOwner, decommission_capacity_mutation_id};
 use crate::error::{
@@ -985,6 +984,24 @@ fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target:
             .is_some_and(|(source_time, target_time)| target_time > source_time)
 }
 
+fn is_equivalent_scanner_backlog_replica(source: &ObjectInfo, target: &ObjectInfo, compare_part_checksums: bool) -> bool {
+    // Scanner publishes this exact payload to surviving sets with CAS. Each
+    // set assigns its own write time; that timestamp is not a ledger generation.
+    // Accept only an identical, known unversioned identity, never a different
+    // record based on timestamp ordering or a similarly named user object.
+    source.bucket == crate::disk::RUSTFS_META_BUCKET
+        && target.bucket == source.bucket
+        && source.name == "buckets/.scanner-pause-backlog.json"
+        && target.name == source.name
+        && is_unversioned_data_movement_object(source)
+        && is_unversioned_data_movement_object(target)
+        && !source.delete_marker
+        && source.mod_time.is_some()
+        && target.mod_time.is_some()
+        && source.etag.as_ref().is_some_and(|etag| !etag.is_empty())
+        && is_equivalent_data_movement_object_identity(source, target, false, compare_part_checksums)
+}
+
 fn is_data_movement_upload_takeover_target(source: &ObjectInfo, target: &ObjectInfo, compare_part_checksums: bool) -> bool {
     let identity = data_movement_upload_identity(source);
     source.mod_time.is_some()
@@ -1200,7 +1217,7 @@ struct SourceCleanupDeleteBarrierState {
     dead_code,
     reason = "installed by set_disk object tests behind `--features test-util` (backlog#1823)"
 )]
-pub struct SourceCleanupDeleteBarrier {
+pub(crate) struct SourceCleanupDeleteBarrier {
     state: Arc<SourceCleanupDeleteBarrierState>,
 }
 
@@ -1214,7 +1231,7 @@ static SOURCE_CLEANUP_DELETE_BARRIERS: std::sync::OnceLock<std::sync::Mutex<Vec<
     reason = "installed by set_disk object tests behind `--features test-util` (backlog#1823)"
 )]
 impl SourceCleanupDeleteBarrier {
-    pub fn install(bucket: &str, object: &str) -> Self {
+    pub(crate) fn install(bucket: &str, object: &str) -> Self {
         let state = Arc::new(SourceCleanupDeleteBarrierState {
             bucket: bucket.to_string(),
             object: object.to_string(),
@@ -1237,7 +1254,7 @@ impl SourceCleanupDeleteBarrier {
         Self { state }
     }
 
-    pub async fn wait_until_paused(&self) {
+    pub(crate) async fn wait_until_paused(&self) {
         tokio::time::timeout(StdDuration::from_secs(30), self.state.arrived.notified())
             .await
             .expect("source cleanup should reach the pre-delete barrier");
@@ -1253,7 +1270,7 @@ impl SourceCleanupDeleteBarrier {
         self.state.is_paused.load(Ordering::Acquire)
     }
 
-    pub fn release(&self) {
+    pub(crate) fn release(&self) {
         self.state.release.notify_one();
     }
 }
@@ -1432,8 +1449,7 @@ fn resolve_data_movement_overwrite_resume_result_for(
     target_pool_idx: usize,
     compare_part_checksums: bool,
 ) -> Result<bool> {
-    if scanner_backlog::is_scanner_pause_backlog(&source.bucket, &source.name)
-        || !should_check_data_movement_overwrite_resume(err)
+    if !should_check_data_movement_overwrite_resume(err)
         || !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx)
     {
         return Ok(false);
@@ -1455,7 +1471,9 @@ fn resolve_data_movement_overwrite_resume_result_for(
         return Ok(true);
     }
 
-    Ok(matches!(err, Error::PreconditionFailed) && is_superseding_unversioned_data_movement_object(source, &target))
+    Ok(matches!(err, Error::PreconditionFailed)
+        && (is_equivalent_scanner_backlog_replica(source, &target, compare_part_checksums)
+            || is_superseding_unversioned_data_movement_object(source, &target)))
 }
 
 #[derive(Clone, Copy)]
@@ -1503,27 +1521,9 @@ fn data_movement_part_stage_error(
     bucket: &str,
     object: &str,
     part_number: usize,
-    err: Error,
+    err: impl std::fmt::Display,
 ) -> Error {
-    let rendered = format!("{op_label}: {stage} failed for {bucket}/{object} part {part_number}: {err}");
-    if matches!(&err, Error::DecommissionCapacityBlocked { .. }) {
-        return data_movement_context_error(rendered, err);
-    }
-    // A missing target part is not evidence that the source can be deleted.
-    // Keep other part errors opaque to the source-cleanup classifiers.
-    Error::other(rendered)
-}
-
-#[cfg(test)]
-pub(crate) fn data_movement_part_stage_error_for_test(
-    op_label: &str,
-    stage: &str,
-    bucket: &str,
-    object: &str,
-    part_number: usize,
-    err: Error,
-) -> Error {
-    data_movement_part_stage_error(op_label, stage, bucket, object, part_number, err)
+    Error::other(format!("{op_label}: {stage} failed for {bucket}/{object} part {part_number}: {err}"))
 }
 
 fn is_data_movement_part_read_error(err: &Error) -> bool {
@@ -1628,9 +1628,6 @@ async fn migrate_object_inner(
     capacity_owner: Option<DecommissionCapacityOwner>,
     mutation_fence: Option<DecommissionFixedReadAnchor>,
 ) -> Result<()> {
-    if scanner_backlog::is_scanner_pause_backlog(&bucket, &rd.object_info.name) {
-        return Err(Error::other("scanner pause backlog requires native retirement handoff"));
-    }
     let mut mutation_fence = mutation_fence;
     let object_info = rd.object_info.clone();
     let capacity_owner = capacity_owner.map(|owner| {
@@ -2431,15 +2428,8 @@ mod tests {
         let err =
             data_movement_part_stage_error("rebalance_object", "put_object_part", "bucket-a", "object-a", 7, Error::SlowDown);
         let message = err.to_string();
-        assert_eq!(
-            message,
-            Error::other(format!(
-                "rebalance_object: put_object_part failed for bucket-a/object-a part 7: {}",
-                Error::SlowDown
-            ))
-            .to_string()
-        );
-        assert!(data_movement_stage_source(&err).is_none());
+        assert!(message.contains("rebalance_object: put_object_part failed for bucket-a/object-a part 7"));
+        assert!(message.contains(Error::SlowDown.to_string().as_str()));
     }
 
     #[test]
@@ -3339,25 +3329,16 @@ mod tests {
     }
 
     #[test]
-    fn test_scanner_backlog_resume_requires_native_cohort_proof_even_for_identical_payload() {
+    fn test_scanner_backlog_resume_accepts_identical_native_replica_with_older_write_time() {
         let (source, target) = scanner_backlog_replica_pair();
         assert!(!is_owned_data_movement_target(&target), "native scanner writes are not migration copies");
         assert!(!is_equivalent_data_movement_object(&source, &target));
         assert!(
-            !scanner_backlog_precondition_resumes(&source, target),
-            "a single identical replica cannot prove native cohort authority"
+            scanner_backlog_precondition_resumes(&source, target),
+            "identical ledger payloads have replica-local write times, not distinct committed generations"
         );
     }
 
-    #[test]
-    fn test_scanner_backlog_resume_rejects_newer_timestamp_and_full_single_replica_identity() {
-        let (source, mut target) = scanner_backlog_replica_pair();
-        target.mod_time = source.mod_time.map(|time| time + time::Duration::SECOND);
-        target.etag = Some("different-native-ledger".to_string());
-        assert!(!scanner_backlog_precondition_resumes(&source, target));
-        assert!(!scanner_backlog_precondition_resumes(&source, source.clone()));
-    }
-
     #[test]
     fn test_scanner_backlog_resume_rejects_changed_payload_or_metadata() {
         let (source, target) = scanner_backlog_replica_pair();

crates/ecstore/src/data_movement/scanner_backlog.rs

@@ -1,292 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use crate::disk::RUSTFS_META_BUCKET;
-use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_found};
-use crate::object_api::ObjectOptions;
-use crate::object_api::{ObjectInfo, PutObjReader, WriteCompletion};
-use crate::set_disk::SetDisks;
-use crate::storage_api_contracts::object::HTTPPreconditions;
-use crate::storage_api_contracts::object::ObjectIO as _;
-use futures::future::join_all;
-use http::HeaderMap;
-use std::sync::{Arc, OnceLock};
-use tokio::io::AsyncReadExt;
-
-pub const MAX_SCANNER_PAUSE_BACKLOG_BYTES: u64 = 64 * 1024;
-pub(crate) const SCANNER_PAUSE_BACKLOG_PATH: &str = "buckets/.scanner-pause-backlog.json";
-
-/// A bounded, storage-fenced native replica. Only a confirmed missing object
-/// has no payload; read failures never enter the Scanner verifier.
-pub struct ScannerPauseBacklogRetirementReplica {
-    pub pool_index: usize,
-    pub set_index: usize,
-    pub data: Option<Vec<u8>>,
-}
-
-/// Native records for a membership handoff. Existing durable ledgers are
-/// preserved; an empty native bootstrap may initialize its first ledger.
-pub struct ScannerPauseBacklogRetirementPlan {
-    pub seed_record: Option<Vec<u8>>,
-    pub commit_record: Vec<u8>,
-    pub stable_record: Vec<u8>,
-}
-
-pub type ScannerPauseBacklogRetirementPlanner =
-    fn(usize, &[ScannerPauseBacklogRetirementReplica]) -> std::result::Result<Option<ScannerPauseBacklogRetirementPlan>, String>;
-
-static RETIREMENT_PLANNER: OnceLock<ScannerPauseBacklogRetirementPlanner> = OnceLock::new();
-
-/// Install the stateless native record planner before storage starts workers.
-/// The scanner runtime switch does not control this storage safety check.
-pub fn register_scanner_pause_backlog_retirement_planner(planner: ScannerPauseBacklogRetirementPlanner) {
-    RETIREMENT_PLANNER.get_or_init(|| planner);
-}
-
-pub(crate) fn is_scanner_pause_backlog(bucket: &str, object: &str) -> bool {
-    bucket == RUSTFS_META_BUCKET && object == SCANNER_PAUSE_BACKLOG_PATH
-}
-
-pub(crate) struct ScannerPauseBacklogRetirementRead {
-    pub replica: ScannerPauseBacklogRetirementReplica,
-    pub etag: Option<String>,
-}
-
-impl ScannerPauseBacklogRetirementRead {
-    pub(crate) fn preconditions(&self) -> HTTPPreconditions {
-        match &self.etag {
-            Some(etag) => HTTPPreconditions {
-                if_match: Some(etag.clone()),
-                ..Default::default()
-            },
-            None => HTTPPreconditions {
-                if_none_match: Some("*".to_string()),
-                ..Default::default()
-            },
-        }
-    }
-}
-
-async fn read_replica(set: Arc<SetDisks>) -> Result<ScannerPauseBacklogRetirementRead> {
-    let mut replica = ScannerPauseBacklogRetirementReplica {
-        pool_index: set.pool_index,
-        set_index: set.set_index,
-        data: None,
-    };
-    let reader = match set
-        .get_object_reader(
-            RUSTFS_META_BUCKET,
-            SCANNER_PAUSE_BACKLOG_PATH,
-            None,
-            HeaderMap::new(),
-            &ObjectOptions {
-                no_lock: true,
-                ..Default::default()
-            },
-        )
-        .await
-    {
-        Ok(reader) => reader,
-        Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
-            return Ok(ScannerPauseBacklogRetirementRead { replica, etag: None });
-        }
-        Err(err) => return Err(err),
-    };
-    let info = &reader.object_info;
-    if info.version_id.is_some_and(|version| !version.is_nil())
-        || info.delete_marker
-        || info.is_dir
-        || info.etag.as_ref().is_none_or(String::is_empty)
-        || info.size < 0
-        || info.size > MAX_SCANNER_PAUSE_BACKLOG_BYTES as i64
-    {
-        return Err(Error::other("scanner pause backlog retirement found an unsupported replica identity"));
-    }
-    let etag = info.etag.clone();
-    let expected_size = info.size as usize;
-    let mut data = Vec::new();
-    reader
-        .take(MAX_SCANNER_PAUSE_BACKLOG_BYTES + 1)
-        .read_to_end(&mut data)
-        .await?;
-    if data.len() != expected_size || data.len() > MAX_SCANNER_PAUSE_BACKLOG_BYTES as usize {
-        return Err(Error::other("scanner pause backlog retirement replica has an invalid payload length"));
-    }
-    replica.data = Some(data);
-    Ok(ScannerPauseBacklogRetirementRead { replica, etag })
-}
-
-/// The caller retains the fixed object write lock and durable topology read
-/// fence through both this snapshot and physical source cleanup.
-pub(crate) async fn read_scanner_pause_backlog_retirement_replicas(
-    source_pool_index: usize,
-    source_set_index: usize,
-    sets: Vec<Arc<SetDisks>>,
-) -> Result<Vec<ScannerPauseBacklogRetirementRead>> {
-    let replicas = join_all(sets.into_iter().map(read_replica))
-        .await
-        .into_iter()
-        .collect::<Result<Vec<_>>>()?;
-    if !replicas.iter().any(|read| {
-        read.replica.pool_index == source_pool_index && read.replica.set_index == source_set_index && read.replica.data.is_some()
-    }) {
-        return Err(Error::other("scanner pause backlog retirement current source replica is missing"));
-    }
-    Ok(replicas)
-}
-
-pub(crate) fn plan_scanner_pause_backlog_retirement(
-    source_pool_index: usize,
-    replicas: &[ScannerPauseBacklogRetirementRead],
-) -> Result<Option<ScannerPauseBacklogRetirementPlan>> {
-    let planner = RETIREMENT_PLANNER
-        .get()
-        .ok_or_else(|| Error::other("scanner pause backlog native retirement planner is unavailable"))?;
-    let snapshots = replicas
-        .iter()
-        .map(|read| ScannerPauseBacklogRetirementReplica {
-            pool_index: read.replica.pool_index,
-            set_index: read.replica.set_index,
-            data: read.replica.data.clone(),
-        })
-        .collect::<Vec<_>>();
-    planner(source_pool_index, &snapshots).map_err(Error::other)
-}
-
-/// The native writer and retirement handoff use the same conditional, full-tail
-/// write. Their callers retain object and durable membership fences until return.
-pub(crate) async fn persist_native_scanner_pause_backlog_replica(
-    set: Arc<SetDisks>,
-    data: Vec<u8>,
-    preconditions: HTTPPreconditions,
-    mut opts: ObjectOptions,
-    _phase: &'static str,
-) -> Result<ObjectInfo> {
-    if data.len() > MAX_SCANNER_PAUSE_BACKLOG_BYTES as usize {
-        return Err(Error::other("scanner pause backlog exceeds its size bound"));
-    }
-    opts.max_parity = true;
-    opts.write_completion = WriteCompletion::TailDrained;
-    opts.http_preconditions = Some(preconditions);
-    #[cfg(feature = "test-util")]
-    let fault = test_util::matching_write(&set, _phase)?;
-    let result = set
-        .put_object(RUSTFS_META_BUCKET, SCANNER_PAUSE_BACKLOG_PATH, &mut PutObjReader::from_vec(data), &opts)
-        .await;
-    #[cfg(feature = "test-util")]
-    if result.is_ok()
-        && let Some(fault) = fault
-    {
-        fault.arrived.notify_one();
-        fault.release.notified().await;
-    }
-    result
-}
-
-#[cfg(feature = "test-util")]
-pub mod test_util {
-    use super::*;
-    use std::sync::Mutex;
-    use std::sync::atomic::{AtomicUsize, Ordering};
-    use tokio::sync::Notify;
-
-    #[derive(Debug, thiserror::Error)]
-    #[error("injected native scanner backlog {phase} write failure")]
-    struct InjectedWriteFailure {
-        phase: &'static str,
-    }
-
-    pub(super) struct WriteFault {
-        set: Arc<SetDisks>,
-        phase: &'static str,
-        remaining: AtomicUsize,
-        fail_before_write: bool,
-        pub(super) arrived: Notify,
-        pub(super) release: Notify,
-    }
-
-    static WRITE_FAULTS: Mutex<Vec<Arc<WriteFault>>> = Mutex::new(Vec::new());
-
-    /// Scope a one-shot fault to the actual set instance, so other stores and
-    /// concurrent tests keep using the ordinary native persistence path.
-    pub struct NativeScannerPauseBacklogWriteFault {
-        state: Arc<WriteFault>,
-    }
-
-    impl NativeScannerPauseBacklogWriteFault {
-        fn install(set: Arc<SetDisks>, phase: &'static str, nth: usize, fail_before_write: bool) -> Self {
-            assert!(nth > 0);
-            let state = Arc::new(WriteFault {
-                set,
-                phase,
-                remaining: AtomicUsize::new(nth),
-                fail_before_write,
-                arrived: Notify::new(),
-                release: Notify::new(),
-            });
-            let mut faults = WRITE_FAULTS.lock().unwrap();
-            assert!(
-                !faults
-                    .iter()
-                    .any(|fault| Arc::ptr_eq(&fault.set, &state.set) && fault.phase == phase)
-            );
-            faults.push(Arc::clone(&state));
-            Self { state }
-        }
-
-        pub fn fail_before_write(set: Arc<SetDisks>, phase: &'static str, nth: usize) -> Self {
-            Self::install(set, phase, nth, true)
-        }
-
-        pub fn pause_after_write(set: Arc<SetDisks>, phase: &'static str) -> Self {
-            Self::install(set, phase, 1, false)
-        }
-
-        pub async fn wait_until_paused(&self) {
-            self.state.arrived.notified().await;
-        }
-
-        pub fn release(&self) {
-            self.state.release.notify_one();
-        }
-    }
-
-    impl Drop for NativeScannerPauseBacklogWriteFault {
-        fn drop(&mut self) {
-            self.release();
-            WRITE_FAULTS.lock().unwrap().retain(|fault| !Arc::ptr_eq(fault, &self.state));
-        }
-    }
-
-    pub(super) fn matching_write(set: &Arc<SetDisks>, phase: &'static str) -> Result<Option<Arc<WriteFault>>> {
-        let fault = WRITE_FAULTS
-            .lock()
-            .unwrap()
-            .iter()
-            .find(|fault| Arc::ptr_eq(&fault.set, set) && fault.phase == phase)
-            .cloned();
-        let Some(fault) = fault else { return Ok(None) };
-        if fault
-            .remaining
-            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| remaining.checked_sub(1))
-            != Ok(1)
-        {
-            return Ok(None);
-        }
-        if fault.fail_before_write {
-            return Err(Error::other(InjectedWriteFailure { phase }));
-        }
-        Ok(Some(fault))
-    }
-}

crates/ecstore/src/data_usage/mod.rs

@@ -3397,7 +3397,6 @@ mod tests {
             scan_plan_digest: Some([1; 32]),
             complete: false,
             tombstone: false,
-            segment_invalidation_proof: None,
         }];
         partial.buckets_usage.insert(
             "bucket".to_string(),
@@ -3470,7 +3469,6 @@ mod tests {
                 scan_plan_digest: Some([1; 32]),
                 complete: true,
                 tombstone: false,
-                segment_invalidation_proof: None,
             }],
             ..Default::default()
         };

crates/ecstore/src/disk/error.rs

@@ -627,13 +627,7 @@ impl From<tokio::task::JoinError> for DiskError {
 impl Clone for DiskError {
     fn clone(&self) -> Self {
         match self {
-            DiskError::Io(io_error) => DiskError::Io(
-                rustfs_rio::clone_internode_http_io_error(io_error)
-                    .and_then(std::io::Error::into_inner)
-                    // The helper derives a kind from the source; Clone must retain the original outer kind.
-                    .map(|source| std::io::Error::new(io_error.kind(), source))
-                    .unwrap_or_else(|| std::io::Error::new(io_error.kind(), io_error.to_string())),
-            ),
+            DiskError::Io(io_error) => DiskError::Io(std::io::Error::new(io_error.kind(), io_error.to_string())),
             DiskError::MaxVersionsExceeded => DiskError::MaxVersionsExceeded,
             DiskError::Unexpected => DiskError::Unexpected,
             DiskError::CorruptedFormat => DiskError::CorruptedFormat,
@@ -1271,49 +1265,6 @@ mod tests {
         assert!(!bad_request.is_retryable_internode_write_failure());
     }
 
-    #[test]
-    fn test_internode_http_clone_preserves_retryability_status_and_context() {
-        use http::StatusCode;
-        use rustfs_rio::InternodeHttpErrorKind::{ConnectionRefused, ConnectionReset, HttpStatus, Unknown};
-
-        for (kind, retryable) in [
-            (ConnectionRefused, true),
-            (ConnectionReset, true),
-            (HttpStatus(StatusCode::TOO_MANY_REQUESTS), true),
-            (HttpStatus(StatusCode::SERVICE_UNAVAILABLE), true),
-            (HttpStatus(StatusCode::CONFLICT), true),
-            (Unknown, false),
-            (HttpStatus(StatusCode::BAD_REQUEST), false),
-            (HttpStatus(StatusCode::INTERNAL_SERVER_ERROR), false),
-        ] {
-            let original = DiskError::from(rustfs_rio::new_test_internode_http_io_error(kind));
-            assert_eq!(original.internode_http_error_kind(), Some(kind));
-            assert_eq!(original.is_retryable_internode_write_failure(), retryable);
-
-            let cloned = original.clone();
-            assert_eq!(cloned, original, "clone must preserve the error bucket for {kind:?}");
-            assert_eq!(
-                cloned.is_retryable_internode_write_failure(),
-                retryable,
-                "clone changed retryability for {kind:?}"
-            );
-            assert_eq!(cloned.internode_http_error_kind(), Some(kind));
-            if let HttpStatus(status) = kind {
-                assert!(cloned.is_internode_http_status(status.as_u16()));
-            }
-            let DiskError::Io(io_error) = &cloned else {
-                panic!("unmarked internode error must remain Io: {cloned:?}");
-            };
-            let source = io_error
-                .get_ref()
-                .and_then(|source| source.downcast_ref::<InternodeHttpError>())
-                .expect("clone must retain the structured internode error");
-            assert_eq!(source.context().method(), "PUT");
-            assert_eq!(source.context().target(), "/rustfs/rpc/put_file_stream");
-            assert_eq!(source.context().operation(), Some(INTERNODE_OPERATION_PUT_FILE_STREAM));
-        }
-    }
-
     #[tokio::test]
     async fn read_stream_conflict_is_not_a_retryable_put_file_failure() {
         use tokio::io::{AsyncReadExt, AsyncWriteExt};
@@ -1358,57 +1309,11 @@ mod tests {
                 !error.is_retryable_internode_write_failure(),
                 "read-operation 409 must not trigger put-file retry"
             );
-            let cloned = error.clone();
-            let reduced = crate::disk::error_reduce::reduce_write_quorum_errs(&[Some(error)], &[], 1)
-                .expect("the read conflict must remain the dominant error");
-            for preserved in [&cloned, &reduced] {
-                assert!(
-                    !preserved.is_retryable_internode_write_failure(),
-                    "cloning or reducing a read conflict must not turn it into a PUT retry"
-                );
-                assert!(preserved.is_internode_http_status(409));
-                let DiskError::Io(io_error) = preserved else {
-                    panic!("read conflict must remain Io: {preserved:?}");
-                };
-                let source = io_error
-                    .get_ref()
-                    .and_then(|source| source.downcast_ref::<InternodeHttpError>())
-                    .expect("read conflict must retain its request context");
-                assert_eq!(source.context().method(), "GET");
-                assert_eq!(source.context().target(), "/rustfs/rpc/read_file_stream");
-                assert_eq!(
-                    source.context().operation(),
-                    Some(rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_READ_FILE_STREAM)
-                );
-            }
         })
         .await
         .expect("isolated read-conflict test must finish within its budget");
     }
 
-    #[test]
-    fn test_internode_http_clone_preserves_outer_io_kind_and_message() {
-        let source = rustfs_rio::new_test_internode_http_io_error(InternodeHttpErrorKind::ConnectionReset)
-            .into_inner()
-            .expect("the internode helper must provide a typed source");
-        let original_io = io::Error::new(io::ErrorKind::InvalidData, source);
-        let message = original_io.to_string();
-        let original = DiskError::from(original_io);
-        assert_eq!(original.internode_http_error_kind(), Some(InternodeHttpErrorKind::ConnectionReset));
-        assert!(original.is_retryable_internode_write_failure());
-
-        let cloned = original.clone();
-        let reduced = crate::disk::error_reduce::reduce_write_quorum_errs(&[Some(original)], &[], 1)
-            .expect("the wrapped internode error must remain the dominant error");
-        for preserved in [&cloned, &reduced] {
-            let DiskError::Io(io_error) = preserved else {
-                panic!("the wrapped error must remain Io: {preserved:?}");
-            };
-            assert_eq!(io_error.kind(), io::ErrorKind::InvalidData);
-            assert_eq!(io_error.to_string(), message);
-        }
-    }
-
     #[test]
     fn test_internode_missing_errors_preserve_disk_error_types() {
         let file_missing = DiskError::from(rustfs_rio::new_test_remote_file_not_found_http_io_error());
@@ -1420,17 +1325,6 @@ mod tests {
         assert_eq!(file_missing, DiskError::FileNotFound);
         assert_eq!(volume_missing, DiskError::VolumeNotFound);
         assert!(matches!(unmarked_server_error, DiskError::Io(_)));
-        for missing in [file_missing, volume_missing] {
-            assert_eq!(missing.clone(), missing);
-            assert_eq!(
-                crate::disk::error_reduce::reduce_write_quorum_errs(
-                    &[Some(missing.clone()), Some(missing.clone()), None],
-                    &[],
-                    2
-                ),
-                Some(missing)
-            );
-        }
     }
 
     #[test]

crates/ecstore/src/disk/error_reduce.rs

@@ -226,78 +226,6 @@ mod tests {
         assert_eq!(res, Some(quorum_err));
     }
 
-    #[test]
-    fn test_write_quorum_reduction_preserves_internode_http_identity() {
-        use http::StatusCode;
-        use rustfs_rio::InternodeHttpErrorKind::{ConnectionRefused, HttpStatus, Unknown};
-
-        for (kind, retryable) in [
-            (ConnectionRefused, true),
-            (HttpStatus(StatusCode::SERVICE_UNAVAILABLE), true),
-            (HttpStatus(StatusCode::CONFLICT), true),
-            (Unknown, false),
-            (HttpStatus(StatusCode::BAD_REQUEST), false),
-        ] {
-            // Construct both producer errors independently: the reducer owns the first clone.
-            let first = Error::from(rustfs_rio::new_test_internode_http_io_error(kind));
-            let second = Error::from(rustfs_rio::new_test_internode_http_io_error(kind));
-            assert_eq!(first.internode_http_error_kind(), Some(kind));
-            assert_eq!(second.internode_http_error_kind(), Some(kind));
-            assert_eq!(first.is_retryable_internode_write_failure(), retryable);
-            let errors = [Some(first), Some(second), None];
-            let reduced = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, 2)
-                .expect("two equal producer errors must dominate one successful write");
-
-            assert_eq!(Some(&reduced), errors[0].as_ref());
-            assert_eq!(
-                reduced.is_retryable_internode_write_failure(),
-                retryable,
-                "quorum reduction changed retryability for {kind:?}"
-            );
-            assert_eq!(reduced.internode_http_error_kind(), Some(kind));
-            if let HttpStatus(status) = kind {
-                assert!(reduced.is_internode_http_status(status.as_u16()));
-            }
-            let Error::Io(io_error) = &reduced else {
-                panic!("the dominant error must remain Io: {reduced:?}");
-            };
-            let source = io_error
-                .get_ref()
-                .and_then(|source| source.downcast_ref::<rustfs_rio::InternodeHttpError>())
-                .expect("quorum reduction must retain the structured internode error");
-            assert_eq!(source.context().method(), "PUT");
-            assert_eq!(source.context().target(), "/rustfs/rpc/put_file_stream");
-            assert_eq!(
-                source.context().operation(),
-                Some(rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_PUT_FILE_STREAM)
-            );
-        }
-    }
-
-    #[test]
-    fn test_clone_and_write_quorum_do_not_promote_non_retryable_errors() {
-        use http::StatusCode;
-        use rustfs_rio::InternodeHttpErrorKind::{HttpStatus, Unknown};
-
-        for original in [
-            Error::from(rustfs_rio::new_test_internode_http_io_error(Unknown)),
-            Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(StatusCode::BAD_REQUEST))),
-            Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(StatusCode::FORBIDDEN))),
-            Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(StatusCode::NOT_FOUND))),
-            Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(
-                StatusCode::INTERNAL_SERVER_ERROR,
-            ))),
-            err_io("internode connection reset: PUT /rustfs/rpc/put_file_stream"),
-        ] {
-            assert!(!original.is_retryable_internode_write_failure());
-            let cloned = original.clone();
-            let reduced =
-                reduce_write_quorum_errs(&[Some(original)], &[], 1).expect("a non-retryable error must remain an error");
-            assert!(!cloned.is_retryable_internode_write_failure());
-            assert!(!reduced.is_retryable_internode_write_failure());
-        }
-    }
-
     #[test]
     fn test_count_errs() {
         let e1 = err_io("a");

crates/ecstore/src/disk/local.rs

@@ -219,7 +219,6 @@ fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, r
     }
 }
 
-#[cfg(test)]
 async fn write_metadata_rollback_backup(object_dir: &Path, rollback_dir: Uuid, data: &[u8]) -> Result<()> {
     write_delete_rollback_file(object_dir, rollback_dir, STORAGE_FORMAT_FILE_BACKUP, data, None).await
 }
@@ -251,7 +250,6 @@ async fn write_delete_rollback_file(
     Ok(())
 }
 
-#[cfg(test)]
 async fn restore_metadata_backup(
     object_dir: &Path,
     xl_path: &Path,
@@ -279,6 +277,15 @@ async fn restore_metadata_backup_with_namespace_owner(
     Ok(())
 }
 
+async fn restore_delete_rollback(
+    object_dir: &Path,
+    xl_path: &Path,
+    rollback_dir: Uuid,
+    publication_root: &os::PublicationRoot,
+) -> Result<()> {
+    restore_delete_rollback_with_namespace_owner(object_dir, xl_path, rollback_dir, publication_root, None).await
+}
+
 async fn restore_delete_rollback_with_namespace_owner(
     object_dir: &Path,
     xl_path: &Path,
@@ -5850,8 +5857,6 @@ impl LocalDisk {
         check_path_length(file_path.to_string_lossy().as_ref())?;
 
         let xl_path = path_join(&[file_path.as_path(), Path::new(STORAGE_FORMAT_FILE)]);
-        let namespace_owner: Option<Arc<dyn Send + Sync>> =
-            Some(os::acquire_metadata_mutation_lease(&self.get_object_path(volume, path)?, namespace_owner).await);
         if opts.old_data_dir.is_some() && opts.undo_write {
             return self.undo_write(file_path.as_path(), &fi, &opts, namespace_owner).await;
         }
@@ -6409,10 +6414,6 @@ impl LocalDisk {
                 // A missing or still-populated directory is benign here; see
                 // is_benign_object_rmdir_error (handles the illumos/Solaris EEXIST
                 // convention, rustfs/rustfs#4978).
-                if is_dir_not_empty_error(&err) {
-                    // A populated directory keeps its ancestors populated; no further pruning is needed.
-                    return Ok(());
-                }
                 if !is_benign_object_rmdir_error(&err) {
                     warn!(
                         event = EVENT_DISK_LOCAL_DELETE_FAILED,
@@ -6713,8 +6714,6 @@ impl LocalDisk {
 
     async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &[FileInfo], opts: &DeleteOptions) -> Result<()> {
         let volume_dir = self.io_get_bucket_path(volume)?;
-        let object_path = self.get_object_path(volume, path)?;
-        let namespace_owner: Option<Arc<dyn Send + Sync>> = Some(os::acquire_metadata_mutation_lease(&object_path, None).await);
         let xlpath = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?;
         let object_dir = xlpath
             .parent()
@@ -6724,24 +6723,10 @@ impl LocalDisk {
             && opts.undo_write
         {
             if opts.undo_delete {
-                return restore_delete_rollback_with_namespace_owner(
-                    object_dir,
-                    &xlpath,
-                    rollback_dir,
-                    &self.publication_root,
-                    namespace_owner.clone(),
-                )
-                .await;
+                return restore_delete_rollback(object_dir, &xlpath, rollback_dir, &self.publication_root).await;
             }
 
-            return restore_metadata_backup_with_namespace_owner(
-                object_dir,
-                &xlpath,
-                rollback_dir,
-                &self.publication_root,
-                namespace_owner.clone(),
-            )
-            .await;
+            return restore_metadata_backup(object_dir, &xlpath, rollback_dir, &self.publication_root).await;
         }
 
         let (data, _) = match self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await {
@@ -6753,14 +6738,7 @@ impl LocalDisk {
                     return Err(DiskError::FileNotFound);
                 };
                 return self
-                    .write_missing_delete_marker(
-                        volume,
-                        path,
-                        delete_marker,
-                        object_dir,
-                        opts.old_data_dir,
-                        namespace_owner.clone(),
-                    )
+                    .write_missing_delete_marker(volume, path, delete_marker, object_dir, opts.old_data_dir, None)
                     .await;
             }
             Err(err) => return Err(err),
@@ -6776,8 +6754,7 @@ impl LocalDisk {
         let rollback_dir = opts.old_data_dir;
         let mut reserved_version_delete = false;
         if let Some(rollback_dir) = rollback_dir {
-            write_delete_rollback_file(object_dir, rollback_dir, STORAGE_FORMAT_FILE_BACKUP, &data, namespace_owner.clone())
-                .await?;
+            write_metadata_rollback_backup(object_dir, rollback_dir, &data).await?;
         }
 
         for fi in fis.iter() {
@@ -6791,16 +6768,13 @@ impl LocalDisk {
 
                     if reserved_version_delete && let Some(rollback_dir) = rollback_dir {
                         return Err(self
-                            .abort_reserved_version_delete_with_failure(
+                            .abort_reserved_version_delete(
                                 object_dir,
                                 rollback_dir,
                                 volume,
                                 path,
-                                DeleteRollbackFailure {
-                                    stage: "delete_versions_metadata_update",
-                                    error: err,
-                                    namespace_owner: namespace_owner.clone(),
-                                },
+                                "delete_versions_metadata_update",
+                                err,
                             )
                             .await);
                     }
@@ -6813,7 +6787,7 @@ impl LocalDisk {
                         DeleteRollbackFailure {
                             stage: "delete_versions_metadata_update",
                             error: err,
-                            namespace_owner: namespace_owner.clone(),
+                            namespace_owner: None,
                         },
                         &self.publication_root,
                     )
@@ -6830,16 +6804,13 @@ impl LocalDisk {
                     Err(err) => {
                         if reserved_version_delete && let Some(rollback_dir) = rollback_dir {
                             return Err(self
-                                .abort_reserved_version_delete_with_failure(
+                                .abort_reserved_version_delete(
                                     object_dir,
                                     rollback_dir,
                                     volume,
                                     path,
-                                    DeleteRollbackFailure {
-                                        stage: "delete_versions_data_path",
-                                        error: err,
-                                        namespace_owner: namespace_owner.clone(),
-                                    },
+                                    "delete_versions_data_path",
+                                    err,
                                 )
                                 .await);
                         }
@@ -6852,7 +6823,7 @@ impl LocalDisk {
                             DeleteRollbackFailure {
                                 stage: "delete_versions_data_path",
                                 error: err,
-                                namespace_owner: namespace_owner.clone(),
+                                namespace_owner: None,
                             },
                             &self.publication_root,
                         )
@@ -6865,16 +6836,13 @@ impl LocalDisk {
                         let err: DiskError = to_file_error(err).into();
                         if reserved_version_delete {
                             return Err(self
-                                .abort_reserved_version_delete_with_failure(
+                                .abort_reserved_version_delete(
                                     object_dir,
                                     rollback_dir,
                                     volume,
                                     path,
-                                    DeleteRollbackFailure {
-                                        stage: "delete_versions_rollback_dir",
-                                        error: err,
-                                        namespace_owner: namespace_owner.clone(),
-                                    },
+                                    "delete_versions_rollback_dir",
+                                    err,
                                 )
                                 .await);
                         }
@@ -6887,29 +6855,23 @@ impl LocalDisk {
                             DeleteRollbackFailure {
                                 stage: "delete_versions_rollback_dir",
                                 error: err,
-                                namespace_owner: namespace_owner.clone(),
+                                namespace_owner: None,
                             },
                             &self.publication_root,
                         )
                         .await);
                     }
-                    let reserved = match self
-                        .reserve_version_delete_with_namespace_owner(volume, path, dir, rollback_dir, namespace_owner.clone())
-                        .await
-                    {
+                    let reserved = match self.reserve_version_delete(volume, path, dir, rollback_dir).await {
                         Ok(reserved) => reserved,
                         Err(err) => {
                             return Err(self
-                                .abort_reserved_version_delete_with_failure(
+                                .abort_reserved_version_delete(
                                     object_dir,
                                     rollback_dir,
                                     volume,
                                     path,
-                                    DeleteRollbackFailure {
-                                        stage: "delete_versions_reserve_data",
-                                        error: err,
-                                        namespace_owner: namespace_owner.clone(),
-                                    },
+                                    "delete_versions_reserve_data",
+                                    err,
                                 )
                                 .await);
                         }
@@ -6917,12 +6879,11 @@ impl LocalDisk {
                     reserved_version_delete |= reserved;
                     let rollback_data_path = rollback_path.join(dir.to_string());
                     if !reserved
-                        && let Err(err) = os::rename_all_ignore_missing_source_with_owner(
+                        && let Err(err) = rename_all_ignore_missing_source(
                             &dir_path,
                             &rollback_data_path,
                             &rollback_path,
                             &self.publication_root,
-                            namespace_owner.clone(),
                         )
                         .await
                     {
@@ -6935,7 +6896,7 @@ impl LocalDisk {
                             DeleteRollbackFailure {
                                 stage: "delete_versions_stage_data",
                                 error: err,
-                                namespace_owner: namespace_owner.clone(),
+                                namespace_owner: None,
                             },
                             &self.publication_root,
                         )
@@ -6944,16 +6905,13 @@ impl LocalDisk {
                     if should_fail_after_delete_data_staged(path) {
                         if reserved_version_delete {
                             return Err(self
-                                .abort_reserved_version_delete_with_failure(
+                                .abort_reserved_version_delete(
                                     object_dir,
                                     rollback_dir,
                                     volume,
                                     path,
-                                    DeleteRollbackFailure {
-                                        stage: "delete_versions_test_after_stage",
-                                        error: DiskError::Unexpected,
-                                        namespace_owner: namespace_owner.clone(),
-                                    },
+                                    "delete_versions_test_after_stage",
+                                    DiskError::Unexpected,
                                 )
                                 .await);
                         }
@@ -6966,15 +6924,13 @@ impl LocalDisk {
                             DeleteRollbackFailure {
                                 stage: "delete_versions_test_after_stage",
                                 error: DiskError::Unexpected,
-                                namespace_owner: namespace_owner.clone(),
+                                namespace_owner: None,
                             },
                             &self.publication_root,
                         )
                         .await);
                     }
-                } else if let Err(err) = self
-                    .move_to_trash_with_namespace_owner(&dir_path, true, false, namespace_owner.clone())
-                    .await
+                } else if let Err(err) = self.move_to_trash(&dir_path, true, false).await
                     && !(err == DiskError::FileNotFound || err == DiskError::VolumeNotFound)
                 {
                     return Err(err);
@@ -6989,22 +6945,16 @@ impl LocalDisk {
 
         // Remove xl.meta when no versions remain
         if fm.versions.is_empty() {
-            if let Err(err) = self
-                .delete_file_with_namespace_owner(&volume_dir, &xlpath, true, false, namespace_owner.clone())
-                .await
-            {
+            if let Err(err) = self.delete_file(&volume_dir, &xlpath, true, false).await {
                 if reserved_version_delete && let Some(rollback_dir) = rollback_dir {
                     return Err(self
-                        .abort_reserved_version_delete_with_failure(
+                        .abort_reserved_version_delete(
                             object_dir,
                             rollback_dir,
                             volume,
                             path,
-                            DeleteRollbackFailure {
-                                stage: "delete_versions_commit_delete",
-                                error: err,
-                                namespace_owner: namespace_owner.clone(),
-                            },
+                            "delete_versions_commit_delete",
+                            err,
                         )
                         .await);
                 }
@@ -7017,7 +6967,7 @@ impl LocalDisk {
                     DeleteRollbackFailure {
                         stage: "delete_versions_commit_delete",
                         error: err,
-                        namespace_owner: namespace_owner.clone(),
+                        namespace_owner: None,
                     },
                     &self.publication_root,
                 )
@@ -7025,22 +6975,10 @@ impl LocalDisk {
             }
             if reserved_version_delete
                 && let Some(rollback_dir) = rollback_dir
-                && let Err(err) = self
-                    .commit_reserved_version_delete_with_namespace_owner(volume, path, rollback_dir, namespace_owner.clone())
-                    .await
+                && let Err(err) = self.commit_reserved_version_delete(volume, path, rollback_dir).await
             {
                 return Err(self
-                    .abort_reserved_version_delete_with_failure(
-                        object_dir,
-                        rollback_dir,
-                        volume,
-                        path,
-                        DeleteRollbackFailure {
-                            stage: "delete_versions_commit_intent",
-                            error: err,
-                            namespace_owner: namespace_owner.clone(),
-                        },
-                    )
+                    .abort_reserved_version_delete(object_dir, rollback_dir, volume, path, "delete_versions_commit_intent", err)
                     .await);
             }
             if should_fail_after_delete_commit(self.root.as_path(), path) {
@@ -7057,16 +6995,13 @@ impl LocalDisk {
                 let err: DiskError = err.into();
                 if reserved_version_delete && let Some(rollback_dir) = rollback_dir {
                     return Err(self
-                        .abort_reserved_version_delete_with_failure(
+                        .abort_reserved_version_delete(
                             object_dir,
                             rollback_dir,
                             volume,
                             path,
-                            DeleteRollbackFailure {
-                                stage: "delete_versions_metadata_encode",
-                                error: err,
-                                namespace_owner: namespace_owner.clone(),
-                            },
+                            "delete_versions_metadata_encode",
+                            err,
                         )
                         .await);
                 }
@@ -7079,7 +7014,7 @@ impl LocalDisk {
                     DeleteRollbackFailure {
                         stage: "delete_versions_metadata_encode",
                         error: err,
-                        namespace_owner: namespace_owner.clone(),
+                        namespace_owner: None,
                     },
                     &self.publication_root,
                 )
@@ -7088,28 +7023,12 @@ impl LocalDisk {
         };
 
         if let Err(err) = self
-            .write_all_meta_with_namespace_owner(
-                volume,
-                format!("{path}/{STORAGE_FORMAT_FILE}").as_str(),
-                &buf,
-                true,
-                namespace_owner.clone(),
-            )
+            .write_all_meta(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str(), &buf, true)
             .await
         {
             if reserved_version_delete && let Some(rollback_dir) = rollback_dir {
                 return Err(self
-                    .abort_reserved_version_delete_with_failure(
-                        object_dir,
-                        rollback_dir,
-                        volume,
-                        path,
-                        DeleteRollbackFailure {
-                            stage: "delete_versions_commit_write",
-                            error: err,
-                            namespace_owner: namespace_owner.clone(),
-                        },
-                    )
+                    .abort_reserved_version_delete(object_dir, rollback_dir, volume, path, "delete_versions_commit_write", err)
                     .await);
             }
             return Err(restore_delete_rollback_after_error(
@@ -7121,7 +7040,7 @@ impl LocalDisk {
                 DeleteRollbackFailure {
                     stage: "delete_versions_commit_write",
                     error: err,
-                    namespace_owner: namespace_owner.clone(),
+                    namespace_owner: None,
                 },
                 &self.publication_root,
             )
@@ -7130,22 +7049,10 @@ impl LocalDisk {
 
         if reserved_version_delete
             && let Some(rollback_dir) = rollback_dir
-            && let Err(err) = self
-                .commit_reserved_version_delete_with_namespace_owner(volume, path, rollback_dir, namespace_owner.clone())
-                .await
+            && let Err(err) = self.commit_reserved_version_delete(volume, path, rollback_dir).await
         {
             return Err(self
-                .abort_reserved_version_delete_with_failure(
-                    object_dir,
-                    rollback_dir,
-                    volume,
-                    path,
-                    DeleteRollbackFailure {
-                        stage: "delete_versions_commit_intent",
-                        error: err,
-                        namespace_owner: namespace_owner.clone(),
-                    },
-                )
+                .abort_reserved_version_delete(object_dir, rollback_dir, volume, path, "delete_versions_commit_intent", err)
                 .await);
         }
 
@@ -7156,107 +7063,6 @@ impl LocalDisk {
         Ok(())
     }
 
-    async fn reconcile_transition_state_metadata(
-        &self,
-        volume: &str,
-        object: &str,
-        version_id: Option<Uuid>,
-        condition: &super::TransitionStateReconcileCondition,
-        namespace_owner: Option<Arc<dyn Send + Sync>>,
-        authority: Arc<crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority>,
-    ) -> Result<()> {
-        condition.target.validate()?;
-        if !authority.is_current()
-            || [
-                condition.expected_metadata_digest.as_str(),
-                condition.unchanged_metadata_digest.as_str(),
-            ]
-            .iter()
-            .any(|digest| {
-                digest.len() != 64
-                    || !digest
-                        .bytes()
-                        .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
-            })
-        {
-            return Err(DiskError::OutdatedXLMeta);
-        }
-        let metadata_path = format!("{object}/{STORAGE_FORMAT_FILE}");
-        let original = self.read_all(volume, &metadata_path).await?;
-        let original_digest = rustfs_utils::crypto::hex(<sha2::Sha256 as sha2::Digest>::digest(&original));
-        let mut metadata = FileMeta::load(&original)?;
-        let (_, selected) = metadata.find_version(version_id)?;
-        if selected.into_fileinfo(volume, object, true)?.transition_tier != condition.tier {
-            return Err(DiskError::OutdatedXLMeta);
-        }
-        let generation = metadata.transition_reconcile_generation(version_id)?;
-        if rustfs_utils::crypto::hex(<sha2::Sha256 as sha2::Digest>::digest(&generation)) != condition.unchanged_metadata_digest {
-            return Err(DiskError::OutdatedXLMeta);
-        }
-        let changed = metadata.reconcile_transition_state(version_id, &condition.target)?;
-        if !changed {
-            return Ok(());
-        }
-        if condition.verify_only || original_digest != condition.expected_metadata_digest {
-            return Err(DiskError::OutdatedXLMeta);
-        }
-        // fsync_dir_std is a no-op outside Unix, so those platforms cannot
-        // yet prove this repair's durable publication requirement.
-        if !cfg!(unix) || !effective_durability(volume).syncs_commit_metadata() {
-            return Err(DiskError::other(
-                "transition reconciliation requires Unix directory sync and enabled bucket metadata durability",
-            ));
-        }
-        // An outstanding rollback can still restore an older whole xl.meta.
-        // Its preparation and execution share this mutation domain; refuse
-        // repair until that transaction has settled and removed its backup.
-        let mut entries = fs::read_dir(self.io_get_object_path(volume, object)?)
-            .await
-            .map_err(to_file_error)?;
-        let mut remaining = 4096usize;
-        while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? {
-            remaining = remaining.checked_sub(1).ok_or(DiskError::OutdatedXLMeta)?;
-            if Uuid::parse_str(&entry.file_name().to_string_lossy()).is_err() {
-                continue;
-            }
-            for marker in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] {
-                if fs::try_exists(entry.path().join(marker)).await.map_err(to_file_error)? {
-                    return Err(DiskError::OutdatedXLMeta);
-                }
-            }
-        }
-        let replacement = metadata.marshal_msg()?;
-        let tmp_volume = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
-        let tmp_file = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, &Uuid::new_v4().to_string())?;
-        // Admission above requires metadata durability. Keep rename and its
-        // directory sync in the same owned executor even after cancellation.
-        self.write_all_internal(&tmp_file, InternalBuf::Ref(&replacement), SyncMode::FileOnly, &tmp_volume)
-            .await?;
-        if crash_inject::should_crash_at(CrashPoint::MetaWriteAfterTmpBeforeRename, &metadata_path) {
-            return Err(DiskError::Unexpected);
-        }
-        os::rename_reconciled_metadata(
-            tmp_file,
-            self.io_get_object_path(volume, &metadata_path)?,
-            self.io_get_bucket_path(volume)?,
-            self.publication_root.clone(),
-            namespace_owner.clone(),
-            authority,
-        )
-        .await?;
-        // Keep the same mutation lease through strong readback. Response loss
-        // leaves a monotonic subset for the coordinator's exact-copy retry.
-        let committed = self.read_all(volume, &metadata_path).await?;
-        let mut committed = FileMeta::load(&committed)?;
-        if committed.reconcile_transition_state(version_id, &condition.target)?
-            || committed.transition_reconcile_generation(version_id)? != generation
-        {
-            return Err(DiskError::OutdatedXLMeta);
-        }
-        Ok(())
-    }
-
-    #[cfg(test)]
     async fn write_all_meta(&self, volume: &str, path: &str, buf: &[u8], sync: bool) -> Result<()> {
         self.write_all_meta_with_namespace_owner(volume, path, buf, sync, None).await
     }
@@ -8515,7 +8321,6 @@ impl LocalDisk {
         Ok(Arc::new(QuotaMutationFenceClaim { state }))
     }
 
-    #[cfg(test)]
     async fn reserve_version_delete(&self, volume: &str, object: &str, data_dir: Uuid, rollback_dir: Uuid) -> Result<bool> {
         self.reserve_version_delete_with_namespace_owner(volume, object, data_dir, rollback_dir, None)
             .await
@@ -8568,7 +8373,6 @@ impl LocalDisk {
         Ok(true)
     }
 
-    #[cfg(test)]
     async fn commit_reserved_version_delete(&self, volume: &str, object: &str, rollback_dir: Uuid) -> Result<()> {
         self.commit_reserved_version_delete_with_namespace_owner(volume, object, rollback_dir, None)
             .await
@@ -8660,6 +8464,29 @@ impl LocalDisk {
         first_err.map_or(Ok(found), Err)
     }
 
+    async fn abort_reserved_version_delete(
+        &self,
+        object_dir: &Path,
+        rollback_dir: Uuid,
+        volume: &str,
+        object: &str,
+        stage: &'static str,
+        err: DiskError,
+    ) -> DiskError {
+        self.abort_reserved_version_delete_with_failure(
+            object_dir,
+            rollback_dir,
+            volume,
+            object,
+            DeleteRollbackFailure {
+                stage,
+                error: err,
+                namespace_owner: None,
+            },
+        )
+        .await
+    }
+
     async fn abort_reserved_version_delete_with_failure(
         &self,
         object_dir: &Path,
@@ -10204,23 +10031,6 @@ impl DiskAPI for LocalDisk {
 
     #[tracing::instrument(level = "trace", skip_all)]
     async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
-        let object_path = self.get_object_path(volume, path)?;
-        if let Some(condition) = &opts.transition_reconcile {
-            if !fi.metadata.is_empty() || opts.no_persistence || opts.replace_user_metadata {
-                return Err(DiskError::FileCorrupt);
-            }
-            let authority =
-                crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority::for_disk(
-                    condition,
-                )
-                .await?;
-            let owner: Option<Arc<dyn Send + Sync>> = Some(authority.clone());
-            let owner: Option<Arc<dyn Send + Sync>> = Some(os::acquire_metadata_mutation_lease(&object_path, owner).await);
-            return self
-                .reconcile_transition_state_metadata(volume, path, fi.version_id, condition, owner, authority)
-                .await;
-        }
-        let namespace_owner: Option<Arc<dyn Send + Sync>> = Some(os::acquire_metadata_mutation_lease(&object_path, None).await);
         if !fi.metadata.is_empty() {
             let file_path = self.io_get_object_path(volume, path)?;
 
@@ -10248,13 +10058,7 @@ impl DiskAPI for LocalDisk {
             let wbuf = xl_meta.marshal_msg()?;
 
             return self
-                .write_all_meta_with_namespace_owner(
-                    volume,
-                    format!("{path}/{STORAGE_FORMAT_FILE}").as_str(),
-                    &wbuf,
-                    !opts.no_persistence,
-                    namespace_owner,
-                )
+                .write_all_meta(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str(), &wbuf, !opts.no_persistence)
                 .await;
         }
 
@@ -10262,10 +10066,7 @@ impl DiskAPI for LocalDisk {
     }
 
     async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
-        let object_path = self.get_object_path(volume, path)?;
-        let namespace_owner: Option<Arc<dyn Send + Sync>> = Some(os::acquire_metadata_mutation_lease(&object_path, None).await);
-        self.write_metadata_with_namespace_owner(volume, path, fi, namespace_owner)
-            .await
+        self.write_metadata_with_namespace_owner(volume, path, fi, None).await
     }
 
     #[tracing::instrument(level = "trace", skip_all)]
@@ -11226,176 +11027,6 @@ mod test {
             (disk, dir)
         }
 
-        #[tokio::test]
-        async fn delete_pruning_stops_at_live_metadata_below_a_guarded_ancestor() {
-            // Tuple fields drop in order, releasing the disk's root handle before the temporary directory.
-            let fixture = new_disk().await;
-            let (disk, _dir) = &fixture;
-            let base = disk.get_bucket_path(RUSTFS_META_BUCKET).expect("resolve metadata volume");
-            let shared = base.join("buckets");
-            let guard = Arc::new(
-                os::mkdir_all_below_existing_base_std(&shared, &base, &disk.publication_root)
-                    .expect("retain the shared publication directory"),
-            );
-
-            for owned in [false, true] {
-                for missing_backup in [false, true] {
-                    let transaction = Uuid::new_v4();
-                    let object = shared.join(".bloomcycle.bin");
-                    let rollback = object.join(transaction.to_string());
-                    let metadata = object.join(STORAGE_FORMAT_FILE);
-                    let backup = rollback.join(STORAGE_FORMAT_FILE_BACKUP);
-                    fs::create_dir_all(&rollback).await.expect("create rollback directory");
-                    fs::write(&metadata, b"committed metadata")
-                        .await
-                        .expect("write live metadata");
-                    if !missing_backup {
-                        fs::write(&backup, b"old metadata").await.expect("write rollback backup");
-                    }
-                    let owner: Option<Arc<dyn Send + Sync>> = if owned { Some(guard.clone()) } else { None };
-                    let result = disk
-                        .delete_with_namespace_owner(
-                            RUSTFS_META_BUCKET,
-                            &format!("buckets/.bloomcycle.bin/{transaction}/{STORAGE_FORMAT_FILE_BACKUP}"),
-                            DeleteOptions::default(),
-                            owner,
-                        )
-                        .await;
-
-                    assert!(!backup.exists(), "backup must be absent, owned={owned}, missing={missing_backup}");
-                    assert!(!rollback.exists(), "empty rollback directory must be pruned");
-                    assert_eq!(fs::read(&metadata).await.expect("read committed metadata"), b"committed metadata");
-                    result.expect("a nonempty object must stop pruning before the guarded ancestor");
-                }
-            }
-        }
-
-        #[tokio::test]
-        async fn delete_pruning_removes_empty_and_missing_ancestors_but_keeps_the_volume() {
-            let fixture = new_disk().await;
-            let (disk, _dir) = &fixture;
-            ensure_test_volume(disk, "pruning").await;
-            let base = disk.get_bucket_path("pruning").expect("resolve test volume");
-
-            for missing in [false, true] {
-                let parent = base.join("parent");
-                let rollback = parent.join("object/transaction");
-                fs::create_dir_all(&rollback).await.expect("create empty ancestor chain");
-                let path = if missing {
-                    "parent/object/transaction/missing/xl.meta.bkp"
-                } else {
-                    fs::write(rollback.join(STORAGE_FORMAT_FILE_BACKUP), b"backup")
-                        .await
-                        .expect("create backup");
-                    "parent/object/transaction/xl.meta.bkp"
-                };
-
-                disk.delete("pruning", path, DeleteOptions::default())
-                    .await
-                    .expect("empty and missing ancestors should be pruned");
-                assert!(!parent.exists(), "the whole empty chain should be removed");
-                assert!(base.is_dir(), "pruning must stop at the volume boundary");
-            }
-        }
-
-        #[tokio::test]
-        async fn delete_pruning_does_not_remove_the_base_or_an_outside_path() {
-            let fixture = new_disk().await;
-            let (disk, dir) = &fixture;
-            let base = dir.path().join("base");
-            let outside = dir.path().join("outside");
-            fs::create_dir(&base).await.expect("create base");
-            fs::write(&outside, b"outside data").await.expect("create outside file");
-
-            disk.delete_file(&base, &base, false, false)
-                .await
-                .expect("base path is protected");
-            disk.delete_file(&base, &outside, false, false)
-                .await
-                .expect("outside path is protected");
-            assert!(base.is_dir(), "the base must not be removed even when empty");
-            assert_eq!(fs::read(&outside).await.expect("read outside file"), b"outside data");
-        }
-
-        #[cfg(windows)]
-        #[tokio::test]
-        async fn delete_pruning_propagates_a_locked_backup_error() {
-            use std::os::windows::fs::OpenOptionsExt;
-            use windows_sys::Win32::{Foundation::ERROR_SHARING_VIOLATION, Storage::FileSystem::FILE_SHARE_READ};
-
-            let fixture = new_disk().await;
-            let (disk, _dir) = &fixture;
-            ensure_test_volume(disk, "pruning").await;
-            let base = disk.get_bucket_path("pruning").expect("resolve test volume");
-            let backup = base.join(STORAGE_FORMAT_FILE_BACKUP);
-            fs::write(&backup, b"backup").await.expect("write backup");
-            let guard = std::fs::OpenOptions::new()
-                .read(true)
-                .share_mode(FILE_SHARE_READ)
-                .open(&backup)
-                .expect("hold the backup without delete sharing");
-
-            let err = disk
-                .delete("pruning", STORAGE_FORMAT_FILE_BACKUP, DeleteOptions::default())
-                .await
-                .expect_err("a genuine target-file deletion failure must propagate");
-            let DiskError::Io(err) = err else {
-                panic!("expected contextual I/O error, got {err:?}");
-            };
-            let context = err
-                .get_ref()
-                .and_then(|err| err.downcast_ref::<FileAccessDeniedWithContext>())
-                .expect("preserve the failing path and original OS error");
-            assert_eq!(context.path, backup);
-            assert_eq!(
-                context.source.raw_os_error(),
-                Some(i32::try_from(ERROR_SHARING_VIOLATION).expect("OS code fits"))
-            );
-            assert_eq!(fs::read(&backup).await.expect("backup remains readable"), b"backup");
-            drop(guard);
-        }
-
-        #[cfg(windows)]
-        #[tokio::test]
-        async fn delete_pruning_propagates_a_locked_empty_parent_error() {
-            use windows_sys::Win32::Foundation::ERROR_SHARING_VIOLATION;
-
-            let fixture = new_disk().await;
-            let (disk, _dir) = &fixture;
-            ensure_test_volume(disk, "pruning").await;
-            let base = disk.get_bucket_path("pruning").expect("resolve test volume");
-            let parent = base.join("parent");
-            let guard = os::mkdir_all_below_existing_base_std(&parent, &base, &disk.publication_root)
-                .expect("retain an empty parent without delete sharing");
-            let backup = parent.join(STORAGE_FORMAT_FILE_BACKUP);
-            fs::write(&backup, b"backup").await.expect("write backup");
-
-            let err = disk
-                .delete("pruning", "parent/xl.meta.bkp", DeleteOptions::default())
-                .await
-                .expect_err("a real parent failure without a nonempty boundary must still propagate");
-            let DiskError::Io(err) = err else {
-                panic!("expected contextual I/O error, got {err:?}");
-            };
-            let context = err
-                .get_ref()
-                .and_then(|err| err.downcast_ref::<FileAccessDeniedWithContext>())
-                .expect("preserve parent failure context");
-            assert_eq!(context.path, parent);
-            assert_eq!(
-                context.source.raw_os_error(),
-                Some(i32::try_from(ERROR_SHARING_VIOLATION).expect("OS code fits"))
-            );
-            assert!(!backup.exists(), "the target was removed before the parent error");
-            assert!(parent.is_dir(), "the guarded parent remains");
-            drop(guard);
-            disk.delete("pruning", "parent/xl.meta.bkp", DeleteOptions::default())
-                .await
-                .expect("pruning should succeed once the actual guard is released");
-            assert!(!parent.exists());
-            assert!(base.is_dir());
-        }
-
         // #948: a genuinely missing source is benign and must still return Ok.
         #[tokio::test]
         async fn windows_and_unix_move_to_trash_missing_source_is_ok() {
@@ -11879,59 +11510,14 @@ mod test {
     /// stale deterministically, instead of sleeping and hoping the filesystem
     /// timestamp granularity (or a backward wall-clock step) cooperates.
     fn backdate_mtime(path: &Path, age: Duration) {
-        use std::fs::{FileTimes, OpenOptions};
+        use std::fs::{File, FileTimes};
         let mtime = std::time::SystemTime::now() - age;
-        let mut options = OpenOptions::new();
-        options.read(true);
-        #[cfg(windows)]
-        {
-            use std::os::windows::fs::OpenOptionsExt;
-            use windows_sys::Win32::Storage::FileSystem::{FILE_FLAG_BACKUP_SEMANTICS, FILE_WRITE_ATTRIBUTES};
-
-            // Directories need backup semantics, and changing mtime needs attribute-write access.
-            options
-                .access_mode(FILE_WRITE_ATTRIBUTES)
-                .custom_flags(FILE_FLAG_BACKUP_SEMANTICS);
-        }
-        options
-            .open(path)
+        File::open(path)
             .expect("path should open to backdate its mtime")
             .set_times(FileTimes::new().set_modified(mtime))
             .expect("mtime should rewind into the past");
     }
 
-    #[test]
-    fn cleanup_tmp_on_startup_backdate_mtime_preserves_files_and_directory_contents() {
-        use std::time::SystemTime;
-
-        let root = tempfile::tempdir().expect("create timestamp fixture root");
-        let directory = root.path().join("directory");
-        let file = directory.join("payload");
-        std::fs::create_dir(&directory).expect("create timestamp fixture directory");
-        std::fs::write(&file, b"unchanged payload").expect("write timestamp fixture payload");
-        let age = Duration::from_secs(60);
-        // Filesystems may round stored timestamps; do not require subsecond precision or sleep.
-        let rounding = Duration::from_secs(2);
-
-        for path in [&file, &directory] {
-            let earliest = SystemTime::now() - age - rounding;
-            backdate_mtime(path, age);
-            let latest = SystemTime::now() - age + rounding;
-            let modified = std::fs::metadata(path)
-                .expect("read backdated path metadata")
-                .modified()
-                .expect("read backdated modification time");
-            assert!(modified >= earliest && modified <= latest, "mtime must be backdated for {path:?}");
-        }
-
-        let moved = root.path().join("moved");
-        std::fs::rename(&directory, &moved).expect("mtime helper must release its handles before cleanup");
-        assert_eq!(
-            std::fs::read(moved.join("payload")).expect("read preserved payload"),
-            b"unchanged payload"
-        );
-    }
-
     #[tokio::test]
     async fn startup_cleanup_barrier_and_tmp_trash_cleanup_cover_noop_and_delete_paths() {
         use tempfile::tempdir;

crates/ecstore/src/disk/local/commit.rs

@@ -47,43 +47,6 @@ use tokio::fs;
 use tracing::{info, warn};
 use uuid::Uuid;
 
-/// Hold later repair publications after admitting one baseline object. The
-/// fixture arms this on one replacement disk before rejoining the cluster.
-#[cfg(feature = "e2e-test-hooks")]
-async fn wait_for_heal_commit_test_barrier(root: &Path, bucket: &str, object: &str) -> Result<()> {
-    use tokio::io::AsyncWriteExt;
-
-    let barrier = root.join(".rustfs.sys/e2e-heal-commit-barrier");
-    let prefix = match fs::read_to_string(&barrier).await {
-        Ok(prefix) => prefix,
-        Err(error) if error.kind() == ErrorKind::NotFound => return Ok(()),
-        Err(error) => return Err(error.into()),
-    };
-    let key = format!("{bucket}/{object}");
-    if prefix.is_empty() || !key.starts_with(&prefix) {
-        return Ok(());
-    }
-    let admitted = barrier.with_extension("admitted");
-    match fs::OpenOptions::new().write(true).create_new(true).open(&admitted).await {
-        Ok(mut file) => {
-            file.write_all(key.as_bytes()).await?;
-            return Ok(());
-        }
-        Err(error) if error.kind() == ErrorKind::AlreadyExists => {}
-        Err(error) => return Err(error.into()),
-    }
-    let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(120);
-    loop {
-        if !fs::try_exists(&barrier).await? || fs::read_to_string(&admitted).await? == key {
-            return Ok(());
-        }
-        if tokio::time::Instant::now() >= deadline {
-            return Err(std::io::Error::new(ErrorKind::TimedOut, "heal commit test barrier was not released").into());
-        }
-        tokio::time::sleep(std::time::Duration::from_millis(10)).await;
-    }
-}
-
 fn rollback_committed_rename_std(
     dst_file_path: &Path,
     new_data_path: Option<&Path>,
@@ -290,10 +253,6 @@ impl LocalDisk {
         state: &mut RenameDataState,
     ) -> Result<RenameDataResp> {
         crate::hp_guard!("LocalDisk::rename_data");
-        #[cfg(feature = "e2e-test-hooks")]
-        if fi.is_healing() {
-            wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
-        }
         let mut fi = fi;
         // A non-force DeleteBucket must not remove a directory while a local
         // object commit is publishing into it. The peer's empty scan remains
@@ -320,14 +279,11 @@ impl LocalDisk {
             Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?),
             None => None,
         };
-        // Quota admission -> metadata RMW -> namespace/volume publication.
-        let metadata_lease =
-            os::acquire_metadata_mutation_lease(&self.get_object_path(dst_volume, dst_path)?, state.namespace_owner.take()).await;
         let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
             &self.root,
             dst_volume,
             &destination_object_path,
-            Some(metadata_lease),
+            state.namespace_owner.take(),
         )
         .await;
         if let Some(claim) = quota_fence_claim {

crates/ecstore/src/disk/mod.rs

@@ -1277,25 +1277,6 @@ pub struct CheckPartsResp {
 pub struct UpdateMetadataOpts {
     pub no_persistence: bool,
     pub replace_user_metadata: bool,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub transition_reconcile: Option<Box<TransitionStateReconcileCondition>>,
-}
-
-/// An exact-copy precondition for the single-version tier repair protocol.
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct TransitionStateReconcileCondition {
-    pub expected_metadata_digest: String,
-    pub unchanged_metadata_digest: String,
-    pub target: rustfs_filemeta::TransitionStateReconcileTarget,
-    pub tier: String,
-    pub topology_generation: String,
-    pub verify_only: bool,
-    /// Local ownership is never accepted from the wire. A remote disk acquires
-    /// its own fleet and backend leases before entering the mutation domain.
-    #[serde(skip)]
-    pub(crate) authority:
-        Option<Arc<crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority>>,
 }
 
 pub struct DiskLocation {

crates/ecstore/src/disk/os.rs

@@ -452,13 +452,13 @@ pub(crate) mod windows_rename_test_hooks {
 
 /// Test-only hooks into the destination-parent walk of rename preparation.
 ///
-/// Pruning and Windows sharing races live between syscalls inside
+/// The prune race lives between two syscalls inside
 /// [`mkdir_all_below_existing_base_std`], so only an injection at that exact
 /// point reproduces it deterministically. Hooks are keyed by the absolute path
 /// of the component just opened and queued per path: a retrying preparation
 /// visits the same component again, so a test models a pruner that keeps
 /// walking upward by queueing one hook per visit.
-#[cfg(all(test, any(unix, windows)))]
+#[cfg(all(test, unix))]
 pub(crate) mod prepare_rename_test_hooks {
     use super::*;
 
@@ -1449,38 +1449,6 @@ fn disk_namespace_mutation_lock(path: &Path) -> Arc<NamespaceMutationLock> {
     lock
 }
 
-static DISK_METADATA_MUTATION_LOCKS: LazyLock<Mutex<NamespaceMutationLockRegistry>> =
-    LazyLock::new(|| Mutex::new(HashMap::new()));
-
-/// Serializes the complete xl.meta read/modify/commit transaction. This domain
-/// precedes namespace/volume publication locks, whose narrower syscall leases
-/// may retain it after cancellation of the async caller.
-pub(crate) struct MetadataMutationLease {
-    _guard: OwnedMutexGuard<()>,
-    _owner: Option<Arc<dyn Send + Sync>>,
-}
-
-pub(crate) async fn acquire_metadata_mutation_lease(
-    object: &Path,
-    owner: Option<Arc<dyn Send + Sync>>,
-) -> Arc<MetadataMutationLease> {
-    let lock = {
-        let mut locks = DISK_METADATA_MUTATION_LOCKS.lock();
-        locks.retain(|_, lock| lock.strong_count() > 0);
-        if let Some(lock) = locks.get(object).and_then(Weak::upgrade) {
-            lock
-        } else {
-            let lock = Arc::new(AsyncMutex::new(()));
-            locks.insert(object.to_path_buf(), Arc::downgrade(&lock));
-            lock
-        }
-    };
-    Arc::new(MetadataMutationLease {
-        _guard: lock.lock_owned().await,
-        _owner: owner,
-    })
-}
-
 /// Keeps a namespace transaction serialized even when its async waiter is
 /// cancelled while a blocking filesystem call is still running.
 pub(crate) struct NamespaceMutationLease {
@@ -2144,41 +2112,6 @@ pub(crate) async fn rename_all_with_lease(
     Ok(())
 }
 
-/// Publish a conditional repair and sync its directory in one owned executor.
-/// Cancellation cannot release its metadata/fleet/tier leases between rename
-/// and fsync. The last authority check runs after destination preparation.
-pub(crate) async fn rename_reconciled_metadata(
-    source: PathBuf,
-    destination: PathBuf,
-    base_dir: PathBuf,
-    publication_root: PublicationRoot,
-    owner: Option<Arc<dyn Send + Sync>>,
-    authority: Arc<crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority>,
-) -> Result<()> {
-    let lease = acquire_namespace_mutation_lease_with_owner(&destination, owner).await;
-    run_blocking_namespace_operation(lease, move || {
-        let preparation = prepare_rename_with_retry(&source, &destination, &base_dir, &publication_root)?;
-        #[cfg(all(any(test, feature = "test-util"), not(windows)))]
-        prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &destination);
-        if !authority.is_current() {
-            return Err(io::Error::new(io::ErrorKind::WouldBlock, "transition reconciliation authority expired"));
-        }
-        rename_prepared(&source, &destination, &preparation)?;
-        if let Some(parent) = destination.parent() {
-            fsync_dir_std(parent)?;
-        }
-        Ok(())
-    })
-    .await
-    .map_err(|err| {
-        if err.kind() == io::ErrorKind::WouldBlock {
-            DiskError::OutdatedXLMeta
-        } else {
-            to_file_error(err).into()
-        }
-    })
-}
-
 #[cfg(windows)]
 #[tracing::instrument(level = "debug", skip_all)]
 pub(crate) async fn rename_all_with_commit_guard(
@@ -4275,14 +4208,11 @@ pub(crate) fn mkdir_all_below_existing_base_std(
         let mut handles = Vec::with_capacity(capacity);
         handles.push(publication_root.directory.clone());
         let mut guard = ExistingBaseDirectoryGuard::new(handles);
-        let mut components = base_relative
-            .components()
-            .map(|component| (component, FILE_OPEN))
-            .chain(relative.components().map(|component| (component, FILE_OPEN_IF)))
-            .filter(|(component, _)| !matches!(component, Component::CurDir))
-            .peekable();
-        while let Some((component, disposition)) = components.next() {
+        for component in base_relative.components() {
             let Component::Normal(component) = component else {
+                if matches!(component, Component::CurDir) {
+                    continue;
+                }
                 return Err(io::Error::new(
                     io::ErrorKind::InvalidInput,
                     "rename base directory contains an invalid path component",
@@ -4292,22 +4222,40 @@ pub(crate) fn mkdir_all_below_existing_base_std(
                 .handles
                 .last()
                 .ok_or_else(|| io::Error::other("Windows publication root guard is empty"))?;
-            // The kernel opens the final parent for write during a relative
-            // rename. Share writes from its first open: a temporary read-only
-            // share would block another rename into the same trash directory.
-            // Ancestors stay strict and no handle shares delete access, keeping
-            // every retained directory identity pinned.
-            let share_access = if components.peek().is_none() {
-                FILE_SHARE_READ | FILE_SHARE_WRITE
-            } else {
-                FILE_SHARE_READ
-            };
-            let child = open_windows_relative_directory_component(parent, component, disposition, share_access)?;
+            let child = open_windows_directory_component(parent, component, FILE_OPEN)?;
             guard.handles.push(child);
-            #[cfg(test)]
-            if components.peek().is_none() {
-                prepare_rename_test_hooks::run_after_component_opened(dir_path);
-            }
+        }
+        for component in relative.components() {
+            let Component::Normal(component) = component else {
+                continue;
+            };
+            let parent = guard
+                .handles
+                .last()
+                .ok_or_else(|| io::Error::other("Windows base directory guard is empty"))?;
+            let child = open_windows_directory_component(parent, component, FILE_OPEN_IF)?;
+            guard.handles.push(child);
+        }
+
+        // Windows resolves a handle-relative rename by opening the target for
+        // write. Keep every ancestor strict, but let that internal open share
+        // the final parent. Delete sharing remains omitted, so the retained
+        // directory entry cannot be renamed or removed during publication.
+        if guard.handles.len() > 1 {
+            let component = dir_path
+                .file_name()
+                .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination parent must have a name"))?;
+            let parent_index = guard.handles.len() - 2;
+            let parent = guard
+                .handles
+                .get(parent_index)
+                .ok_or_else(|| io::Error::other("Windows destination guard lost its parent handle"))?;
+            let rename_parent =
+                open_windows_relative_directory_component(parent, component, FILE_OPEN, FILE_SHARE_READ | FILE_SHARE_WRITE)?;
+            *guard
+                .handles
+                .last_mut()
+                .ok_or_else(|| io::Error::other("Windows destination guard is empty"))? = rename_parent;
         }
 
         Ok(guard)
@@ -5498,111 +5446,6 @@ mod tests {
         assert_eq!(std::fs::read(dst).expect("read committed metadata"), b"metadata");
     }
 
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn rename_all_concurrent_trash_renames_remove_every_rollback_directory() {
-        let temp_dir = tempdir().expect("create temp dir");
-        let trash = temp_dir.path().join(".rustfs.sys/tmp/.trash");
-        std::fs::create_dir_all(&trash).expect("create trash directory");
-        let publication_root = PublicationRoot::new(temp_dir.path()).expect("open publication root");
-        let sources: Vec<_> = (0..16)
-            .map(|index| {
-                let source = temp_dir
-                    .path()
-                    .join(format!("bucket/{index}.mp4"))
-                    .join(uuid::Uuid::new_v4().to_string());
-                std::fs::create_dir_all(&source).expect("create rollback directory");
-                std::fs::write(source.join("xl.meta.bkp"), b"rollback metadata").expect("write metadata backup");
-                source
-            })
-            .collect();
-
-        let results = futures::future::join_all(sources.iter().enumerate().map(|(index, source)| {
-            super::rename_all_ignore_missing_source(source, trash.join(index.to_string()), &trash, &publication_root)
-        }))
-        .await;
-
-        for (index, (source, result)) in sources.iter().zip(results).enumerate() {
-            result.expect("concurrent rollback cleanup must reach the shared trash directory");
-            assert!(!source.exists(), "rollback cleanup must not leave a directory in the bucket");
-            assert_eq!(
-                std::fs::read(trash.join(index.to_string()).join("xl.meta.bkp")).expect("read moved metadata backup"),
-                b"rollback metadata",
-                "trash staging must retain the complete backup"
-            );
-        }
-    }
-
-    #[cfg(windows)]
-    #[test]
-    fn windows_trash_rename_succeeds_during_concurrent_parent_preparation() {
-        use std::os::windows::fs::OpenOptionsExt;
-        use std::sync::atomic::{AtomicBool, Ordering};
-        use windows_sys::Win32::{
-            Foundation::{ERROR_SHARING_VIOLATION, GENERIC_WRITE},
-            Storage::FileSystem::{
-                DELETE, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_DELETE, FILE_SHARE_READ,
-                FILE_SHARE_WRITE,
-            },
-        };
-
-        // Exercise the final parent both in the existing base walk and in the
-        // creatable suffix walk. Neither may briefly deny write sharing.
-        for nested_parent in [false, true] {
-            let temp_dir = tempdir().expect("create temp dir");
-            let tmp = temp_dir.path().join(".rustfs.sys/tmp");
-            let trash = tmp.join(".trash");
-            std::fs::create_dir_all(&trash).expect("create trash directory");
-            let base = if nested_parent { &tmp } else { &trash };
-            let publication_root = PublicationRoot::new(temp_dir.path()).expect("open publication root");
-            let sources = ["first", "second"].map(|name| {
-                let source = temp_dir
-                    .path()
-                    .join("bucket")
-                    .join(name)
-                    .join(uuid::Uuid::new_v4().to_string());
-                std::fs::create_dir_all(&source).expect("create rollback directory");
-                std::fs::write(source.join("xl.meta.bkp"), name.as_bytes()).expect("write metadata backup");
-                source
-            });
-            let destinations = [trash.join("first"), trash.join("second")];
-            let first_preparation = prepare_rename_with_retry(&sources[0], &destinations[0], base, &publication_root)
-                .expect("prepare the first trash rename");
-            let first_source = sources[0].clone();
-            let first_destination = destinations[0].clone();
-            let interleaved = Arc::new(AtomicBool::new(false));
-            let interleaved_hook = Arc::clone(&interleaved);
-            prepare_rename_test_hooks::queue_after_component_opened(&trash, move || {
-                interleaved_hook.store(true, Ordering::Release);
-                rename_prepared(&first_source, &first_destination, &first_preparation)
-                    .expect("another preparation's first parent handle must allow the pending trash rename");
-            });
-
-            let second_preparation = prepare_rename_with_retry(&sources[1], &destinations[1], base, &publication_root)
-                .expect("prepare the second trash rename");
-            assert!(interleaved.load(Ordering::Acquire), "the competing parent-open window must be exercised");
-            for (path, access) in [(&tmp, GENERIC_WRITE), (&trash, DELETE)] {
-                let err = std::fs::OpenOptions::new()
-                    .access_mode(access)
-                    .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
-                    .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
-                    .open(path)
-                    .expect_err("ancestor writes and final-parent deletion must remain excluded");
-                assert_eq!(
-                    err.raw_os_error(),
-                    Some(i32::try_from(ERROR_SHARING_VIOLATION).expect("Windows error code must fit i32"))
-                );
-            }
-            rename_prepared(&sources[1], &destinations[1], &second_preparation).expect("publish the second trash rename");
-            for (index, payload) in [b"first".as_slice(), b"second".as_slice()].into_iter().enumerate() {
-                assert!(!sources[index].exists(), "both rollback directories must leave the bucket");
-                assert_eq!(
-                    std::fs::read(destinations[index].join("xl.meta.bkp")).expect("read the staged backup"),
-                    payload
-                );
-            }
-        }
-    }
-
     #[cfg(windows)]
     #[tokio::test]
     async fn windows_rename_all_supports_same_parent_publication() {

crates/ecstore/src/runtime/sources.rs

@@ -14,7 +14,7 @@
 
 use std::{
     collections::{HashMap, HashSet},
-    sync::{Arc, LazyLock, OnceLock, RwLock as StdRwLock},
+    sync::{Arc, OnceLock},
     time::SystemTime,
 };
 
@@ -57,13 +57,6 @@ use uuid::Uuid;
 const TEST_RPC_SECRET: &str = "test-rpc-secret";
 
 pub(crate) type WorkloadSnapshotProviderRef = Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>;
-pub type ScannerDirtyUsageMutationObserver = Arc<dyn Fn(&str, &str, ScannerDirtyUsageMutationSource) + Send + Sync + 'static>;
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum ScannerDirtyUsageMutationSource {
-    Replication,
-    TierExpiration,
-}
 
 #[derive(Clone, Default)]
 pub(crate) struct LockRegistry {
@@ -95,8 +88,6 @@ impl LockRegistry {
 }
 
 static WORKLOAD_ADMISSION_SNAPSHOT_PROVIDER: OnceLock<WorkloadSnapshotProviderRef> = OnceLock::new();
-static SCANNER_DIRTY_USAGE_MUTATION_OBSERVER: LazyLock<StdRwLock<Option<ScannerDirtyUsageMutationObserver>>> =
-    LazyLock::new(|| StdRwLock::new(None));
 
 pub(crate) fn set_workload_admission_snapshot_provider(
     provider: WorkloadSnapshotProviderRef,
@@ -108,28 +99,6 @@ pub(crate) fn workload_admission_snapshot_provider() -> Option<WorkloadSnapshotP
     WORKLOAD_ADMISSION_SNAPSHOT_PROVIDER.get().cloned()
 }
 
-pub fn set_scanner_dirty_usage_mutation_observer(
-    observer: Option<ScannerDirtyUsageMutationObserver>,
-) -> Option<ScannerDirtyUsageMutationObserver> {
-    let mut slot = SCANNER_DIRTY_USAGE_MUTATION_OBSERVER
-        .write()
-        .unwrap_or_else(|poisoned| poisoned.into_inner());
-    std::mem::replace(&mut *slot, observer)
-}
-
-pub(crate) fn notify_scanner_dirty_usage_mutation(bucket: &str, object: &str, source: ScannerDirtyUsageMutationSource) {
-    if bucket.is_empty() || object.is_empty() {
-        return;
-    }
-    let observer = SCANNER_DIRTY_USAGE_MUTATION_OBSERVER
-        .read()
-        .unwrap_or_else(|poisoned| poisoned.into_inner())
-        .clone();
-    if let Some(observer) = observer {
-        observer(bucket, object, source);
-    }
-}
-
 pub(crate) fn record_erasure_write_quorum_failure(stage: &'static str, dominant_error: &'static str) {
     global_internode_metrics().record_erasure_write_quorum_failure(stage, dominant_error);
 }
@@ -611,16 +580,12 @@ pub(crate) async fn init_tier_config_mgr(store: Arc<ECStore>) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::{
-        LockRegistry, ScannerDirtyUsageMutationSource, clear_local_disk_id_map_for_test, local_disk_path_by_id, local_node_name,
-        notify_scanner_dirty_usage_mutation, reconcile_local_disk_ids, replace_local_disk_id, set_local_node_name,
-        set_scanner_dirty_usage_mutation_observer,
+        LockRegistry, clear_local_disk_id_map_for_test, local_disk_path_by_id, local_node_name, reconcile_local_disk_ids,
+        replace_local_disk_id, set_local_node_name,
     };
     use crate::disk::endpoint::Endpoint;
     use rustfs_lock::{LocalClient, LockClient};
-    use std::{
-        collections::HashMap,
-        sync::{Arc, Mutex},
-    };
+    use std::{collections::HashMap, sync::Arc};
     use uuid::Uuid;
 
     fn url_endpoint(raw: &str) -> Endpoint {
@@ -655,42 +620,6 @@ mod tests {
         assert!(Arc::ptr_eq(&clients[1], &client_b));
     }
 
-    #[test]
-    #[serial_test::serial(scanner_dirty_usage_mutation_observer)]
-    fn scanner_dirty_usage_mutation_observer_filters_empty_identity_and_preserves_source() {
-        let observed = Arc::new(Mutex::new(Vec::new()));
-        let observed_clone = Arc::clone(&observed);
-        let previous = set_scanner_dirty_usage_mutation_observer(Some(Arc::new(move |bucket, object, source| {
-            observed_clone.lock().expect("observer lock should not be poisoned").push((
-                bucket.to_string(),
-                object.to_string(),
-                source,
-            ));
-        })));
-
-        notify_scanner_dirty_usage_mutation("photos", "2026/image.jpg", ScannerDirtyUsageMutationSource::Replication);
-        notify_scanner_dirty_usage_mutation("", "2026/empty-bucket.jpg", ScannerDirtyUsageMutationSource::TierExpiration);
-        notify_scanner_dirty_usage_mutation("photos", "", ScannerDirtyUsageMutationSource::TierExpiration);
-        notify_scanner_dirty_usage_mutation("archive", "expired.bin", ScannerDirtyUsageMutationSource::TierExpiration);
-        set_scanner_dirty_usage_mutation_observer(previous);
-
-        assert_eq!(
-            *observed.lock().expect("observer lock should not be poisoned"),
-            vec![
-                (
-                    "photos".to_string(),
-                    "2026/image.jpg".to_string(),
-                    ScannerDirtyUsageMutationSource::Replication
-                ),
-                (
-                    "archive".to_string(),
-                    "expired.bin".to_string(),
-                    ScannerDirtyUsageMutationSource::TierExpiration
-                ),
-            ]
-        );
-    }
-
     #[tokio::test]
     #[serial_test::serial]
     async fn local_node_name_round_trips_through_common_runtime_helper() {

crates/ecstore/src/services/notification_sys.rs

@@ -67,9 +67,12 @@ const DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
 // Keep this synchronized with the version served by node_service. Including
 // the local member in the minimum prevents an older coordinator from
 // self-authorizing a policy implemented only by newer remote peers.
-const LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION: u32 = 5;
-/// Version 5 preserves explicit transition state/destination bindings and
-/// supports exact-generation metadata repair with strong all-copy readback.
+const LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
+/// Version 5 is reserved for a fleet whose every metadata writer preserves
+/// explicit transition version state and destination identity, and implements
+/// conditional per-generation `xl.meta` writes with strong readback. The node
+/// service must not advertise this version until the conditional writer from
+/// rustfs/backlog#684 is available.
 const LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION: u32 = 5;
 
 fn resolve_admin_peer_probe_timeout_secs(configured: Option<u64>) -> u64 {
@@ -565,10 +568,6 @@ pub(crate) fn tier_delete_journal_topology_generation(proof: &TierDeleteJournalF
     stable_tier_delete_journal_topology_generation(&proof.token.topology_fingerprint)
 }
 
-pub(crate) fn cross_pool_fence_topology_generation(proof: &CrossPoolFenceFleetProofToken) -> String {
-    stable_tier_delete_journal_topology_generation(&proof.0.topology_fingerprint)
-}
-
 /// Acquire one non-cloneable authority that must span the complete reconcile
 /// effect window, including its final strong readback.
 pub async fn acquire_legacy_transition_state_reconcile_fleet_proof() -> Option<LegacyTransitionStateReconcileFleetProofToken> {
@@ -604,10 +603,6 @@ fn acquire_legacy_transition_state_reconcile_fleet_proof_from(
 }
 
 async fn observe_legacy_transition_state_reconcile_fleet(expected_topology: &str) -> Option<BTreeMap<String, Uuid>> {
-    #[cfg(all(test, feature = "test-util"))]
-    if let Ok(observation) = LEGACY_RECONCILE_TEST_OBSERVATION.try_with(Clone::clone) {
-        return Some(observation);
-    }
     let notification_sys = get_global_notification_sys()?;
     let (peer_epochs, minimum_version) = timeout(
         REMOTE_VERSION_STATE_PROBE_TIMEOUT,
@@ -620,34 +615,6 @@ async fn observe_legacy_transition_state_reconcile_fleet(expected_topology: &str
     reconcile_result.ok()
 }
 
-#[cfg(all(test, feature = "test-util"))]
-tokio::task_local! {
-    static LEGACY_RECONCILE_TEST_OBSERVATION: BTreeMap<String, Uuid>;
-}
-
-#[cfg(all(test, feature = "test-util"))]
-pub(crate) async fn with_legacy_transition_state_fleet_proof_for_test<F: std::future::Future>(future: F) -> F::Output {
-    struct Revoke;
-    impl Drop for Revoke {
-        fn drop(&mut self) {
-            revoke_fleet_capability_proof(legacy_transition_state_reconcile_fleet_proof_slot());
-        }
-    }
-    let topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get().expect("test store topology");
-    assert!(
-        publish_fleet_capability_probe_result(
-            legacy_transition_state_reconcile_fleet_proof_slot(),
-            topology,
-            Ok(BTreeMap::new()),
-            Instant::now(),
-        )
-        .is_none()
-    );
-    let _revoke = Revoke;
-    let _remote_version = install_current_remote_version_state_fleet_proof_for_test();
-    LEGACY_RECONCILE_TEST_OBSERVATION.scope(BTreeMap::new(), future).await
-}
-
 /// Revalidate the exact fleet generation captured by a reconcile token with a
 /// fresh synchronous observation. Callers must await this before each
 /// conditional metadata write and after the final strong readback.
@@ -666,18 +633,6 @@ pub async fn legacy_transition_state_reconcile_fleet_proof_matches(
     .await
 }
 
-/// Final local check in the disk publication executor. The corresponding
-/// counted permit remains owned until the filesystem operation has drained.
-pub(crate) fn legacy_transition_state_reconcile_fleet_proof_current(
-    proof: &LegacyTransitionStateReconcileFleetProofToken,
-) -> bool {
-    let Some(topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else { return false };
-    let state = legacy_transition_state_reconcile_fleet_proof_slot()
-        .read()
-        .unwrap_or_else(std::sync::PoisonError::into_inner);
-    legacy_transition_state_reconcile_fleet_proof_matches_at(&state, proof, topology, Instant::now())
-}
-
 pub async fn acquire_ilm_recovery_export_fleet_proof() -> Option<IlmRecoveryExportFleetProofToken> {
     let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
     let proof = {
@@ -1165,14 +1120,6 @@ pub(crate) fn install_remote_version_state_fleet_proof_for_test(topology_fingerp
     RemoteVersionStateFleetProofGuard
 }
 
-#[cfg(all(test, feature = "test-util"))]
-pub(crate) fn install_current_remote_version_state_fleet_proof_for_test() -> RemoteVersionStateFleetProofGuard {
-    let topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY
-        .get()
-        .expect("the test store must bind its fleet topology before installing a writer proof");
-    install_remote_version_state_fleet_proof_for_test(topology)
-}
-
 #[cfg(all(test, feature = "test-util"))]
 pub(crate) struct TransitionTransactionCompactionFleetProofGuard;
 
@@ -3967,11 +3914,15 @@ mod tests {
         assert!(decommission_v3.is_err(), "v3 members do not understand the per-target decommission fence");
         assert!(reconcile_v3.is_err());
 
-        let (generic_v4, journal_v4, decommission_v4, reconcile_v4) = cross_pool_fence_policy_results(peers.clone(), 4);
+        let (generic_v4, journal_v4, decommission_v4, reconcile_v4) =
+            cross_pool_fence_policy_results(peers.clone(), LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION);
         assert!(generic_v4.is_ok());
         assert!(journal_v4.is_ok());
         assert!(decommission_v4.is_ok(), "an all-v4 fleet may create sticky per-target reservations");
-        assert!(reconcile_v4.is_err(), "v4 does not support conditional transition metadata writes");
+        assert!(
+            reconcile_v4.is_err(),
+            "the current local policy lacks the conditional xl.meta writer required by reconcile"
+        );
 
         let (generic_v5, journal_v5, decommission_v5, reconcile_v5) = cross_pool_fence_policy_results(peers, 5);
         assert!(generic_v5.is_ok());
@@ -4649,7 +4600,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn legacy_transition_state_reconcile_single_node_advertises_conditional_writer() {
+    async fn legacy_transition_state_reconcile_single_node_stays_closed_before_local_cas_support() {
         let notification_sys = NotificationSys {
             peer_clients: Vec::new(),
             all_peer_clients: vec![None],
@@ -4665,8 +4616,8 @@ mod tests {
         assert_eq!(minimum_version, LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION);
         let (_, _, _, reconcile_result) = cross_pool_fence_policy_results(peers, minimum_version);
         assert!(
-            reconcile_result.is_ok(),
-            "the current node implements the conditional writer and preserves repaired bindings"
+            reconcile_result.is_err(),
+            "the current node must not self-authorize reconcile before the conditional writer lands"
         );
     }
 

crates/ecstore/src/services/rebalance/control.rs

@@ -572,7 +572,7 @@ impl ECStore {
     where
         S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
     {
-        // Lock order: pool_meta_save_gate -> rebalance.bin -> pool.bin.
+        // Lock order: pool_meta_save_gate -> pool.bin -> rebalance.bin.
         let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
         pool_meta_guard.ensure_write_safe("rebalance worker activation")?;
         // Classify the durable rebalance record while holding both namespace

crates/ecstore/src/services/rebalance/entry.rs

@@ -50,11 +50,6 @@ fn ensure_rebalance_entry_active(cancel: &CancellationToken) -> Result<()> {
     Ok(())
 }
 
-#[cfg(test)]
-tokio::task_local! {
-    static REBALANCE_ENTRY_RUN_FENCE_BARRIER: (Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>);
-}
-
 #[derive(Debug)]
 struct RebalanceEntryTarget {
     bucket: String,
@@ -261,15 +256,9 @@ impl ECStore {
             .sort_by_key(|v| (v.mod_time.is_none(), std::cmp::Reverse(v.mod_time)));
 
         // Entry lock order is bucket incarnation -> activation_gate -> rebalance.bin -> movement gate.
-        // Target capacity admission can then acquire pool.bin under the run fence.
         // Stop waits for in-flight entries through cleanup, but not for entries admitted later.
         ensure_rebalance_entry_active(&cancel)?;
         let run_guard = self.rebalance_run_guard(rebalance_id.as_ref(), "rebalance entry").await?;
-        #[cfg(test)]
-        if let Ok((arrived, release)) = REBALANCE_ENTRY_RUN_FENCE_BARRIER.try_with(Clone::clone) {
-            arrived.notify_one();
-            release.notified().await;
-        }
         let lock_lost_signal = run_guard.lock_lost_signal();
         #[cfg(test)]
         let _run_signal_test_fence = lock_lost_signal
@@ -1248,130 +1237,6 @@ mod tests {
         assert_eq!(pool_stats.cleanup_warnings.count, 1, "deferred cleanup must not add a permanent warning");
     }
 
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn real_rebalance_entry_progresses_while_peer_activation_waits_for_run_fence() {
-        const REBALANCE_ID: &str = "rebalance-peer-activation-lock-order";
-        let (_temp_dirs, store, peer) = crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(Some(
-            active_rebalance_meta(REBALANCE_ID),
-        ))
-        .await;
-        assert!(!Arc::ptr_eq(&store.ctx, &peer.ctx), "node-local movement gates must be independent");
-        {
-            let mut meta = peer.rebalance_meta.write().await;
-            let meta = meta.as_mut().expect("peer should know the durable run");
-            meta.activation_gate = Arc::default();
-            meta.cancel = None;
-        }
-        let bucket = crate::disk::RUSTFS_META_BUCKET;
-        let object = "rebalance-peer-activation-object";
-        let version_id = uuid::Uuid::new_v4();
-        let payload = b"entry must drain before peer activation takes the pool fence".repeat(1024);
-        let source_set = store.pools[0].get_disks_by_key(object);
-        let target_set = store.pools[1].get_disks_by_key(object);
-        let opts = ObjectOptions {
-            versioned: true,
-            version_id: Some(version_id.to_string()),
-            ..Default::default()
-        };
-        let mut writer = PutObjReader::from_vec(payload.clone());
-        let source_before = source_set
-            .put_object(bucket, object, &mut writer, &opts)
-            .await
-            .expect("source version should be written");
-        let entry = metacache_entry_from_source(&source_set, bucket, object).await;
-        let arrived = Arc::new(tokio::sync::Notify::new());
-        let release = Arc::new(tokio::sync::Notify::new());
-        // JoinSet aborts both scoped tasks if an assertion or timeout fails.
-        let mut tasks = tokio::task::JoinSet::new();
-        let entry_store = Arc::clone(&store);
-        tasks.spawn(
-            REBALANCE_ENTRY_RUN_FENCE_BARRIER.scope((Arc::clone(&arrived), Arc::clone(&release)), async move {
-                entry_store
-                    .rebalance_entry(
-                        RebalanceEntryTarget {
-                            bucket: bucket.to_string(),
-                            pool_index: 0,
-                        },
-                        entry,
-                        source_set,
-                        Arc::new(RebalanceBucketConfigs::default()),
-                        Arc::from(REBALANCE_ID),
-                        CancellationToken::new(),
-                    )
-                    .await
-            }),
-        );
-        tokio::time::timeout(StdDuration::from_secs(30), arrived.notified())
-            .await
-            .expect("real entry must acquire its persisted run read fence");
-
-        let attempted = Arc::new(tokio::sync::Notify::new());
-        let peer_pool = Arc::clone(&peer.pools[0]);
-        let (activation_done, activation_result) = tokio::sync::oneshot::channel();
-        tasks.spawn(
-            crate::core::pools::REBALANCE_ACTIVATION_LOCK_ATTEMPT.scope(Arc::clone(&attempted), async move {
-                let result = peer.fence_rebalance_worker_activation(peer_pool, REBALANCE_ID).await;
-                let result = result.map(|fence| match fence {
-                    super::super::control::RebalanceWorkerActivationFence::Ready(fence) => {
-                        fence.ensure_held().expect("peer activation must retain both fences");
-                    }
-                    super::super::control::RebalanceWorkerActivationFence::NotStartedTerminal => {
-                        panic!("the paused entry's run must still require activation");
-                    }
-                });
-                activation_done.send(result).expect("activation receiver should remain alive");
-                Ok(RebalanceEntryOutcome::Completed)
-            }),
-        );
-        tokio::time::timeout(StdDuration::from_secs(30), attempted.notified())
-            .await
-            .expect("peer activation must attempt the persisted rebalance write fence");
-        release.notify_one();
-
-        tokio::time::timeout(StdDuration::from_secs(30), async {
-            while let Some(result) = tasks.join_next().await {
-                assert!(matches!(
-                    result
-                        .expect("scoped task must not panic")
-                        .expect("entry must not fail or defer"),
-                    RebalanceEntryOutcome::Completed
-                ));
-            }
-        })
-        .await
-        .expect("entry and peer activation must both make progress");
-        activation_result
-            .await
-            .expect("peer activation result should be sent")
-            .expect("peer activation must not time out behind the entry it blocks");
-
-        let mut reader = target_set
-            .get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
-            .await
-            .expect("the exact target version must be readable");
-        let mut actual = Vec::new();
-        reader
-            .stream
-            .read_to_end(&mut actual)
-            .await
-            .expect("target body should drain completely");
-        assert_eq!(actual, payload);
-        assert_eq!(reader.object_info.version_id, source_before.version_id);
-        assert_eq!(reader.object_info.etag, source_before.etag);
-        assert_eq!(reader.object_info.mod_time, source_before.mod_time);
-        let source_error = store.pools[0]
-            .get_object_info(bucket, object, &opts)
-            .await
-            .expect_err("completed entry must clean up the source version");
-        assert!(crate::error::is_err_object_not_found(&source_error) || crate::error::is_err_version_not_found(&source_error));
-        let meta = store.rebalance_meta.read().await;
-        let stats = &meta.as_ref().expect("local run must remain installed").pool_stats[0];
-        assert_eq!(stats.num_objects, 1);
-        assert_eq!(stats.num_versions, 1);
-        assert_eq!(stats.cleanup_warnings.count, 0);
-    }
-
     #[tokio::test]
     #[serial_test::serial]
     async fn real_rebalance_run_fence_loss_before_target_commit_preserves_target_and_source() {

crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs

@@ -1907,124 +1907,6 @@ fn test_is_transient_rebalance_error_accepts_wrapped_disk_timeout() {
     assert!(is_transient_rebalance_error(&Error::Io(std::io::Error::other(DiskError::Timeout))));
 }
 
-#[test]
-fn test_rebalance_stage_wrapped_transient_errors_remain_retryable() {
-    let cases = [
-        Error::Lock(rustfs_lock::LockError::timeout(".rustfs.sys/pool.bin@latest", Duration::from_secs(5))),
-        Error::Lock(rustfs_lock::LockError::network(
-            "peer unavailable",
-            std::io::Error::from(std::io::ErrorKind::ConnectionReset),
-        )),
-        Error::SlowDown,
-        Error::ErasureReadQuorum,
-        Error::ErasureWriteQuorum,
-        Error::Io(std::io::Error::other(DiskError::Timeout)),
-        Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),
-    ];
-    for mut error in cases {
-        for depth in 0..=3 {
-            assert!(is_transient_rebalance_error(&error), "transient source lost at depth {depth}: {error:?}");
-            assert!(
-                should_defer_rebalance_entry_failure(&error),
-                "exhausted transient entries must be deferred"
-            );
-            assert!(should_retry_rebalance_listing(&error, 0, 3));
-            assert!(
-                !should_retry_rebalance_listing(&error, 2, 3),
-                "wrapping must not bypass the attempt limit"
-            );
-            error = data_movement::data_movement_stage_error_for_test(
-                "rebalance_object",
-                "put_object",
-                "bucket",
-                "baseline/00042.bin",
-                error,
-            );
-        }
-    }
-}
-
-#[test]
-fn test_rebalance_stage_wrapped_terminal_errors_remain_terminal() {
-    let cases = [
-        Error::FileAccessDenied,
-        Error::FileCorrupt,
-        Error::OperationCanceled,
-        Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()),
-        Error::Lock(rustfs_lock::LockError::already_locked("bucket/object", "owner")),
-        Error::other("permission denied"),
-    ];
-    for mut error in cases {
-        for depth in 0..=3 {
-            assert!(
-                !is_transient_rebalance_error(&error),
-                "terminal source must survive depth {depth}: {error:?}"
-            );
-            assert!(!should_defer_rebalance_entry_failure(&error));
-            // Object names are untrusted context, not evidence of a transient failure.
-            error = data_movement::data_movement_stage_error_for_test(
-                "rebalance_object",
-                "put_object",
-                "bucket",
-                "remote lock rpc timed out",
-                error,
-            );
-        }
-    }
-}
-
-#[tokio::test]
-async fn test_rebalance_stage_wrapped_lock_timeout_retries_real_migration_loop() {
-    for succeeds_on_retry in [true, false] {
-        let backend = MigrationBackendSpy::new(None, None);
-        let attempts = AtomicUsize::new(0);
-        let waits = AtomicUsize::new(0);
-        let mut transfer = |_, _, _| {
-            let attempt = attempts.fetch_add(1, Ordering::SeqCst);
-            async move {
-                if succeeds_on_retry && attempt > 0 {
-                    return Ok(());
-                }
-                Err(data_movement::data_movement_stage_error_for_test(
-                    "rebalance_object",
-                    "put_object",
-                    "bucket",
-                    "baseline/00042.bin",
-                    Error::Lock(rustfs_lock::LockError::timeout(".rustfs.sys/pool.bin@latest", Duration::from_secs(5))),
-                ))
-            }
-        };
-        let version = version_normal();
-        let result = migrate_entry_version_with_retry_wait(
-            &backend,
-            "bucket".to_string(),
-            0,
-            &version,
-            None,
-            3,
-            false,
-            &mut transfer,
-            |_: String, _: String, _: ObjectOptions| async { Ok::<_, Error>(ObjectInfo::default()) },
-            |_| {
-                waits.fetch_add(1, Ordering::SeqCst);
-                std::future::ready(())
-            },
-        )
-        .await;
-        assert_eq!(result.moved, succeeds_on_retry);
-        assert_eq!(result.failed, !succeeds_on_retry);
-        assert_eq!(attempts.load(Ordering::SeqCst), if succeeds_on_retry { 2 } else { 3 });
-        assert_eq!(backend.get_calls(), attempts.load(Ordering::SeqCst));
-        assert_eq!(waits.load(Ordering::SeqCst), attempts.load(Ordering::SeqCst) - 1);
-        if !succeeds_on_retry {
-            assert_eq!(result.stage, Some("write_target"));
-            assert!(should_defer_rebalance_entry_failure(
-                result.error.as_ref().expect("exhaustion must retain its source error")
-            ));
-        }
-    }
-}
-
 #[test]
 fn test_is_transient_rebalance_error_accepts_io_timeout_message() {
     assert!(is_transient_rebalance_error(&Error::Io(std::io::Error::other("timeout"))));

crates/ecstore/src/services/rebalance/worker.rs

@@ -244,7 +244,6 @@ pub(super) fn resolve_rebalance_bucket_result(
 }
 
 pub(super) fn is_transient_rebalance_error(err: &Error) -> bool {
-    let err = rebalance_error_source(err);
     match err {
         Error::SlowDown
         | Error::ErasureReadQuorum
@@ -257,15 +256,6 @@ pub(super) fn is_transient_rebalance_error(err: &Error) -> bool {
     }
 }
 
-fn rebalance_error_source(mut err: &Error) -> &Error {
-    // Stage context contains object names, so classify the preserved source,
-    // not timeout-like text supplied by an object name. Iterate nested stages.
-    while let Some(source) = crate::data_movement::data_movement_stage_source(err) {
-        err = source;
-    }
-    err
-}
-
 fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
     match err {
         rustfs_lock::LockError::Timeout { .. } | rustfs_lock::LockError::Network { .. } => true,
@@ -319,7 +309,6 @@ pub(super) fn rebalance_listing_retry_delay(attempt: usize) -> Duration {
 }
 
 fn is_rebalance_lock_or_rpc_timeout(err: &Error) -> bool {
-    let err = rebalance_error_source(err);
     match err {
         Error::Lock(rustfs_lock::LockError::Timeout { .. }) | Error::Lock(rustfs_lock::LockError::Network { .. }) => true,
         Error::Io(io_err) => is_rebalance_lock_or_rpc_timeout_message(&io_err.to_string()),
@@ -596,48 +585,3 @@ impl SetDisks {
         Ok(())
     }
 }
-
-#[cfg(test)]
-mod error_source_tests {
-    use super::*;
-
-    #[test]
-    fn stage_wrapped_errors_select_the_source_backoff_policy() {
-        let cases = [
-            (
-                Error::Lock(rustfs_lock::LockError::timeout(".rustfs.sys/pool.bin@latest", Duration::from_secs(5))),
-                true,
-            ),
-            (
-                Error::Lock(rustfs_lock::LockError::network(
-                    "peer unavailable",
-                    std::io::Error::from(std::io::ErrorKind::ConnectionReset),
-                )),
-                true,
-            ),
-            (Error::other("remote lock rpc timed out"), true),
-            (Error::SlowDown, false),
-            (Error::Io(std::io::Error::other(DiskError::Timeout)), false),
-            (Error::FileAccessDenied, false),
-        ];
-        for (mut error, lock_backoff) in cases {
-            for depth in 0..=3 {
-                assert_eq!(
-                    is_rebalance_lock_or_rpc_timeout(&error),
-                    lock_backoff,
-                    "wrong backoff at depth {depth}: {error:?}"
-                );
-                if !lock_backoff {
-                    assert_eq!(rebalance_migration_retry_delay(1, &error), REBALANCE_MIGRATION_RETRY_BASE_DELAY * 2);
-                }
-                error = crate::data_movement::data_movement_stage_error_for_test(
-                    "rebalance_object",
-                    "put_object",
-                    "bucket",
-                    "remote lock rpc timed out",
-                    error,
-                );
-            }
-        }
-    }
-}

crates/ecstore/src/services/tier/test_util.rs

@@ -653,26 +653,6 @@ impl MockWarmBackend {
 
 #[async_trait]
 impl WarmBackend for MockWarmBackend {
-    async fn probe_legacy_metadata(
-        &self,
-        object: &str,
-        remote_version: Option<&str>,
-    ) -> Result<super::warm_backend::LegacyTransitionStateProbe, std::io::Error> {
-        use super::warm_backend::LegacyTransitionStateProbe as Probe;
-        let candidate = match remote_version {
-            Some(version) if !version.is_empty() => self.probe_transition_version(object, version).await?,
-            _ => self.probe_transition_candidate(object).await?,
-        };
-        Ok(match candidate {
-            TransitionCandidateProbe::Missing => Probe::Missing,
-            TransitionCandidateProbe::UnversionedPresent => Probe::UnversionedPresent,
-            TransitionCandidateProbe::VersionedPresent(version) if version == "null" => Probe::SuspendedNullPresent,
-            TransitionCandidateProbe::VersionedPresent(version) => Probe::VersionedPresent(version),
-            TransitionCandidateProbe::Ambiguous => Probe::Ambiguous,
-            TransitionCandidateProbe::Unsupported => Probe::Unsupported,
-        })
-    }
-
     fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> {
         if remote_version_id.is_empty() {
             return Ok(());
@@ -894,9 +874,8 @@ pub async fn register_mock_tier_backend(handle: &Arc<RwLock<TierConfigMgr>>, tie
             ..Default::default()
         },
     );
-    drop(tier_config_mgr);
-    TierConfigMgr::install_test_driver_in(handle, tier_name, Box::new(backend))
-        .await
+    tier_config_mgr
+        .install_test_driver(tier_name, Box::new(backend))
         .expect("mock tier driver should install");
 }
 

crates/ecstore/src/services/tier/tier.rs

@@ -2322,14 +2322,6 @@ struct SharedWarmBackendProxy(SharedWarmBackend);
 
 #[async_trait::async_trait]
 impl WarmBackend for SharedWarmBackendProxy {
-    async fn probe_legacy_metadata(
-        &self,
-        object: &str,
-        remote_version: Option<&str>,
-    ) -> io::Result<crate::services::tier::warm_backend::LegacyTransitionStateProbe> {
-        self.0.probe_legacy_metadata(object, remote_version).await
-    }
-
     async fn validate(&self) -> io::Result<()> {
         self.0.validate().await
     }
@@ -2498,27 +2490,6 @@ impl TierOperationLease {
         self.inner.driver.probe_transition_version(object, remote_version_id).await
     }
 
-    pub(crate) async fn probe_legacy_transition_state(
-        &self,
-        object: &str,
-        remote_version: Option<&str>,
-    ) -> io::Result<crate::services::tier::warm_backend::LegacyTransitionStateProbe> {
-        let Some(reconciler) = self
-            .inner
-            .reconciler
-            .get_or_try_init(|| async {
-                crate::services::tier::warm_backend::new_transition_candidate_reconciler(&self.inner.tier_config)
-                    .await
-                    .map(|reconciler| reconciler.map(Arc::from))
-            })
-            .await
-            .map_err(|err| io::Error::other(err.message))?
-        else {
-            return self.inner.driver.probe_legacy_metadata(object, remote_version).await;
-        };
-        reconciler.probe_legacy_transition_state(object, remote_version).await
-    }
-
     pub(crate) fn is_current_generation(&self) -> bool {
         lock_unpoisoned(&self.runtime)
             .generations
@@ -6042,19 +6013,6 @@ impl TierConfigMgr {
         Ok(())
     }
 
-    #[cfg(any(test, feature = "test-util"))]
-    pub(crate) async fn install_test_driver_in(
-        handle: &Arc<RwLock<Self>>,
-        tier_name: &str,
-        driver: WarmBackendImpl,
-    ) -> std::result::Result<(), AdminError> {
-        let mut manager = handle.write().await;
-        // Register the generation runtime before installing the mock so its
-        // explicit lack of a network reconciler survives the first lease.
-        tier_driver_runtime(handle, &manager);
-        manager.install_test_driver(tier_name, driver)
-    }
-
     #[cfg(any(test, feature = "test-util"))]
     pub(crate) fn install_test_driver(
         &mut self,
@@ -6433,48 +6391,9 @@ impl TierConfigMgr {
     }
 
     pub(crate) async fn refresh_tier_config_handle(handle: Arc<RwLock<Self>>, api: Arc<ECStore>) {
-        Self::refresh_tier_config_handle_with_weak(handle, Arc::downgrade(&api)).await;
+        Self::refresh_tier_config_handle_with(handle, api).await;
     }
 
-    async fn refresh_tier_config_handle_with_weak(handle: Arc<RwLock<Self>>, api: Weak<ECStore>) {
-        // The periodic refresh remains the recovery fallback; committed mutations
-        // notify this worker so a successful peer commit converges immediately.
-        let mutation_refresh = Self::mutation_refresh_notifier(&handle).await;
-        let r = rand::rng().random_range(0.0..1.0);
-        let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64);
-
-        let refresh_interval = TIER_CFG_REFRESH + rand_interval();
-        let mut t = delayed_tier_refresh_interval(refresh_interval);
-        loop {
-            select! {
-                _ = t.tick() => {
-                    let Some(api) = Weak::upgrade(&api) else {
-                        return;
-                    };
-                    if let Err(err) = Self::reload_handle_with(&handle, api).await {
-                        warn!(
-                            event = EVENT_TIER_CONFIG_REFRESH,
-                            component = LOG_COMPONENT_ECSTORE,
-                            subsystem = LOG_SUBSYSTEM_TIER,
-                            trigger = "periodic",
-                            result = "failed",
-                            error = ?err,
-                            "tier configuration refresh"
-                        );
-                    }
-                }
-                _ = mutation_refresh.notified() => {
-                    let Some(api) = Weak::upgrade(&api) else {
-                        return;
-                    };
-                    Self::reload_after_committed_mutation(&handle, api).await;
-                }
-            }
-            t.reset();
-        }
-    }
-
-    #[allow(dead_code, reason = "used by focused tier refresh tests and non-ECStore generic harnesses")]
     pub(crate) async fn refresh_tier_config_handle_with<S>(handle: Arc<RwLock<Self>>, api: Arc<S>)
     where
         S: EcstoreObjectIO
@@ -17579,66 +17498,6 @@ mod tests {
         assert!(current.tiers.contains_key("COLD-B"));
     }
 
-    async fn wait_for_reference_proof_barrier(
-        barrier: &TierDriverBuildBarrier,
-        update: &mut tokio::task::JoinHandle<std::result::Result<(), TierConfigUpdateError>>,
-    ) -> std::result::Result<(), String> {
-        tokio::select! {
-            biased;
-            result = &mut *update => Err(format!("tier update exited before the reference proof barrier: {result:?}")),
-            () = barrier.arrived.notified() => Ok(()),
-            () = tokio::time::sleep(Duration::from_secs(30)) => {
-                // Aborting the caller does not stop its owned mutation task.
-                // Let a late arrival pass the test-only barrier.
-                barrier.release.add_permits(1);
-                update.abort();
-                Err("timed out waiting for the reference proof barrier".to_string())
-            }
-        }
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn reference_proof_barrier_reports_update_failure_before_arrival() {
-        let manager = TierConfigMgr::new();
-        let store = Arc::new(CasConfigStore::default());
-        let mut persisted = empty_mgr();
-        persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
-        persisted
-            .save_tiering_config_if_current(store.clone(), None)
-            .await
-            .expect("early update failure fixture should persist");
-        let barrier = tier_reference_proof_test_barrier();
-        let scoped_barrier = barrier.clone();
-        let factory: TierDriverTestFactory =
-            Arc::new(|_| Err(AdminError::msg("injected driver initialization failure before reference proof")));
-        let mut update = tokio::spawn(async move {
-            TIER_REFERENCE_PROOF_TEST_BARRIER
-                .scope(
-                    scoped_barrier,
-                    TIER_DRIVER_TEST_FACTORY.scope(
-                        factory,
-                        TIER_MUTATION_TEST_PEERS.scope(
-                            Vec::new(),
-                            TierConfigMgr::update_candidate_with_config_lock(
-                                &manager,
-                                store,
-                                TierCandidateMutation::Remove("COLD-A".to_string(), true),
-                            ),
-                        ),
-                    ),
-                )
-                .await
-        });
-
-        let err = tokio::time::timeout(Duration::from_secs(5), wait_for_reference_proof_barrier(&barrier, &mut update))
-            .await
-            .expect("an early update failure should be observed without waiting for the barrier deadline")
-            .expect_err("a failed update cannot reach the reference proof barrier");
-        assert!(err.contains("Mutation"), "{err}");
-        assert!(err.contains("injected driver initialization failure before reference proof"), "{err}");
-    }
-
     #[tokio::test]
     #[serial_test::serial]
     async fn reference_proof_rejects_a_changed_prepared_fence_revision_before_publish() {
@@ -17659,7 +17518,7 @@ mod tests {
         let scoped_barrier = barrier.clone();
         let update_manager = manager.clone();
         let update_store = store.clone();
-        let mut update = tokio::spawn(async move {
+        let update = tokio::spawn(async move {
             TIER_REFERENCE_PROOF_TEST_BARRIER
                 .scope(
                     scoped_barrier,
@@ -17674,9 +17533,7 @@ mod tests {
                 )
                 .await
         });
-        wait_for_reference_proof_barrier(&barrier, &mut update)
-            .await
-            .expect("tier update should reach the reference proof barrier");
+        barrier.arrived.notified().await;
 
         let unrelated = prepared_remove_intent("COLD-B", uuid::Uuid::from_u128(0x2237));
         TierConfigMgr::apply_prepared_mutation_intent_block(&manager, &unrelated)
@@ -17684,9 +17541,8 @@ mod tests {
             .expect("an unrelated prepared fence should advance the runtime revision");
         barrier.release.add_permits(1);
 
-        let err = tokio::time::timeout(Duration::from_secs(30), update)
+        let err = update
             .await
-            .expect("tier update should finish after the reference proof barrier releases")
             .expect("tier update task should join")
             .expect_err("a reference proof cannot authorize publication across a fence revision change");
         let TierConfigUpdateError::Publish(err) = err else {

crates/ecstore/src/services/tier/warm_backend.rs

@@ -89,18 +89,6 @@ pub enum TransitionCandidateProbe {
     Unsupported,
 }
 
-/// Live evidence for repairing legacy metadata. Ordinary candidate GETs do not
-/// establish the bucket's versioning model and cannot supply this authority.
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub enum LegacyTransitionStateProbe {
-    Missing,
-    UnversionedPresent,
-    SuspendedNullPresent,
-    VersionedPresent(String),
-    Ambiguous,
-    Unsupported,
-}
-
 #[derive(Clone, Copy)]
 pub(crate) struct TransitionCandidateIdentity {
     pub transaction_id: uuid::Uuid,
@@ -109,14 +97,6 @@ pub(crate) struct TransitionCandidateIdentity {
 
 #[async_trait::async_trait]
 pub(crate) trait TransitionCandidateReconciler {
-    async fn probe_legacy_transition_state(
-        &self,
-        _object: &str,
-        _remote_version: Option<&str>,
-    ) -> Result<LegacyTransitionStateProbe, std::io::Error> {
-        Ok(LegacyTransitionStateProbe::Unsupported)
-    }
-
     async fn probe_transition_candidate_for(
         &self,
         object: &str,
@@ -126,14 +106,6 @@ pub(crate) trait TransitionCandidateReconciler {
 
 #[async_trait::async_trait]
 pub trait WarmBackend {
-    async fn probe_legacy_metadata(
-        &self,
-        _object: &str,
-        _remote_version: Option<&str>,
-    ) -> Result<LegacyTransitionStateProbe, std::io::Error> {
-        Ok(LegacyTransitionStateProbe::Unsupported)
-    }
-
     async fn validate(&self) -> Result<(), std::io::Error> {
         Ok(())
     }
@@ -476,18 +448,6 @@ impl MeteredWarmBackend {
 
 #[async_trait::async_trait]
 impl WarmBackend for MeteredWarmBackend {
-    async fn probe_legacy_metadata(
-        &self,
-        object: &str,
-        remote_version: Option<&str>,
-    ) -> Result<LegacyTransitionStateProbe, std::io::Error> {
-        let result = self.inner.probe_legacy_metadata(object, remote_version).await;
-        if matches!(result, Ok(LegacyTransitionStateProbe::Unsupported)) {
-            return result;
-        }
-        Self::record(TierRequestOperation::Probe, result)
-    }
-
     /// Delegated without a counter: only one backend issues a remote request
     /// here, and every other one takes the trait default, so a `validate`
     /// counter would mostly record requests that never happened.
@@ -564,18 +524,6 @@ struct MeteredTransitionCandidateReconciler {
 
 #[async_trait::async_trait]
 impl TransitionCandidateReconciler for MeteredTransitionCandidateReconciler {
-    async fn probe_legacy_transition_state(
-        &self,
-        object: &str,
-        remote_version: Option<&str>,
-    ) -> Result<LegacyTransitionStateProbe, std::io::Error> {
-        let result = self.inner.probe_legacy_transition_state(object, remote_version).await;
-        if matches!(result, Ok(LegacyTransitionStateProbe::Unsupported)) {
-            return result;
-        }
-        MeteredWarmBackend::record(TierRequestOperation::Probe, result)
-    }
-
     async fn probe_transition_candidate_for(
         &self,
         object: &str,

crates/ecstore/src/services/tier/warm_backend_minio.rs

@@ -101,19 +101,6 @@ impl WarmBackend for WarmBackendMinIO {
 
 #[async_trait::async_trait]
 impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendMinIO {
-    async fn probe_legacy_transition_state(
-        &self,
-        object: &str,
-        remote_version: Option<&str>,
-    ) -> Result<super::warm_backend::LegacyTransitionStateProbe, std::io::Error> {
-        crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_legacy_transition_state(
-            &self.0,
-            object,
-            remote_version,
-        )
-        .await
-    }
-
     async fn probe_transition_candidate_for(
         &self,
         object: &str,

crates/ecstore/src/services/tier/warm_backend_rustfs.rs

@@ -146,19 +146,6 @@ impl WarmBackend for WarmBackendRustFS {
 
 #[async_trait::async_trait]
 impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendRustFS {
-    async fn probe_legacy_transition_state(
-        &self,
-        object: &str,
-        remote_version: Option<&str>,
-    ) -> Result<super::warm_backend::LegacyTransitionStateProbe, std::io::Error> {
-        crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_legacy_transition_state(
-            &self.0,
-            object,
-            remote_version,
-        )
-        .await
-    }
-
     async fn probe_transition_candidate_for(
         &self,
         object: &str,

crates/ecstore/src/services/tier/warm_backend_s3.rs

@@ -376,6 +376,7 @@ struct TransitionCandidateVersions {
 }
 
 impl TransitionCandidateVersions {
+    #[cfg(test)]
     fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) {
         for version in versions.versions.iter().filter(|version| version.key == remote_object) {
             if self.version_id.is_some() {
@@ -511,20 +512,6 @@ mod tests {
     }
 
     async fn candidate_probe_fixture() -> Option<(WarmBackendS3, tokio::task::JoinHandle<Vec<String>>)> {
-        scripted_probe_fixture([
-                "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: opaque-version\r\nConnection: close\r\n\r\nx",
-                "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nConnection: close\r\n\r\nx",
-                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
-                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\n<Error><Code>NoSuchObject</Code><Message>missing</Message></Error>",
-                "HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\n<Error><Code>AccessDenied</Code><Message>denied</Message></Error>",
-                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
-                "HTTP/1.1 416 Range Not Satisfiable\r\nContent-Type: application/xml\r\nContent-Length: 72\r\nConnection: close\r\n\r\n<Error><Code>InvalidRange</Code><Message>empty version</Message></Error>",
-                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 67\r\nConnection: close\r\n\r\n<Error><Code>NoSuchVersion</Code><Message>missing</Message></Error>",
-                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
-            ].into_iter().map(str::to_owned).collect()).await
-    }
-
-    async fn scripted_probe_fixture(responses: Vec<String>) -> Option<(WarmBackendS3, tokio::task::JoinHandle<Vec<String>>)> {
         let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
             Ok(listener) => listener,
             Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
@@ -535,6 +522,17 @@ mod tests {
             .expect("listener local address should be available")
             .to_string();
         let fixture = tokio::spawn(async move {
+            let responses = [
+                "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: opaque-version\r\nConnection: close\r\n\r\nx",
+                "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nConnection: close\r\n\r\nx",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\n<Error><Code>NoSuchObject</Code><Message>missing</Message></Error>",
+                "HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\n<Error><Code>AccessDenied</Code><Message>denied</Message></Error>",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
+                "HTTP/1.1 416 Range Not Satisfiable\r\nContent-Type: application/xml\r\nContent-Length: 72\r\nConnection: close\r\n\r\n<Error><Code>InvalidRange</Code><Message>empty version</Message></Error>",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 67\r\nConnection: close\r\n\r\n<Error><Code>NoSuchVersion</Code><Message>missing</Message></Error>",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
+            ];
             let mut requests = Vec::new();
             for response in responses {
                 let (mut stream, _) = listener.accept().await.expect("fixture should accept candidate GET");
@@ -675,117 +673,6 @@ mod tests {
         assert!(requests[8].to_ascii_lowercase().contains("?versionid=historical-version"));
     }
 
-    fn legacy_probe_xml_response(body: &str) -> String {
-        format!(
-            "HTTP/1.1 200 OK\r\nContent-Type: application/xml\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
-            body.len()
-        )
-    }
-
-    fn legacy_probe_versioning_response(status: &str) -> String {
-        let state = if status.is_empty() {
-            String::new()
-        } else {
-            format!("<Status>{status}</Status>")
-        };
-        legacy_probe_xml_response(&format!(
-            "<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">{state}</VersioningConfiguration>"
-        ))
-    }
-
-    fn legacy_probe_versions_response(versions: &[&str]) -> String {
-        let versions = versions.iter().map(|version| format!(
-            "<Version><Key>archive/object</Key><VersionId>{version}</VersionId><IsLatest>true</IsLatest><LastModified>2026-09-01T00:00:00Z</LastModified><ETag>\"legacy-etag\"</ETag><Size>7</Size><StorageClass>STANDARD</StorageClass></Version>"
-        )).collect::<String>();
-        legacy_probe_xml_response(&format!(
-            "<ListVersionsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Name>bucket</Name><Prefix>archive/object</Prefix><KeyMarker/><VersionIdMarker/><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>{versions}</ListVersionsResult>"
-        ))
-    }
-
-    #[tokio::test]
-    async fn legacy_transition_state_probe_verifies_disabled_suspended_and_enabled_responses() {
-        use super::super::warm_backend::LegacyTransitionStateProbe as Probe;
-        for (initial, confirmed, version, expected) in [
-            ("", "", "null", Probe::UnversionedPresent),
-            ("Suspended", "Suspended", "null", Probe::SuspendedNullPresent),
-            ("Enabled", "Enabled", "version-a", Probe::VersionedPresent("version-a".to_string())),
-            ("Enabled", "Enabled", "null", Probe::SuspendedNullPresent),
-            ("", "", "unexpected-version", Probe::Ambiguous),
-            ("Suspended", "Enabled", "null", Probe::Ambiguous),
-        ] {
-            let responses = vec![
-                legacy_probe_versioning_response(initial),
-                legacy_probe_versions_response(&[version]),
-                legacy_probe_versioning_response(confirmed),
-            ];
-            let (backend, fixture) = scripted_probe_fixture(responses)
-                .await
-                .expect("legacy probe loopback fixture");
-            let result =
-                tokio::time::timeout(Duration::from_secs(10), backend.probe_legacy_transition_state("archive/object", None))
-                    .await
-                    .expect("legacy probe must finish")
-                    .expect("legacy probe should decode provider XML");
-            assert_eq!(result, expected, "initial={initial} confirmed={confirmed} version={version}");
-            let requests = fixture.await.expect("legacy probe fixture should finish");
-            assert_eq!(requests.len(), 3);
-            assert!(requests.iter().all(|request| request.starts_with("GET ")));
-            assert!(requests[0].lines().next().expect("request line").contains("versioning"));
-            assert!(requests[1].lines().next().expect("request line").contains("versions"));
-            assert!(requests[2].lines().next().expect("request line").contains("versioning"));
-        }
-    }
-
-    #[tokio::test]
-    async fn legacy_transition_state_probe_preserves_historical_exact_version() {
-        use super::super::warm_backend::LegacyTransitionStateProbe as Probe;
-        let responses = vec![
-            legacy_probe_versioning_response("Enabled"),
-            "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: historical-version\r\nConnection: close\r\n\r\nx".to_string(),
-            legacy_probe_versioning_response("Enabled"),
-        ];
-        let (backend, fixture) = scripted_probe_fixture(responses).await.expect("exact legacy probe fixture");
-        assert_eq!(
-            backend
-                .probe_legacy_transition_state("archive/object", Some("historical-version"))
-                .await
-                .expect("exact version proof"),
-            Probe::VersionedPresent("historical-version".to_string())
-        );
-        let requests = fixture.await.expect("exact probe fixture should finish");
-        assert!(
-            requests[1]
-                .lines()
-                .next()
-                .expect("request line")
-                .contains("versionId=historical-version")
-        );
-        assert!(requests[1].to_ascii_lowercase().contains("range: bytes=0-0"));
-        assert!(requests.iter().all(|request| request.starts_with("GET ")));
-    }
-
-    #[tokio::test]
-    async fn legacy_transition_state_probe_retains_multiple_candidates() {
-        use super::super::warm_backend::LegacyTransitionStateProbe as Probe;
-        let responses = vec![
-            legacy_probe_versioning_response("Enabled"),
-            legacy_probe_versions_response(&["version-a", "version-b"]),
-        ];
-        let (backend, fixture) = scripted_probe_fixture(responses)
-            .await
-            .expect("ambiguous legacy probe fixture");
-        assert_eq!(
-            backend
-                .probe_legacy_transition_state("archive/object", None)
-                .await
-                .expect("ambiguous proof"),
-            Probe::Ambiguous
-        );
-        let requests = fixture.await.expect("ambiguous fixture should finish");
-        assert_eq!(requests.len(), 2);
-        assert!(requests.iter().all(|request| request.starts_with("GET ")));
-    }
-
     fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult {
         ListVersionsResult {
             versions: versions
@@ -1034,56 +921,6 @@ impl WarmBackend for WarmBackendS3 {
 
 #[async_trait::async_trait]
 impl TransitionCandidateReconciler for WarmBackendS3 {
-    async fn probe_legacy_transition_state(
-        &self,
-        object: &str,
-        remote_version: Option<&str>,
-    ) -> Result<super::warm_backend::LegacyTransitionStateProbe, std::io::Error> {
-        use super::warm_backend::LegacyTransitionStateProbe as Probe;
-
-        let initial_versioning = self.remote_bucket_versioning().await?;
-        let candidate = if let Some(version) = remote_version.filter(|version| !version.is_empty()) {
-            validate_remote_version_id(version)?;
-            match self.probe_transition_version(object, version).await? {
-                TransitionCandidateProbe::VersionedPresent(actual) if actual == version => Some(actual),
-                TransitionCandidateProbe::Missing => return Ok(Probe::Missing),
-                _ => return Ok(Probe::Ambiguous),
-            }
-        } else {
-            let remote_object = self.get_dest(object);
-            let mut opts = ListObjectsOptions::default();
-            opts.set("prefix", &remote_object);
-            opts.set("max-keys", "1000");
-            let mut key_marker = String::new();
-            let mut version_marker = String::new();
-            let mut candidates = TransitionCandidateVersions::default();
-            let mut complete = false;
-            // This is one synchronous record inspection, not an unbounded
-            // remote history scan. The caller also bounds the whole probe.
-            for _ in 0..128 {
-                let page = self
-                    .client
-                    .list_object_versions_query(&self.bucket, &opts, &key_marker, &version_marker, "")
-                    .await?;
-                candidates.extend(&remote_object, &page);
-                if candidates.ambiguous {
-                    return Ok(Probe::Ambiguous);
-                }
-                if !page.is_truncated {
-                    complete = true;
-                    break;
-                }
-                advance_version_markers(&mut key_marker, &mut version_marker, &page)?;
-            }
-            if !complete {
-                return Ok(Probe::Ambiguous);
-            }
-            candidates.version_id
-        };
-        let confirmed_versioning = self.remote_bucket_versioning().await?;
-        classify_legacy_transition_state(candidate.as_deref(), initial_versioning, confirmed_versioning)
-    }
-
     async fn probe_transition_candidate_for(
         &self,
         object: &str,
@@ -1094,32 +931,3 @@ impl TransitionCandidateReconciler for WarmBackendS3 {
             .await
     }
 }
-
-fn classify_legacy_transition_state(
-    candidate: Option<&str>,
-    initial: RemoteBucketVersioning,
-    confirmed: RemoteBucketVersioning,
-) -> Result<super::warm_backend::LegacyTransitionStateProbe, std::io::Error> {
-    use super::warm_backend::LegacyTransitionStateProbe as Probe;
-    if initial != confirmed {
-        return Ok(Probe::Ambiguous);
-    }
-    let Some(version) = candidate else {
-        return Ok(Probe::Missing);
-    };
-    if !version.is_empty() {
-        validate_remote_version_id(version)?;
-        if uuid::Uuid::parse_str(version).is_ok_and(|id| id.is_nil()) {
-            return Err(std::io::Error::new(
-                std::io::ErrorKind::InvalidData,
-                "legacy tier probe returned a nil version identifier",
-            ));
-        }
-    }
-    Ok(match (confirmed, version) {
-        (RemoteBucketVersioning::Disabled, "" | "null") => Probe::UnversionedPresent,
-        (RemoteBucketVersioning::Disabled, _) | (_, "") => Probe::Ambiguous,
-        (_, "null") => Probe::SuspendedNullPresent,
-        (_, version) => Probe::VersionedPresent(version.to_string()),
-    })
-}

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -51,12 +51,12 @@ use super::super::{
     can_try_inline_data_shards_direct, capacity_scope_from_disks, codec_streaming_rollout_applies, coding,
     collect_inline_data_shard_fileinfos_by_index_or_reason, current_dirty_generation, debug, disk,
     file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, info, inline_erasure_shard_file_offset,
-    inline_erasure_shard_size, is_get_metadata_data_read_early_stop_enabled, is_get_metadata_early_stop_bounded_fanout_enabled,
-    is_get_metadata_early_stop_enabled, is_get_metadata_non_inline_data_read_early_stop_enabled, is_object_dangling,
-    is_version_early_stop_enabled, issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure,
-    merge_file_meta_versions, object_fits_single_block, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs,
-    reduce_write_quorum_errs, send_heal_request_with_admission, should_prevent_write, to_object_err,
-    try_read_inline_data_shards_direct, warn,
+    inline_erasure_shard_size, is_err_object_not_found, is_err_version_not_found, is_get_metadata_data_read_early_stop_enabled,
+    is_get_metadata_early_stop_bounded_fanout_enabled, is_get_metadata_early_stop_enabled,
+    is_get_metadata_non_inline_data_read_early_stop_enabled, is_object_dangling, is_version_early_stop_enabled,
+    issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure, merge_file_meta_versions,
+    object_fits_single_block, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs, reduce_write_quorum_errs,
+    send_heal_request_with_admission, should_prevent_write, to_object_err, try_read_inline_data_shards_direct, warn,
 };
 #[cfg(test)]
 pub(in crate::set_disk) use super::metadata_quorum::MetadataEarlyStopDecision;
@@ -3086,61 +3086,21 @@ impl SetDisks {
         let read_quorum = disks.len().div_ceil(2).max(1);
         let (raw_fileinfos, errs) = Self::read_all_raw_file_info(&disks, bucket, disk_object.as_str(), false).await;
 
-        if let Some(err) = errs
-            .iter()
-            .flatten()
-            .find(|err| !matches!(err, DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound))
-        {
-            return Err(to_object_err(err.clone().into(), vec![bucket, object]));
-        }
-        // A minority live owner must not disappear behind majority absence.
-        // Only explicit absence on every readable disk proves no ownership.
-        if raw_fileinfos.iter().all(Option::is_none) {
-            return Ok(None);
+        if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
+            let object_err = to_object_err(err.into(), vec![bucket, object]);
+            if is_err_object_not_found(&object_err) || is_err_version_not_found(&object_err) {
+                return Ok(None);
+            }
+            return Err(object_err);
         }
 
         let mut shallow_versions = Vec::with_capacity(raw_fileinfos.len());
-        type TransitionCopy = (FileInfo, Option<crate::services::tier::tier::TierDestinationId>);
-        let mut transition_copies: std::collections::HashMap<Option<Uuid>, Vec<TransitionCopy>> =
-            std::collections::HashMap::new();
-        let decode_error = |err| Error::other(format!("exact object versions decode failed for {bucket}/{object}: {err}"));
         for raw_fileinfo in raw_fileinfos.into_iter().flatten() {
             let meta = FileMeta::load(&raw_fileinfo.buf)
                 .map_err(|err| Error::other(format!("exact object metadata decode failed for {bucket}/{object}: {err}")))?;
-            let versions = meta.get_all_file_info_versions(bucket, object, true).map_err(decode_error)?;
-            for version in versions.versions.into_iter().chain(versions.free_versions) {
-                if version.transition_status != rustfs_filemeta::TRANSITION_COMPLETE {
-                    continue;
-                }
-                let destination =
-                    crate::services::tier::tier::tier_destination_id_from_metadata(&version.metadata).map_err(Error::other)?;
-                transition_copies
-                    .entry(version.version_id.filter(|id| !id.is_nil()))
-                    .or_default()
-                    .push((version, destination));
-            }
             shallow_versions.push(meta.versions);
         }
 
-        // Exact cleanup/recovery reads must not select a repaired majority
-        // while another physical copy still carries legacy absence. Missing
-        // copies permit deletion retries; an unreadable disk proves nothing.
-        for copies in transition_copies
-            .values()
-            .filter(|copies| copies.iter().any(|(_, destination)| destination.is_some()))
-        {
-            let (first, destination) = &copies[0];
-            if copies.iter().any(|(copy, identity)| {
-                identity != destination
-                    || copy.transition_version_state != first.transition_version_state
-                    || copy.transition_version != first.transition_version
-                    || copy.transition_tier != first.transition_tier
-                    || copy.transitioned_objname != first.transitioned_objname
-            }) {
-                return Err(Error::other("exact transition metadata has not converged across physical copies"));
-            }
-        }
-
         if shallow_versions.len() < read_quorum {
             return Err(to_object_err(StorageError::ErasureReadQuorum, vec![bucket, object]));
         }
@@ -3157,7 +3117,7 @@ impl SetDisks {
             ..Default::default()
         }
         .get_all_file_info_versions(bucket, object, true)
-        .map_err(decode_error)?;
+        .map_err(|err| Error::other(format!("exact object versions decode failed for {bucket}/{object}: {err}")))?;
 
         for file_info in file_info_versions
             .versions
@@ -5855,17 +5815,17 @@ impl SetDisks {
                 }
 
                 if let Some(disk) = disks[i].as_ref() {
-                    // A failed version-only copy owns only its new version.
-                    // Removing the whole xl.meta would also erase existing
-                    // versions and any concurrently reconciled tier binding.
-                    let mut rollback = FileInfo {
-                        version_id: files[i].version_id,
-                        ..Default::default()
-                    };
-                    rollback.set_skip_tier_free_version();
+                    let path = path_join_buf(&[prefix, STORAGE_FORMAT_FILE]);
                     revert_futures.push(async move {
                         if let Err(err) = disk
-                            .delete_version(bucket, prefix, rollback, false, DeleteOptions::default())
+                            .delete(
+                                bucket,
+                                &path,
+                                DeleteOptions {
+                                    recursive: true,
+                                    ..Default::default()
+                                },
+                            )
                             .await
                         {
                             warn!("write meta revert err {:?}", err);
@@ -12132,9 +12092,7 @@ mod tests {
         let bucket = "write-unique-bucket";
         let object = "object";
         let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
-        let mut fi = metadata_test_fileinfo(object);
-        fi.mod_time = Some(OffsetDateTime::now_utc());
-        let files = vec![fi.clone(), fi];
+        let files = vec![metadata_test_fileinfo(object), metadata_test_fileinfo(object)];
 
         let result = SetDisks::write_unique_file_info(&[Some(disk.clone()), None], bucket, bucket, object, &files, 2).await;
 
@@ -12148,53 +12106,6 @@ mod tests {
         );
     }
 
-    #[tokio::test]
-    async fn write_unique_file_info_rollback_preserves_existing_reconciled_version() {
-        let bucket = "write-unique-existing";
-        let object = "object";
-        let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
-        let mut original = metadata_test_fileinfo(object);
-        original.version_id = Some(Uuid::from_u128(1));
-        original.data_dir = Some(Uuid::from_u128(3));
-        original.mod_time = Some(OffsetDateTime::now_utc());
-        original.transition_status = rustfs_filemeta::TRANSITION_COMPLETE.to_string();
-        original.transition_tier = "WARM".to_string();
-        original.transitioned_objname = "remote-original".to_string();
-        original.transition_version_state = rustfs_filemeta::TransitionVersionState::KnownDisabled;
-        rustfs_utils::http::insert_str(
-            &mut original.metadata,
-            rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
-            "ab".repeat(32),
-        );
-        disk.write_metadata(bucket, bucket, object, original.clone())
-            .await
-            .expect("existing reconciled source");
-        let raw = disk
-            .read_all(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}"))
-            .await
-            .expect("original metadata");
-        let before = FileMeta::load(&raw).unwrap().find_version(original.version_id).unwrap().1;
-        let mut added = original.clone();
-        added.version_id = Some(Uuid::from_u128(2));
-        let result =
-            SetDisks::write_unique_file_info(&[Some(disk.clone()), None], bucket, bucket, object, &[added.clone(), added], 2)
-                .await;
-        assert!(result.is_err());
-        let raw = disk
-            .read_all(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}"))
-            .await
-            .expect("preserved xl.meta");
-        let after = FileMeta::load(&raw).expect("preserved metadata");
-        assert_eq!(
-            after
-                .find_version(original.version_id)
-                .expect("original source survives rollback")
-                .1,
-            before
-        );
-        assert!(after.find_version(Some(Uuid::from_u128(2))).is_err());
-    }
-
     #[tokio::test]
     async fn update_object_meta_handles_empty_metadata_and_missing_quorum() {
         let set = io_primitives_test_set(vec![None, None], 1).await;

crates/ecstore/src/set_disk/metadata.rs

@@ -326,15 +326,14 @@ impl SetDisks {
         let parity_blocks = Self::common_parity(&parities, default_parity_count as i32);
 
         if parity_blocks < 0 {
-            // A consistent layout can require more replies than the initial
-            // half-set probe. Reaching that probe alone is not corruption;
-            // only invalid or conflicting healthy replies establish that.
+            // No parity value reached read quorum. Distinguish two cases:
+            // enough disks answered with valid-looking metadata that simply
+            // cannot be reconciled (corrupt/foreign entries — retrying cannot
+            // help, and heal should see Corrupt, rustfs#5801) versus too few
+            // healthy answers (a genuine quorum condition where retry may
+            // succeed once disks recover).
             let healthy_replies = errs.iter().filter(|err| err.is_none()).count();
-            let consistent_parity = parities
-                .iter()
-                .find(|&&parity| parity >= 0)
-                .filter(|&&parity| parities.iter().filter(|&&candidate| candidate == parity).count() == healthy_replies);
-            if healthy_replies >= expected_rquorum && consistent_parity.is_none() {
+            if healthy_replies >= expected_rquorum {
                 error!(
                     "object_quorum_from_meta: irreconcilable parity across {healthy_replies} healthy replies (corrupt metadata), errs={errs:?}"
                 );
@@ -1653,40 +1652,6 @@ mod tests {
         assert_eq!(err, DiskError::FileCorrupt);
     }
 
-    #[test]
-    fn consistent_parity_below_its_data_shard_quorum_is_not_corruption() {
-        for (drive_count, parity) in [(6, 2), (8, 2), (12, 4)] {
-            let data = drive_count - parity;
-            let mut metas = (1..=drive_count)
-                .map(|index| {
-                    let mut info = FileInfo::new("bucket/object", data, parity);
-                    info.size = 1024;
-                    info.erasure.index = index;
-                    info
-                })
-                .collect::<Vec<_>>();
-            let mut errs = vec![Some(DiskError::DiskNotFound); drive_count];
-            errs[..data].fill(None);
-            assert_eq!(
-                SetDisks::object_quorum_from_meta(&metas, &errs, parity).expect("exact data quorum should resolve"),
-                (data as i32, data as i32)
-            );
-
-            errs[data - 1] = Some(DiskError::DiskNotFound);
-            assert_eq!(
-                SetDisks::object_quorum_from_meta(&metas, &errs, parity).expect_err("one fewer shard cannot resolve"),
-                DiskError::ErasureReadQuorum,
-                "layout {drive_count}/{parity} has consistent metadata but insufficient shards"
-            );
-
-            metas[0].erasure.parity_blocks = usize::MAX;
-            assert_eq!(
-                SetDisks::object_quorum_from_meta(&metas, &errs, parity).expect_err("corrupt healthy replies must be rejected"),
-                DiskError::FileCorrupt
-            );
-        }
-    }
-
     /// Too few healthy replies remains a genuine quorum condition where a
     /// retry may succeed once disks recover.
     #[test]

crates/ecstore/src/set_disk/mod.rs

@@ -865,7 +865,6 @@ pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, ren
 mod ctx;
 mod metadata;
 mod ops;
-pub(crate) use ops::bucket::BucketInfoQuorum;
 
 #[cfg(test)]
 pub(crate) use ops::hermetic_set_disks_isolated;
@@ -3852,7 +3851,7 @@ pub struct SetDisks {
     pub default_parity_count: usize,
     pub set_index: usize,
     pub pool_index: usize,
-    /// Stable namespace shared by every object lock created for this pool.
+    /// Stable namespace shared by every object lock created for this set.
     set_lock_namespace: Arc<str>,
     pub format: FormatV3,
     #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
@@ -3889,33 +3888,6 @@ pub struct SetDisks {
     >,
 }
 
-/// Read every physical copy before selecting a version quorum. A minority
-/// legacy record is still evidence and must not disappear behind a majority
-/// not-found result. Only an explicit file/volume absence produces `None`;
-/// an unreadable disk cannot prove that no conflicting copy exists.
-pub(crate) async fn read_legacy_transition_state_metadata_copies(
-    set: &SetDisks,
-    bucket: &str,
-    object: &str,
-) -> std::result::Result<Vec<Option<Vec<u8>>>, DiskError> {
-    let disk_object = rustfs_utils::path::encode_dir_object(object);
-    let disks = set.get_disks_internal().await;
-    if disks.is_empty() {
-        return Err(DiskError::DiskNotFound);
-    }
-
-    // Include inline bytes in the generation: a conditional repair preserves
-    // the entire xl.meta, including payloads belonging to other versions.
-    let (copies, errs) = SetDisks::read_all_raw_file_info(&disks, bucket, disk_object.as_str(), true).await;
-    for err in errs.into_iter().flatten() {
-        if !matches!(err, DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound) {
-            return Err(err);
-        }
-    }
-
-    Ok(copies.into_iter().map(|copy| copy.map(|copy| copy.buf)).collect())
-}
-
 // DistributedLock sends the raw ObjectKey to its clients; LockRegistry clones
 // each endpoint's canonical Arc, so an exact Arc set identifies the lock domain.
 pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
@@ -4264,7 +4236,7 @@ impl SetDisks {
         self.get_object_metadata_cache_generations[generation.index].load(Ordering::Acquire) == generation.value
     }
 
-    pub(crate) async fn invalidate_get_object_metadata_cache(&self, bucket: &str, object: &str) {
+    async fn invalidate_get_object_metadata_cache(&self, bucket: &str, object: &str) {
         let hash = self.get_object_metadata_cache_hash(bucket, object);
         let hash_bytes = hash.to_le_bytes();
         let index = usize::from(u16::from_le_bytes([hash_bytes[0], hash_bytes[1]]) % GET_OBJECT_METADATA_CACHE_FENCE_SHARDS);
@@ -4491,7 +4463,7 @@ impl SetDisks {
         instance_ctx: Arc<InstanceContext>,
     ) -> Arc<Self> {
         let ctx = instance_ctx;
-        let set_lock_namespace: Arc<str> = format!("pool-{pool_index}").into();
+        let set_lock_namespace: Arc<str> = format!("set-{pool_index}-{set_index}").into();
         let shared_lockers = Arc::from(lockers.to_vec());
         Arc::new(SetDisks {
             locker_owner,
@@ -4605,9 +4577,7 @@ impl SetDisks {
     pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
         match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
             (false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
-            (true, true) => {
-                self.set_lock_namespace == other.set_lock_namespace && same_distributed_lock_domain(&self.lockers, &other.lockers)
-            }
+            (true, true) => same_distributed_lock_domain(&self.lockers, &other.lockers),
             _ => false,
         }
     }
@@ -7125,7 +7095,7 @@ mod tests {
         ctx.update_erasure_type(SetupType::Erasure).await;
         let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
 
-        assert_eq!(&*set.set_lock_namespace, "pool-0");
+        assert_eq!(&*set.set_lock_namespace, "set-0-0");
         let before = Arc::strong_count(&set.set_lock_namespace);
         let lock = set
             .new_ns_lock("bucket", "object")
@@ -8350,78 +8320,6 @@ mod tests {
         );
     }
 
-    #[tokio::test(flavor = "multi_thread")]
-    #[serial]
-    async fn test_new_ns_lock_distributed_write_succeeds_with_three_lockers_one_offline() {
-        let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
-
-        let manager_a = Arc::new(rustfs_lock::GlobalLockManager::new());
-        let manager_b = Arc::new(rustfs_lock::GlobalLockManager::new());
-        let healthy_a: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager_a));
-        let healthy_b: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager_b));
-        let failing_client: Arc<dyn LockClient> = Arc::new(FailingClient);
-        let set_disks = make_test_set_disks(vec![healthy_a, failing_client, healthy_b]).await;
-
-        let guard = set_disks
-            .new_ns_lock("bucket", "object")
-            .await
-            .expect("namespace lock should be created")
-            .get_write_lock(Duration::from_millis(500))
-            .await
-            .expect("two healthy lockers should satisfy the three-locker write quorum");
-
-        match guard {
-            NamespaceLockGuard::Standard(_) => {}
-            NamespaceLockGuard::Fast(_) => panic!("Expected distributed guard for dist-erasure"),
-        }
-    }
-
-    #[tokio::test(flavor = "multi_thread")]
-    #[serial]
-    async fn namespace_lock_domain_includes_pool_namespace() {
-        let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
-
-        let first: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
-        let second: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
-        let lockers = vec![first, second];
-        let same_pool_first_set = make_test_set_disks_with_ctx(lockers.clone(), bootstrap_ctx()).await;
-        let same_pool_second_set = SetDisks::new_with_instance_ctx(
-            "test-owner".to_string(),
-            Arc::new(RwLock::new(vec![None, None])),
-            2,
-            1,
-            1,
-            0,
-            same_pool_first_set.set_endpoints.clone(),
-            FormatV3::new(2, 2),
-            lockers.clone(),
-            bootstrap_ctx(),
-        )
-        .await;
-        let other_pool_set = SetDisks::new_with_instance_ctx(
-            "test-owner".to_string(),
-            Arc::new(RwLock::new(vec![None, None])),
-            2,
-            1,
-            0,
-            1,
-            same_pool_first_set.set_endpoints.clone(),
-            FormatV3::new(1, 2),
-            lockers,
-            bootstrap_ctx(),
-        )
-        .await;
-
-        assert!(
-            same_pool_first_set.shares_namespace_lock_domain(&same_pool_second_set).await,
-            "sets in the same pool share the object namespace lock domain"
-        );
-        assert!(
-            !same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
-            "different pool namespaces must not be deduplicated solely by identical clients"
-        );
-    }
-
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
     async fn streaming_reader_holds_read_lock_until_eof() {

crates/ecstore/src/set_disk/ops/bucket.rs

@@ -21,72 +21,12 @@
 
 use super::super::{
     BUCKET_OP_IGNORED_ERRS, BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, DiskError, Error, HashMap,
-    MakeBucketOptions, Result, SetDisks, is_reserved_or_invalid_bucket, join_all, reduce_read_quorum_errs,
-    reduce_write_quorum_errs,
+    MakeBucketOptions, Result, SetDisks, is_reserved_or_invalid_bucket, join_all, reduce_write_quorum_errs,
 };
 use crate::api::bucket::metadata_sys;
 use crate::disk::DiskAPI;
 
-#[derive(Clone, Copy)]
-pub(crate) enum BucketInfoQuorum {
-    Read,
-    Write,
-}
-
 impl SetDisks {
-    pub(crate) async fn stat_bucket_with_quorum(&self, bucket: &str, quorum: BucketInfoQuorum) -> Result<BucketInfo> {
-        let disks = self.disk_inventory().await;
-        let disk_count = disks.len();
-        let mut futures = Vec::with_capacity(disk_count);
-        for disk in disks {
-            let bucket = bucket.to_string();
-            futures.push(async move {
-                match disk {
-                    Some(disk) => disk.stat_volume(&bucket).await,
-                    None => Err(DiskError::DiskNotFound),
-                }
-            });
-        }
-
-        let results = join_all(futures).await;
-        let mut infos = Vec::with_capacity(results.len());
-        let mut errs = Vec::with_capacity(results.len());
-        for result in results {
-            match result {
-                Ok(info) => {
-                    infos.push(Some(info));
-                    errs.push(None);
-                }
-                Err(err) => {
-                    infos.push(None);
-                    errs.push(Some(err));
-                }
-            }
-        }
-
-        let error = match quorum {
-            // Bucket mutations use a majority regardless of object storage
-            // class. A namespace read must intersect that majority; object
-            // readers still enforce the persisted layout's data-shard quorum.
-            BucketInfoQuorum::Read => reduce_read_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, disk_count.div_ceil(2).max(1)),
-            BucketInfoQuorum::Write => reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, disk_count / 2 + 1),
-        };
-        if let Some(err) = error {
-            return Err(err.into());
-        }
-
-        infos
-            .into_iter()
-            .flatten()
-            .next()
-            .map(|info| BucketInfo {
-                name: info.name,
-                created: info.created,
-                ..Default::default()
-            })
-            .ok_or(Error::VolumeNotFound)
-    }
-
     pub(crate) async fn list_bucket_for_scanner(&self, _opts: &BucketOptions) -> Result<(Vec<BucketInfo>, bool)> {
         let disks = self.disk_inventory().await;
         let write_quorum = (disks.len() / 2) + 1;
@@ -191,12 +131,59 @@ impl BucketOperations for SetDisks {
 
     #[tracing::instrument(skip(self))]
     async fn get_bucket_info(&self, bucket: &str, _opts: &BucketOptions) -> Result<BucketInfo> {
-        let mut info = self.stat_bucket_with_quorum(bucket, BucketInfoQuorum::Write).await?;
-        if let Ok(sys) = metadata_sys::get(bucket).await {
-            info.versioning = sys.versioning();
-            info.object_locking = sys.object_locking();
+        let disks = self.disk_inventory().await;
+        let write_quorum = (disks.len() / 2) + 1;
+
+        let mut futures = Vec::with_capacity(disks.len());
+        for disk in disks {
+            let bucket = bucket.to_string();
+            futures.push(async move {
+                match disk {
+                    Some(disk) => disk.stat_volume(&bucket).await,
+                    None => Err(DiskError::DiskNotFound),
+                }
+            });
         }
-        Ok(info)
+
+        let results = join_all(futures).await;
+        let mut infos = Vec::with_capacity(results.len());
+        let mut errs = Vec::with_capacity(results.len());
+        for result in results {
+            match result {
+                Ok(info) => {
+                    infos.push(Some(info));
+                    errs.push(None);
+                }
+                Err(err) => {
+                    infos.push(None);
+                    errs.push(Some(err));
+                }
+            }
+        }
+
+        if let Some(err) = reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, write_quorum) {
+            return Err(err.into());
+        }
+
+        let mut versioning = false;
+        let mut object_locking = false;
+        if let Ok(sys) = metadata_sys::get(bucket).await {
+            versioning = sys.versioning();
+            object_locking = sys.object_locking();
+        }
+
+        infos
+            .into_iter()
+            .flatten()
+            .next()
+            .map(|info| BucketInfo {
+                name: info.name,
+                created: info.created,
+                versioning,
+                object_locking,
+                ..Default::default()
+            })
+            .ok_or(Error::VolumeNotFound)
     }
 
     #[tracing::instrument(skip(self))]

crates/ecstore/src/set_disk/ops/object.rs

@@ -13807,17 +13807,7 @@ mod transition_commit_failure_tests {
     #[tokio::test]
     #[serial_test::serial]
     async fn restore_failure_after_snapshot_cleans_exact_generation_and_returns_primary_error() {
-        assert_restore_failure_cleanup_boundary(true).await;
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn restore_failure_after_snapshot_preserves_corrupt_known_transition_metadata() {
-        assert_restore_failure_cleanup_boundary(false).await;
-    }
-
-    async fn assert_restore_failure_cleanup_boundary(legacy_unknown: bool) {
-        let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
         let bucket = "restore-post-snapshot-cleanup-bucket";
         let object = "object.bin";
         for disk in &disk_stores {
@@ -13826,15 +13816,7 @@ mod transition_commit_failure_tests {
 
         let mut reader = PutObjReader::from_vec(b"post-snapshot cleanup source".repeat(1024));
         let original = set_disks
-            .put_object(
-                bucket,
-                object,
-                &mut reader,
-                &ObjectOptions {
-                    write_completion: WriteCompletion::TailDrained,
-                    ..Default::default()
-                },
-            )
+            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
             .await
             .expect("source object should be written");
         let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
@@ -13875,70 +13857,16 @@ mod transition_commit_failure_tests {
             .await
             .expect("transitioned metadata should be readable")
             .into_owned();
-        let known_state = source_fi.transition_version_state;
-        assert_ne!(known_state, rustfs_filemeta::TransitionVersionState::Unknown);
         source_fi.metadata.extend(restore_metadata(operation_id, true));
+        rustfs_utils::http::insert_str(
+            &mut source_fi.metadata,
+            rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+            "invalid".to_string(),
+        );
         set_disks
             .update_object_meta(bucket, object, source_fi, &online_disks)
             .await
-            .expect("restore markers should be persisted");
-
-        // Normal writes reject damage to a reconciled binding. Model on-disk
-        // corruption directly, with and without the legacy missing-state field.
-        let mut corrupted_metadata = Vec::new();
-        for temp_dir in &temp_dirs {
-            let metadata_path = temp_dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE);
-            let encoded = tokio::fs::read(&metadata_path)
-                .await
-                .expect("transition metadata should be readable");
-            let mut metadata = FileMeta::load(&encoded).expect("transition metadata should decode");
-            let (version_index, mut version) = metadata
-                .find_version(original.version_id)
-                .expect("transitioned version should exist");
-            let object_meta = version.object.as_mut().expect("transitioned version should be an object");
-            rustfs_utils::http::insert_bytes(
-                &mut object_meta.meta_sys,
-                rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
-                b"invalid".to_vec(),
-            );
-            if legacy_unknown {
-                rustfs_utils::http::remove_bytes(
-                    &mut object_meta.meta_sys,
-                    rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
-                );
-            }
-            metadata.versions[version_index] =
-                rustfs_filemeta::FileMetaShallowVersion::try_from(version).expect("corrupt fixture should re-encode");
-            tokio::fs::write(&metadata_path, metadata.marshal_msg().expect("corrupt fixture should encode"))
-                .await
-                .expect("corrupt fixture should be written");
-            let persisted = tokio::fs::read(&metadata_path)
-                .await
-                .expect("corrupt fixture should be readable");
-            let fixture = FileMeta::load(&persisted)
-                .expect("corrupt fixture should decode")
-                .find_version(original.version_id)
-                .expect("corrupt version should exist")
-                .1
-                .into_fileinfo(bucket, object, true)
-                .expect("corrupt version should decode");
-            assert_eq!(
-                fixture.transition_version_state,
-                if legacy_unknown {
-                    rustfs_filemeta::TransitionVersionState::Unknown
-                } else {
-                    known_state
-                }
-            );
-            assert_eq!(
-                rustfs_utils::http::get_str(&fixture.metadata, rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID),
-                Some("invalid".to_string())
-            );
-            for (key, value) in restore_metadata(operation_id, true) {
-                assert_eq!(fixture.metadata.get(&key), Some(&value), "fixture must retain restore marker {key}");
-            }
-            corrupted_metadata.push((metadata_path, persisted));
-        }
+            .expect("invalid backend identity fixture should be persisted");
         set_disks.invalidate_get_object_metadata_cache(bucket, object).await;
 
         let mut opts = ObjectOptions::default();
@@ -13961,23 +13889,6 @@ mod transition_commit_failure_tests {
             .await
             .expect("cleanup should leave the transitioned object readable");
         assert_eq!(cleaned.transitioned_object.status, TRANSITION_COMPLETE);
-        if !legacy_unknown {
-            // Known bindings with corrupt identities must be repaired before
-            // cleanup; rejection must preserve both the binding and markers.
-            for (key, value) in restore_metadata(operation_id, true) {
-                assert_eq!(cleaned.user_defined.get(&key), Some(&value), "cleanup must preserve restore marker {key}");
-            }
-            for (metadata_path, before) in corrupted_metadata {
-                assert_eq!(
-                    tokio::fs::read(metadata_path)
-                        .await
-                        .expect("rejected cleanup metadata should remain readable"),
-                    before,
-                    "rejected cleanup must leave corrupt known metadata unchanged"
-                );
-            }
-            return;
-        }
         assert!(!cleaned.user_defined.contains_key(s3s::header::X_AMZ_RESTORE.as_str()));
         assert!(
             rustfs_utils::http::get_str(cleaned.user_defined.as_ref(), rustfs_utils::http::SUFFIX_RESTORE_OPERATION_ID,)

crates/ecstore/src/store/bucket.rs

@@ -19,7 +19,7 @@ use crate::bucket::{
 };
 use crate::error::is_err_bucket_not_found;
 use crate::runtime::sources as runtime_sources;
-use crate::set_disk::{BucketInfoQuorum, get_lock_acquire_timeout};
+use crate::set_disk::get_lock_acquire_timeout;
 use crate::storage_api_contracts::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, SRBucketDeleteOp};
 use crate::storage_api_contracts::namespace::NamespaceLocking as _;
 use futures::stream::{self, StreamExt};
@@ -364,19 +364,6 @@ impl ECStore {
         Ok(pieces.into_guard(bucket, registration.token))
     }
 
-    /// Hold this guard through recursive-delete authorization and mutation so
-    /// writers cannot introduce an unchecked object into the deletion scope.
-    pub async fn lock_bucket_for_recursive_delete(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
-        if self.ctx.lock_manager().is_disabled() {
-            return Err(StorageError::InvalidArgument(
-                bucket.to_owned(),
-                String::new(),
-                "Recursive deletion requires namespace locking".to_owned(),
-            ));
-        }
-        self.acquire_bucket_lifecycle_write_lock(bucket).await
-    }
-
     pub(crate) async fn acquire_bucket_lifecycle_write_lock(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
         let lock = self.new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT).await?;
         lock.get_write_lock(get_lock_acquire_timeout())
@@ -783,66 +770,19 @@ impl ECStore {
         Ok(())
     }
 
-    /// Prove a live bucket generation before repairing missing expansion volumes.
-    /// Unlike request validation, repair only needs one erasure set to confirm
-    /// existence; an incomplete expansion set is precisely what repair fixes.
-    /// Callers must hold the bucket namespace lock through the subsequent heal.
-    pub(crate) async fn bucket_exists_for_heal(&self, bucket: &str) -> Result<bool> {
-        let results = futures::future::join_all(
-            self.bucket_sets()
-                .map(|(_, _, set)| async move { set.get_bucket_info(bucket, &BucketOptions::default()).await }),
-        )
-        .await;
-        let mut first_error = None;
-        for result in results {
-            match result {
-                Ok(_) => return Ok(true),
-                Err(err) if is_err_strict_volume_not_found(&err) => {}
-                Err(err) if first_error.is_none() => first_error = Some(err),
-                Err(_) => {}
-            }
-        }
-        match first_error {
-            Some(err) => Err(err),
-            None => Ok(false),
-        }
-    }
-
     #[instrument(skip(self))]
     pub(crate) async fn get_bucket_info_from_sets(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
-        self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Write)
-            .await
-    }
-
-    pub(crate) async fn get_bucket_info_from_sets_at_read_quorum(
-        &self,
-        bucket: &str,
-        opts: &BucketOptions,
-    ) -> Result<BucketInfo> {
-        self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Read)
-            .await
-    }
-
-    async fn get_bucket_info_from_sets_with_quorum(
-        &self,
-        bucket: &str,
-        opts: &BucketOptions,
-        quorum: BucketInfoQuorum,
-    ) -> Result<BucketInfo> {
         // One host may participate in several pools after expansion. Resolve the
         // namespace against each erasure set so disks from different pools can
         // never be combined into one bucket quorum.
         // Bucket validation is request-path IO. Keep the previous peer fanout's
         // latency shape by probing every set concurrently; scanner listings use
         // a separate bounded path below because they run continuously.
-        let mut scoped_results = futures::future::join_all(self.bucket_sets().map(|(pool_index, set_index, set)| async move {
-            let result = match quorum {
-                BucketInfoQuorum::Read => set.stat_bucket_with_quorum(bucket, quorum).await,
-                BucketInfoQuorum::Write => set.get_bucket_info(bucket, opts).await,
-            };
-            (pool_index, set_index, result)
-        }))
-        .await;
+        let mut scoped_results =
+            futures::future::join_all(self.bucket_sets().map(|(pool_index, set_index, set)| async move {
+                (pool_index, set_index, set.get_bucket_info(bucket, opts).await)
+            }))
+            .await;
         scoped_results.sort_unstable_by_key(|(pool_index, set_index, _)| (*pool_index, *set_index));
 
         let mut first_info = None;
@@ -866,11 +806,7 @@ impl ECStore {
 
     #[instrument(skip(self))]
     pub(super) async fn handle_get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
-        let mut info = match self.get_bucket_info_from_sets(bucket, opts).await {
-            Ok(info) => info,
-            Err(Error::ErasureWriteQuorum) => return self.get_bucket_info_at_read_quorum(bucket, opts).await,
-            Err(err) => return Err(err),
-        };
+        let mut info = self.get_bucket_info_from_sets(bucket, opts).await?;
 
         if let Ok(sys) = metadata_sys::get_in(&self.ctx, bucket).await {
             if should_override_created_from_metadata(sys.created) {
@@ -883,35 +819,6 @@ impl ECStore {
         Ok(info)
     }
 
-    async fn get_bucket_info_at_read_quorum(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
-        // Lock order: bucket lifecycle -> internal metadata object read locks.
-        // Keep create/delete from changing the namespace while a read quorum
-        // confirms both physical presence and persisted bucket metadata.
-        let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
-        await_bucket_namespace_operation(Some(&guard), bucket, "bucket read quorum validation", async {
-            let mut info = self
-                .get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Read)
-                .await?;
-            let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
-            if !persisted {
-                // A minority of directories left by failed creation is not an
-                // authoritative bucket. Never turn fabricated defaults into
-                // permission to serve degraded reads.
-                return Err(Error::ErasureReadQuorum);
-            }
-            if metadata.name != bucket {
-                return Err(Error::FileCorrupt);
-            }
-            if should_override_created_from_metadata(metadata.created) {
-                info.created = Some(metadata.created);
-            }
-            info.versioning = metadata.versioning();
-            info.object_locking = metadata.object_locking();
-            Ok(info)
-        })
-        .await
-    }
-
     #[instrument(skip(self))]
     pub(super) async fn handle_list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
         // TODO(backlog): support cached bucket listing via opts.cached
@@ -1142,7 +1049,7 @@ mod tests {
         run_physical_bucket_deletion, scan_metadata_less_residue, scan_metadata_less_residue_with_budget,
         should_override_created_from_metadata, validate_table_bucket_delete_allowed,
     };
-    use crate::bucket::metadata::{BucketMetadata, table_bucket_catalog_metadata_prefix};
+    use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
     use crate::bucket::metadata_sys;
     use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
     use crate::disk::{BUCKET_META_PREFIX, DiskAPI, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE};
@@ -1169,7 +1076,6 @@ mod tests {
     use std::sync::atomic::{AtomicBool, Ordering};
     use std::time::{Duration, SystemTime};
     use time::OffsetDateTime;
-    use tokio::io::AsyncReadExt;
     use tokio::sync::{Notify, OnceCell};
     use tokio_util::sync::CancellationToken;
     use uuid::Uuid;
@@ -1453,18 +1359,11 @@ mod tests {
     }
 
     async fn setup_multi_pool_bucket_test_env() -> (tempfile::TempDir, Arc<ECStore>) {
-        setup_bucket_quorum_test_env(&[4, 4], None).await
-    }
-
-    async fn setup_bucket_quorum_test_env(
-        drives_per_pool: &[usize],
-        standard_parity: Option<usize>,
-    ) -> (tempfile::TempDir, Arc<ECStore>) {
         let temp_dir = tempfile::tempdir().expect("multi-pool bucket test directory should be created");
         let mut pools = Vec::new();
-        for (pool_index, &drive_count) in drives_per_pool.iter().enumerate() {
+        for pool_index in 0..2 {
             let mut endpoints = Vec::new();
-            for disk_index in 0..drive_count {
+            for disk_index in 0..4 {
                 let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
                 tokio::fs::create_dir_all(&disk_path)
                     .await
@@ -1479,7 +1378,7 @@ mod tests {
             pools.push(PoolEndpoints {
                 legacy: false,
                 set_count: 1,
-                drives_per_set: drive_count,
+                drives_per_set: 4,
                 endpoints: Endpoints::from(endpoints),
                 cmd_line: format!("bucket-test-pool-{pool_index}"),
                 platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
@@ -1500,12 +1399,9 @@ mod tests {
         )
         .await
         .expect("multi-pool ECStore should initialize");
-        let mut storage_class_kvs = rustfs_config::server_config::KVS::new();
-        if let Some(parity) = standard_parity {
-            storage_class_kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), format!("EC:{parity}"));
-        }
-        let storage_class = crate::config::storageclass::lookup_config_for_pools_without_env(&storage_class_kvs, drives_per_pool)
-            .expect("storage class should match every test erasure set");
+        let storage_class =
+            crate::config::storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[4, 4])
+                .expect("multi-pool storage class should match both four-disk pools");
         for pool in &ecstore.pools {
             for set in &pool.disk_set {
                 set.set_test_storage_class_config(storage_class.clone());
@@ -2154,98 +2050,6 @@ mod tests {
             .expect("metadata initialization should recreate the bucket volume in the new pool");
     }
 
-    #[tokio::test]
-    #[serial]
-    async fn bucket_metadata_init_repairs_half_created_expansion_pool() {
-        // Check both pool orders: an incomplete set must not hide a later
-        // complete set, and a complete set must not weaken request validation.
-        for complete_pool in 0..2 {
-            let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
-            let bucket = format!("partial-expansion-{}", Uuid::new_v4().simple());
-            for pool_index in 0..2 {
-                let present_disks = if pool_index == complete_pool { 4 } else { 2 };
-                for disk_index in 0..present_disks {
-                    tokio::fs::create_dir(
-                        temp_dir
-                            .path()
-                            .join(format!("pool{pool_index}-disk{disk_index}"))
-                            .join(&bucket),
-                    )
-                    .await
-                    .expect("fixture bucket volume should be created");
-                }
-            }
-            assert_eq!(
-                ecstore
-                    .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
-                    .await
-                    .expect_err("request validation must reject a half-created expansion set"),
-                StorageError::ErasureWriteQuorum
-            );
-
-            metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
-
-            for pool_index in 0..2 {
-                for disk_index in 0..4 {
-                    assert!(
-                        temp_dir
-                            .path()
-                            .join(format!("pool{pool_index}-disk{disk_index}"))
-                            .join(&bucket)
-                            .is_dir(),
-                        "metadata initialization must heal every missing expansion volume"
-                    );
-                }
-            }
-            ecstore
-                .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
-                .await
-                .expect("strict request validation should succeed after volume repair");
-        }
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn bucket_metadata_init_does_not_combine_partial_set_evidence() {
-        let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
-        let bucket = format!("no-quorum-expansion-{}", Uuid::new_v4().simple());
-        for pool_index in 0..2 {
-            for disk_index in 0..2 {
-                tokio::fs::create_dir(
-                    temp_dir
-                        .path()
-                        .join(format!("pool{pool_index}-disk{disk_index}"))
-                        .join(&bucket),
-                )
-                .await
-                .expect("fixture bucket volume should be created");
-            }
-        }
-        assert_eq!(
-            ecstore
-                .bucket_exists_for_heal(&bucket)
-                .await
-                .expect_err("repair must require a complete quorum within one set"),
-            StorageError::ErasureWriteQuorum
-        );
-
-        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
-
-        for pool_index in 0..2 {
-            for disk_index in 0..4 {
-                assert_eq!(
-                    temp_dir
-                        .path()
-                        .join(format!("pool{pool_index}-disk{disk_index}"))
-                        .join(&bucket)
-                        .is_dir(),
-                    disk_index < 2,
-                    "unproven bucket generations must not recreate missing volumes"
-                );
-            }
-        }
-    }
-
     #[tokio::test]
     #[serial]
     async fn bucket_metadata_init_does_not_recreate_stale_bucket_name() {
@@ -2263,218 +2067,6 @@ mod tests {
         }
     }
 
-    #[tokio::test]
-    #[serial]
-    async fn bucket_info_read_quorum_tracks_erasure_layout() {
-        for (drive_count, parity) in [(2, 1), (3, 1), (4, 2), (5, 2), (6, 3), (8, 4), (6, 2), (12, 6)] {
-            let (_temp_dir, store) = setup_bucket_quorum_test_env(&[drive_count], Some(parity)).await;
-            metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
-            let bucket = format!("read-quorum-{drive_count}-{parity}");
-            let object = "uncached-object";
-            let body = b"erasure read quorum must follow the persisted layout".repeat(32_768);
-            store
-                .make_bucket(&bucket, &MakeBucketOptions::default())
-                .await
-                .expect("healthy namespace should accept bucket creation");
-            store
-                .put_object(&bucket, object, &mut PutObjReader::from_vec(body.clone()), &ObjectOptions::default())
-                .await
-                .expect("healthy erasure set should accept the seed object");
-            let set = &store.pools[0].disk_set[0];
-            let lock = set
-                .new_ns_lock(&bucket, object)
-                .await
-                .expect("seed namespace lock should resolve");
-            drop(
-                lock.get_write_lock(Duration::from_secs(30))
-                    .await
-                    .expect("seed physical fanout must finish before taking disks offline"),
-            );
-            if (drive_count, parity) == (6, 3) {
-                let mut kvs = rustfs_config::server_config::KVS::new();
-                kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), "EC:2".to_string());
-                set.set_test_storage_class_config(
-                    crate::config::storageclass::lookup_config_for_pools_without_env(&kvs, &[drive_count])
-                        .expect("a later storage-class change must not raise old objects' read quorum"),
-                );
-            }
-
-            let offline_indexes = (0..parity).collect::<Vec<_>>();
-            let offline = take_set_disks_offline(&store, set, &offline_indexes).await;
-            let info = store
-                .get_bucket_info(&bucket, &BucketOptions::default())
-                .await
-                .expect("bucket validation must admit the object's exact read quorum");
-            assert_eq!(info.name, bucket);
-
-            let mut reader = store
-                .get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
-                .await
-                .expect("the persisted layout should remain readable at its exact data-shard quorum");
-            let mut restored = Vec::new();
-            reader
-                .stream
-                .read_to_end(&mut restored)
-                .await
-                .expect("quorum read should reconstruct the body");
-            assert_eq!(restored, body, "layout {drive_count}/{parity} must retain exact object contents");
-            drop(reader);
-
-            if drive_count - parity == drive_count / 2 {
-                let error = store
-                    .get_bucket_info_from_sets(&bucket, &BucketOptions::default())
-                    .await
-                    .expect_err("bucket mutations must retain their majority namespace check");
-                assert_eq!(error, StorageError::ErasureWriteQuorum);
-            }
-
-            let below_quorum = take_set_disks_offline(&store, set, &[parity]).await;
-            let read = store
-                .get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
-                .await;
-            match read {
-                Ok(mut reader) => assert!(
-                    reader.stream.read_to_end(&mut Vec::new()).await.is_err(),
-                    "layout {drive_count}/{parity} must reject fewer than its data-shard quorum"
-                ),
-                Err(error) => assert!(
-                    matches!(error, StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _)),
-                    "a missing shard must report read quorum loss, got {error}"
-                ),
-            }
-            restore_set_disks(&store, set, below_quorum).await;
-            restore_set_disks(&store, set, offline).await;
-        }
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn bucket_info_read_quorum_is_scoped_to_each_erasure_set() {
-        let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4, 6], None).await;
-        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
-        let bucket = "read-quorum-mixed-pools";
-        store
-            .make_bucket(bucket, &MakeBucketOptions::default())
-            .await
-            .expect("healthy pools should accept bucket creation");
-
-        let first_set = &store.pools[0].disk_set[0];
-        let second_set = &store.pools[1].disk_set[0];
-        let first_offline = take_set_disks_offline(&store, first_set, &[0, 1]).await;
-        let second_offline = take_set_disks_offline(&store, second_set, &[0, 1, 2]).await;
-        store
-            .get_bucket_info(bucket, &BucketOptions::default())
-            .await
-            .expect("each set independently satisfies its namespace read quorum");
-
-        for (set, extra_disk) in [(first_set, 2), (second_set, 3)] {
-            let extra_offline = take_set_disks_offline(&store, set, &[extra_disk]).await;
-            assert_eq!(
-                store
-                    .get_bucket_info(bucket, &BucketOptions::default())
-                    .await
-                    .expect_err("another pool must not subsidize a set below its read quorum"),
-                StorageError::ErasureReadQuorum
-            );
-            restore_set_disks(&store, set, extra_offline).await;
-        }
-        restore_set_disks(&store, first_set, first_offline).await;
-        restore_set_disks(&store, second_set, second_offline).await;
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn bucket_info_read_quorum_requires_authoritative_metadata() {
-        for state in ["missing", "corrupt", "foreign", "incarnation"] {
-            let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await;
-            metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
-            let bucket = format!("read-quorum-{state}-metadata");
-            let mut metadata = if state == "missing" {
-                store
-                    .make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
-                    .await
-                    .expect("simulate directories left before bucket metadata is published");
-                BucketMetadata::new(&bucket)
-            } else {
-                store
-                    .make_bucket(&bucket, &MakeBucketOptions::default())
-                    .await
-                    .expect("healthy bucket should publish metadata");
-                metadata_sys::get_in(&store.ctx, &bucket)
-                    .await
-                    .expect("seed metadata should be cached")
-                    .as_ref()
-                    .clone()
-            };
-            let path = metadata.save_file_path();
-            match state {
-                "corrupt" => crate::config::com::save_config(store.clone(), &path, b"corrupt".to_vec())
-                    .await
-                    .expect("persist corrupt metadata while the cached copy remains valid"),
-                "foreign" => {
-                    metadata.name = "different-bucket".to_string();
-                    let mut encoded = vec![1, 0, 1, 0];
-                    encoded.extend(metadata.marshal_msg().expect("foreign metadata should encode"));
-                    crate::config::com::save_config(store.clone(), &path, encoded)
-                        .await
-                        .expect("persist metadata for a different bucket at the requested path");
-                }
-                "incarnation" => crate::bucket::metadata::save_bucket_incarnation(store.clone(), &bucket, Uuid::new_v4())
-                    .await
-                    .expect("persist a different bucket generation"),
-                _ => {}
-            }
-
-            let set = &store.pools[0].disk_set[0];
-            let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
-            let error = store
-                .get_bucket_info(&bucket, &BucketOptions::default())
-                .await
-                .expect_err("read admission must not trust residual directories or cached metadata");
-            match state {
-                "missing" => assert_eq!(error, StorageError::ErasureReadQuorum),
-                "foreign" => assert_eq!(error, StorageError::FileCorrupt),
-                "incarnation" => assert!(error.to_string().contains("sidecar does not match bucket metadata")),
-                "corrupt" => assert!(error.to_string().contains("format invalid"), "unexpected corruption error: {error}"),
-                _ => unreachable!(),
-            }
-            restore_set_disks(&store, set, offline).await;
-        }
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn bucket_info_read_quorum_accepts_persisted_legacy_metadata() {
-        let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await;
-        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
-        let bucket = "interop";
-        store
-            .make_bucket_on_sets(bucket, &MakeBucketOptions::default())
-            .await
-            .expect("legacy bucket directories should exist");
-        let hex = include_str!("../../tests/fixtures/minio/bucket_metadata.blob.hex")
-            .split_whitespace()
-            .collect::<String>();
-        let body = (0..hex.len())
-            .step_by(2)
-            .map(|index| u8::from_str_radix(&hex[index..index + 2], 16).expect("pinned MinIO metadata fixture"))
-            .collect();
-        crate::config::com::save_config(store.clone(), &BucketMetadata::new(bucket).save_file_path(), body)
-            .await
-            .expect("legacy metadata should be persisted without an incarnation sidecar");
-
-        let set = &store.pools[0].disk_set[0];
-        let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
-        let info = store
-            .get_bucket_info(bucket, &BucketOptions::default())
-            .await
-            .expect("persisted MinIO metadata should authorize reads at the namespace read quorum");
-        assert_eq!(info.name, bucket);
-        assert!(info.versioning);
-        assert!(info.object_locking);
-        restore_set_disks(&store, set, offline).await;
-    }
-
     #[tokio::test]
     #[serial]
     async fn bucket_namespace_reads_report_missing_when_every_set_is_absent() {
@@ -2498,7 +2090,6 @@ mod tests {
     #[serial]
     async fn bucket_namespace_reads_fail_closed_when_any_set_loses_quorum() {
         let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
-        metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
         let bucket = format!("degraded-expansion-{}", Uuid::new_v4().simple());
         ecstore.pools[0].disk_set[0]
             .make_bucket(&bucket, &MakeBucketOptions::default())
@@ -2506,7 +2097,6 @@ mod tests {
             .expect("bucket should be created in the original pool only");
         ecstore.pools[1].disk_set[0].disks.write().await[0] = None;
         ecstore.pools[1].disk_set[0].disks.write().await[1] = None;
-        ecstore.pools[1].disk_set[0].disks.write().await[2] = None;
 
         let list_err = ecstore
             .list_bucket(&BucketOptions::default())
@@ -2518,7 +2108,7 @@ mod tests {
             .get_bucket_info(&bucket, &BucketOptions::default())
             .await
             .expect_err("bucket validation must fail when an expansion pool is unavailable");
-        assert_eq!(info_err, StorageError::ErasureReadQuorum);
+        assert_eq!(info_err, StorageError::ErasureWriteQuorum);
     }
 
     #[tokio::test]

crates/ecstore/src/store/init.rs

@@ -95,6 +95,7 @@ fn preflight_startup_rpc_secret_with(
     }
 }
 
+const LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY: Duration = Duration::from_secs(60 * 3);
 const LOCAL_DECOMMISSION_RESUME_RETRY_DELAY: Duration = Duration::from_secs(30);
 const LOCAL_DECOMMISSION_WATCHDOG_INTERVAL: Duration = Duration::from_secs(30);
 const LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY: Duration = Duration::from_secs(60 * 5);
@@ -279,26 +280,20 @@ where
     }
 }
 
-async fn reconcile_local_decommission_after_init(store: &Arc<ECStore>, rx: CancellationToken) -> Result<()> {
-    store
-        .ensure_pool_meta_side_effects_safe("decommission worker recovery blocked while pool metadata requires recovery")
-        .await?;
-    if store.has_active_local_decommission_worker().await {
-        return Ok(());
-    }
-    store.refresh_pool_status_meta().await?;
-    let resume_required = pool_meta_has_active_decommission(&*store.pool_meta.read().await);
-    if resume_required {
-        crate::core::pools::acquire_pool_activation_fleet_proof(&store.ctx).await?;
-    }
-    store.spawn_missing_local_decommission_routines_with_token(rx).await
-}
-
 async fn supervise_local_decommission_after_init(store: Arc<ECStore>, rx: CancellationToken) {
     run_local_decommission_watchdog(rx.clone(), || {
         let store = store.clone();
         let worker_rx = rx.clone();
-        async move { reconcile_local_decommission_after_init(&store, worker_rx).await }
+        async move {
+            store
+                .ensure_pool_meta_side_effects_safe("decommission worker recovery blocked while pool metadata requires recovery")
+                .await?;
+            if store.has_active_local_decommission_worker().await {
+                return Ok(());
+            }
+            store.refresh_pool_status_meta().await?;
+            store.spawn_missing_local_decommission_routines_with_token(worker_rx).await
+        }
     })
     .await;
 }
@@ -789,9 +784,14 @@ impl ECStore {
             );
         }
         if has_local_decommission_leadership {
-            // The watchdog checks recovery safety and retries transient failures.
-            // Resume persisted work without an unconditional cold-start delay.
-            tokio::spawn(supervise_local_decommission_after_init(self.clone(), rx.clone()));
+            let store = self.clone();
+            let decommission_rx = rx.clone();
+            tokio::spawn(async move {
+                if !wait_for_local_decommission_resume_delay(&decommission_rx, LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY).await {
+                    return;
+                }
+                supervise_local_decommission_after_init(store, decommission_rx).await;
+            });
         }
 
         let recovery_store = self.clone();
@@ -2104,44 +2104,6 @@ mod tests {
         );
     }
 
-    #[tokio::test(start_paused = true)]
-    async fn test_local_decommission_watchdog_cancelled_start_does_not_reconcile() {
-        let rx = CancellationToken::new();
-        rx.cancel();
-        run_local_decommission_watchdog(rx, || async {
-            panic!("cancelled startup must not schedule persisted work");
-        })
-        .await;
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn test_local_decommission_recovery_waits_for_live_fleet_proof_before_reserving_worker() {
-        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
-        mark_test_pool_decommissioning(&store, 0).await;
-        assert!(store.ctx.is_dist_erasure().await);
-        let worker_rx = CancellationToken::new();
-
-        {
-            let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
-            let err = super::reconcile_local_decommission_after_init(&store, worker_rx.clone())
-                .await
-                .expect_err("cold distributed recovery must wait for live fleet proof");
-            assert!(
-                crate::core::pools::is_pool_activation_fleet_proof_error(&err),
-                "recovery must reach the live fleet proof gate: {err:?}"
-            );
-            assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
-            assert!(pool_meta_has_active_decommission(&*store.pool_meta.read().await));
-        }
-
-        super::reconcile_local_decommission_after_init(&store, worker_rx.clone())
-            .await
-            .expect("restored fleet proof should admit the persisted worker");
-        assert!(store.has_active_local_decommission_worker().await);
-        worker_rx.cancel();
-    }
-
     #[tokio::test(start_paused = true)]
     async fn test_local_decommission_watchdog_retries_general_failures_until_cancelled() {
         let rx = CancellationToken::new();
@@ -2153,13 +2115,11 @@ mod tests {
                 let attempts = attempts.clone();
                 let rx = rx.clone();
                 async move {
-                    match attempts.fetch_add(1, Ordering::SeqCst) {
-                        0 => Err(StorageError::other("pool activation requires a live fleet capability proof")),
-                        1 => Err(StorageError::SlowDown),
-                        _ => {
-                            rx.cancel();
-                            Ok(())
-                        }
+                    if attempts.fetch_add(1, Ordering::SeqCst) == 0 {
+                        Err(StorageError::SlowDown)
+                    } else {
+                        rx.cancel();
+                        Ok(())
                     }
                 }
             }
@@ -2168,11 +2128,8 @@ mod tests {
         tokio::task::yield_now().await;
         assert_eq!(attempts.load(Ordering::SeqCst), 1);
         tokio::time::advance(LOCAL_DECOMMISSION_RESUME_RETRY_DELAY).await;
-        tokio::task::yield_now().await;
-        assert_eq!(attempts.load(Ordering::SeqCst), 2);
-        tokio::time::advance(local_decommission_watchdog_retry_delay(2)).await;
         task.await.expect("watchdog task should exit after cancellation");
-        assert_eq!(attempts.load(Ordering::SeqCst), 3);
+        assert_eq!(attempts.load(Ordering::SeqCst), 2);
     }
 
     #[tokio::test(start_paused = true)]
@@ -3707,31 +3664,6 @@ mod tests {
         OfflineTestDisks { disks }
     }
 
-    #[cfg(feature = "test-util")]
-    async fn force_set_disk_range_offline_for_test(
-        set: &Arc<crate::set_disk::SetDisks>,
-        range: std::ops::Range<usize>,
-    ) -> OfflineTestDisks {
-        let disks = set
-            .disks
-            .read()
-            .await
-            .get(range)
-            .expect("offline test range must fit the set")
-            .iter()
-            .map(|disk| disk.clone().expect("fault-injection disk should start online"))
-            .collect::<Vec<_>>();
-        for disk in &disks {
-            disk.close().await.expect("fault injection should stop per-disk monitoring");
-            disk.force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Offline);
-        }
-        set.connect_disks().await;
-        for disk in &disks {
-            assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
-        }
-        OfflineTestDisks { disks }
-    }
-
     fn active_rebalance_meta_for_pool(pool_count: usize, active_pool_idx: usize) -> RebalanceMeta {
         let now = OffsetDateTime::now_utc();
         let mut pool_stats = vec![RebalanceStats::default(); pool_count];
@@ -12828,422 +12760,6 @@ mod tests {
         body
     }
 
-    #[cfg(feature = "test-util")]
-    #[test]
-    #[serial_test::serial(storage_class_env)]
-    fn legacy_transition_state_inspection_and_apply_keep_all_disk_copies_unchanged() {
-        run_large_stack_async_test("legacy-state-reconcile-inspection", || {
-            legacy_transition_state_inspection_and_apply_case(false)
-        });
-    }
-
-    #[cfg(feature = "test-util")]
-    #[cfg(not(windows))]
-    #[test]
-    #[serial_test::serial(storage_class_env)]
-    fn legacy_transition_state_backfill_retries_partial_commits_and_preserves_other_bytes() {
-        run_large_stack_async_test("legacy-state-reconcile-backfill", || {
-            legacy_transition_state_inspection_and_apply_case(true)
-        });
-    }
-
-    #[cfg(feature = "test-util")]
-    async fn legacy_transition_state_inspection_and_apply_case(write_enabled: bool) {
-        #[cfg(windows)]
-        assert!(!write_enabled, "Windows supports inspection but cannot prove repair directory durability");
-        use crate::bucket::lifecycle::legacy_transition_state_reconcile::{
-            LegacyTransitionStateReconcileOutcome as Outcome, LegacyTransitionStateReconcileRequest,
-            LegacyTransitionStateReconcileSelector,
-        };
-        for (remote_version, expected_state) in [
-            ("", rustfs_filemeta::TransitionVersionState::KnownDisabled),
-            ("null", rustfs_filemeta::TransitionVersionState::SuspendedNull),
-            ("opaque-version", rustfs_filemeta::TransitionVersionState::Exact),
-        ] {
-            let temp_dir = tempfile::tempdir().expect("legacy reconcile store directory");
-            let (ctx, store, _shutdown) =
-                without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-state-reconcile-inspect", &[4]))
-                    .await;
-            crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
-            let tier_name = "LEGACY-RECONCILE";
-            let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
-            backend.set_put_remote_version(Some(remote_version.to_string())).await;
-            let bucket = "legacy-state-reconcile-bucket";
-            let object = "archive.bin";
-            store
-                .make_bucket(bucket, &MakeBucketOptions::default())
-                .await
-                .expect("create legacy fixture bucket");
-            let mut reader = PutObjReader::from_vec(b"legacy reconcile body".repeat(1024));
-            let source = store
-                .put_object(bucket, object, &mut reader, &ObjectOptions::default())
-                .await
-                .expect("write source");
-            {
-                // Create the fixture under the existing remote-version writer
-                // gate. This does not authorize legacy metadata reconciliation.
-                let _proof = crate::services::notification_sys::install_current_remote_version_state_fleet_proof_for_test();
-                temp_env::async_with_vars(
-                    [
-                        (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, Some("true")),
-                        (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, Some("true")),
-                    ],
-                    store.transition_object(
-                        bucket,
-                        object,
-                        &ObjectOptions {
-                            transition: TransitionOptions {
-                                status: TRANSITION_PENDING.to_string(),
-                                tier: tier_name.to_string(),
-                                etag: source.etag.clone().expect("source ETag"),
-                                ..Default::default()
-                            },
-                            mod_time: source.mod_time,
-                            ..Default::default()
-                        },
-                    ),
-                )
-                .await
-                .expect("transition source");
-            }
-            assert!(
-                crate::services::notification_sys::acquire_legacy_transition_state_reconcile_fleet_proof()
-                    .await
-                    .is_none(),
-                "fixture setup must not grant the missing reconciliation write capability"
-            );
-            let selector = LegacyTransitionStateReconcileSelector {
-                bucket: bucket.to_string(),
-                object: object.to_string(),
-                version_id: "null".to_string(),
-            };
-            if expected_state == rustfs_filemeta::TransitionVersionState::Exact {
-                backend
-                    .set_transition_candidate_probe_override(Some(
-                        crate::services::tier::warm_backend::TransitionCandidateProbe::Ambiguous,
-                    ))
-                    .await;
-            }
-            let converged = store
-                .inspect_legacy_transition_state(selector.clone())
-                .await
-                .expect("inspect an already explicit transition");
-            assert_eq!(converged.outcome, Outcome::Migrated, "{converged:?}");
-            assert!(!converged.changed);
-            backend.set_transition_candidate_probe_override(None).await;
-            rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, remote_version.is_empty()).await;
-            let paths = (0..4)
-                .map(|disk| {
-                    temp_dir
-                        .path()
-                        .join(format!("pool0/set0/disk{disk}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"))
-                })
-                .collect::<Vec<_>>();
-            let mut original = Vec::new();
-            for path in &paths {
-                original.push(tokio::fs::read(path).await.expect("original xl.meta"));
-            }
-            backend.clear_op_log().await;
-            let inspection = store.inspect_legacy_transition_state(selector.clone());
-            assert!(
-                std::mem::size_of_val(&inspection) <= 4 * 1024,
-                "admin inspection future must remain stack-bounded"
-            );
-            let inspected = inspection.await.expect("inspect legacy state");
-            assert_eq!(inspected.outcome, Outcome::ReadyToMigrate, "{inspected:?}");
-            assert!(!inspected.readiness.post_ready, "current fleet cannot authorize conditional writes");
-            let target = inspected.target.expect("live probe should establish one model");
-            assert_eq!(target.state, expected_state);
-            let request = LegacyTransitionStateReconcileRequest {
-                confirm: true,
-                selector,
-                source: inspected.source.expect("immutable source"),
-                original_sets: inspected.original_sets,
-                target,
-                reconciliation_digest: inspected.reconciliation_digest.expect("expected tuple digest"),
-            };
-            #[cfg(not(windows))]
-            if write_enabled {
-                crate::services::notification_sys::with_legacy_transition_state_fleet_proof_for_test(async {
-                    crate::disk::local::bucket_durability::set(bucket, Some(crate::disk::local::DurabilityMode::None));
-                    let unsynced = store.reconcile_legacy_transition_state(request.clone()).await;
-                    crate::disk::local::bucket_durability::set(bucket, None);
-                    let unsynced = unsynced.expect("repair without metadata durability");
-                    assert_eq!(unsynced.outcome, Outcome::BackendUnavailable);
-                    assert!(!unsynced.changed);
-                    let rollback = paths[0].parent().expect("object directory").join(Uuid::new_v4().to_string());
-                    tokio::fs::create_dir(&rollback).await.expect("pending rollback directory");
-                    tokio::fs::write(rollback.join(crate::disk::STORAGE_FORMAT_FILE_BACKUP), &original[0])
-                        .await
-                        .expect("pending old metadata backup");
-                    let unsettled = store.reconcile_legacy_transition_state(request.clone()).await;
-                    tokio::fs::remove_dir_all(&rollback).await.expect("settle fixture rollback");
-                    let unsettled = unsettled.expect("repair must wait for rollback");
-                    assert_eq!(unsettled.outcome, Outcome::BackendUnavailable);
-                    assert!(!unsettled.changed);
-                    for (path, bytes) in paths.iter().zip(&original) {
-                        assert_eq!(tokio::fs::read(path).await.expect("blocked repair leaves original bytes"), *bytes);
-                    }
-                    // The first disk commits; the second stops after staging.
-                    // This models an interrupted cross-disk effect without rollback.
-                    let disks = store.all_set_disks()[0].disk_inventory().await;
-                    let first_disk = disks[0].as_ref().expect("first physical disk");
-                    let publication_path = first_disk
-                        .get_object_path_for_io_if_local(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}"))
-                        .expect("local disk")
-                        .expect("publication path");
-                    let crash_key = format!("{object}/{STORAGE_FORMAT_FILE}");
-                    let _hook = crate::disk::os::prepared_publication_test_hooks::install_at(
-                        crate::disk::os::prepared_publication_test_hooks::Stage::Rename,
-                        &publication_path,
-                        move || {
-                            crate::crash_inject::arm(crate::crash_inject::CrashPoint::MetaWriteAfterTmpBeforeRename, &crash_key);
-                        },
-                    );
-                    let partial = store
-                        .reconcile_legacy_transition_state(request.clone())
-                        .await
-                        .expect("partial repair response");
-                    assert_eq!(partial.outcome, Outcome::BackendUnavailable, "{partial:?}");
-                    assert!(partial.changed, "first copy was committed: {partial:?}");
-                    assert!(partial.changes_indeterminate);
-                    assert_ne!(tokio::fs::read(&paths[0]).await.expect("first committed copy"), original[0]);
-                    for (path, bytes) in paths[1..].iter().zip(&original[1..]) {
-                        assert_eq!(tokio::fs::read(path).await.expect("uncommitted copy"), *bytes);
-                    }
-                    assert!(
-                        store.all_set_disks()[0]
-                            .load_file_info_versions_exact(bucket, object)
-                            .await
-                            .is_err(),
-                        "cleanup cannot select a partial repair subset"
-                    );
-                    let repaired = store
-                        .reconcile_legacy_transition_state(request.clone())
-                        .await
-                        .expect("retry original snapshot");
-                    assert_eq!(repaired.outcome, Outcome::Migrated, "{repaired:?}");
-                    assert!(repaired.changed);
-                    assert!(!repaired.changes_indeterminate);
-                    let mut committed = Vec::new();
-                    for (path, original) in paths.iter().zip(&original) {
-                        let raw = tokio::fs::read(path).await.expect("repaired copy");
-                        let metadata = FileMeta::load(&raw).expect("decode repaired copy");
-                        let previous = FileMeta::load(original).expect("decode original copy");
-                        assert_eq!(
-                            metadata.transition_reconcile_generation(None).unwrap(),
-                            previous.transition_reconcile_generation(None).unwrap()
-                        );
-                        let (_, version) = metadata.find_version(None).expect("selected version");
-                        let info = version.into_fileinfo(bucket, object, true).expect("repaired FileInfo");
-                        assert_eq!(info.transition_version_state, expected_state);
-                        assert_eq!(info.transition_version, request.target.remote_version);
-                        committed.push(raw);
-                    }
-                    assert!(
-                        store.all_set_disks()[0]
-                            .load_file_info_versions_exact(bucket, object)
-                            .await
-                            .expect("converged cleanup snapshot")
-                            .is_some()
-                    );
-                    let replay = store
-                        .reconcile_legacy_transition_state(request.clone())
-                        .await
-                        .expect("idempotent original request replay");
-                    assert_eq!(replay.outcome, Outcome::Migrated, "{replay:?}");
-                    assert!(!replay.changed);
-                    for (path, expected) in paths.iter().zip(&committed) {
-                        assert_eq!(
-                            tokio::fs::read(path).await.expect("replayed copy"),
-                            *expected,
-                            "idempotence preserves raw encoding"
-                        );
-                    }
-                    for (path, bytes) in paths.iter().zip(&original) {
-                        tokio::fs::write(path, bytes)
-                            .await
-                            .expect("reset independent cancellation fixture");
-                    }
-                    let (entered_tx, entered) = tokio::sync::oneshot::channel();
-                    let (release, released) = std::sync::mpsc::channel::<()>();
-                    let _pause = crate::disk::os::prepared_publication_test_hooks::install_at(
-                        crate::disk::os::prepared_publication_test_hooks::Stage::Rename,
-                        &publication_path,
-                        move || {
-                            let _ = entered_tx.send(());
-                            let _ = released.recv();
-                        },
-                    );
-                    let mut repair = Box::pin(store.reconcile_legacy_transition_state(request.clone()));
-                    tokio::select! {
-                        result = &mut repair => panic!("repair completed before publication pause: {result:?}"),
-                        result = entered => result.expect("publication executor entered"),
-                    }
-                    let update_options = crate::disk::UpdateMetadataOpts::default();
-                    let mut update = Box::pin(first_disk.update_metadata(
-                        bucket,
-                        object,
-                        FileInfo {
-                            metadata: HashMap::from([("x-amz-meta-concurrent".to_string(), "kept".to_string())]),
-                            ..Default::default()
-                        },
-                        &update_options,
-                    ));
-                    assert!(
-                        tokio::time::timeout(std::time::Duration::from_millis(25), update.as_mut())
-                            .await
-                            .is_err(),
-                        "another metadata RMW must wait for publication"
-                    );
-                    drop(repair);
-                    assert!(
-                        tokio::time::timeout(std::time::Duration::from_millis(25), update.as_mut())
-                            .await
-                            .is_err(),
-                        "cancelling the coordinator must not release an in-flight disk mutation"
-                    );
-                    release.send(()).expect("resume owned publication");
-                    update.await.expect("serialized metadata update");
-                    let raw = tokio::fs::read(&paths[0])
-                        .await
-                        .expect("cancelled repair and later metadata update");
-                    let (_, version) = FileMeta::load(&raw)
-                        .expect("metadata after cancellation")
-                        .find_version(None)
-                        .expect("selected version");
-                    let info = version
-                        .into_fileinfo(bucket, object, true)
-                        .expect("metadata after serialized update");
-                    assert_eq!(info.transition_version_state, expected_state);
-                    assert_eq!(info.metadata.get("x-amz-meta-concurrent").map(String::as_str), Some("kept"));
-                    let stale = store
-                        .reconcile_legacy_transition_state(request.clone())
-                        .await
-                        .expect("stale original request");
-                    assert_eq!(
-                        stale.outcome,
-                        Outcome::Corrupt,
-                        "unrelated metadata change invalidates the original generation: {stale:?}"
-                    );
-                    assert!(!stale.changed);
-                    assert_eq!(tokio::fs::read(&paths[0]).await.expect("stale write leaves bytes unchanged"), raw);
-                    assert_eq!(backend.remove_count().await, 0);
-                    // A pinned version probe uses GET to verify that exact
-                    // candidate; every backend operation still targets it.
-                    let operations = backend.op_log().await;
-                    assert!(
-                        operations.iter().all(|operation| match operation {
-                            MockWarmOp::Probe { object } | MockWarmOp::Get { object } => object == &request.source.remote_object,
-                            _ => false,
-                        }),
-                        "unexpected backend effects: {operations:?}"
-                    );
-                })
-                .await;
-                continue;
-            }
-            let mut tampered = request.clone();
-            tampered.source.remote_object.push_str("-other");
-            let probes_before = backend.op_log().await.len();
-            let rejected = store
-                .reconcile_legacy_transition_state(tampered)
-                .await
-                .expect("reject tampered tuple");
-            assert_eq!(rejected.outcome, Outcome::Corrupt);
-            assert_eq!(backend.op_log().await.len(), probes_before, "invalid digest must not probe the backend");
-            let applied = store
-                .reconcile_legacy_transition_state(request)
-                .await
-                .expect("apply must report unavailable write authority");
-            assert_eq!(applied.outcome, Outcome::BackendUnavailable, "{applied:?}");
-            assert_eq!(applied.reason_code, "write_fence_unavailable");
-            assert!(!applied.changed);
-            for (path, expected) in paths.iter().zip(&original) {
-                assert_eq!(tokio::fs::read(path).await.expect("xl.meta after inspection"), *expected);
-            }
-            assert_eq!(backend.remove_count().await, 0);
-            assert!(
-                backend
-                    .op_log()
-                    .await
-                    .iter()
-                    .all(|operation| matches!(operation, MockWarmOp::Probe { .. }))
-            );
-
-            backend.set_unreachable(true).await;
-            let unavailable = store
-                .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
-                    bucket: bucket.to_string(),
-                    object: object.to_string(),
-                    version_id: "null".to_string(),
-                })
-                .await
-                .expect("unreachable tier is a diagnostic outcome");
-            assert_eq!(unavailable.outcome, Outcome::BackendUnavailable);
-            assert!(!unavailable.changed);
-            backend.set_unreachable(false).await;
-            for candidate in ["", "00000000-0000-0000-0000-000000000000", "bad\nversion"] {
-                backend
-                    .set_transition_candidate_probe_override(Some(
-                        crate::services::tier::warm_backend::TransitionCandidateProbe::VersionedPresent(candidate.to_string()),
-                    ))
-                    .await;
-                let invalid_proof = store
-                    .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
-                        bucket: bucket.to_string(),
-                        object: object.to_string(),
-                        version_id: "null".to_string(),
-                    })
-                    .await
-                    .expect("invalid backend proof is a diagnostic outcome");
-                assert_eq!(invalid_proof.outcome, Outcome::BackendUnavailable, "{invalid_proof:?}");
-                assert!(invalid_proof.target.is_none());
-            }
-            backend.set_transition_candidate_probe_override(None).await;
-
-            backend.clear_op_log().await;
-            for path in &paths[1..] {
-                tokio::fs::remove_file(path)
-                    .await
-                    .expect("hide majority metadata copies in fixture");
-            }
-            let minority = store
-                .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
-                    bucket: bucket.to_string(),
-                    object: object.to_string(),
-                    version_id: "null".to_string(),
-                })
-                .await
-                .expect("inspect minority legacy record");
-            assert_eq!(
-                minority.outcome,
-                Outcome::BackendUnavailable,
-                "a minority owner must remain visible: {minority:?}"
-            );
-            assert!(
-                backend.op_log().await.is_empty(),
-                "unproven metadata quorum cannot initiate a remote probe"
-            );
-            for (path, bytes) in paths.iter().zip(&original) {
-                tokio::fs::write(path, bytes).await.expect("restore fixture copies");
-            }
-            tokio::fs::write(&paths[0], b"corrupt-xl-meta")
-                .await
-                .expect("inject corrupt copy");
-            let corrupt = store
-                .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
-                    bucket: bucket.to_string(),
-                    object: object.to_string(),
-                    version_id: "null".to_string(),
-                })
-                .await
-                .expect("inspect corrupt legacy record");
-            assert_eq!(corrupt.outcome, Outcome::Corrupt, "{corrupt:?}");
-            assert!(backend.op_log().await.is_empty(), "corruption must fail before backend I/O");
-        }
-    }
-
     #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
@@ -15686,24 +15202,25 @@ mod tests {
         assert!(deleted[0].found, "the aggregate error must retain the committed pool result");
         drop(injection);
 
-        // Exact reads can see subquorum metadata while workers remove each
-        // disk's free version. Inspect the final state after cleanup drains.
-        wait_for_expiry_workers_idle(&store).await;
-        for pool in &store.pools {
-            assert!(
-                pool.get_disks_by_key(object)
-                    .load_file_info_versions_exact(bucket, object)
-                    .await
-                    .expect("aggregate-error cleanup metadata should remain readable")
-                    .is_none(),
-                "aggregate failure must not suppress committed receipt cleanup"
-            );
-        }
-        assert_eq!(
-            backend.remove_count().await,
-            1,
-            "committed receipts must remove the shared remote object once"
-        );
+        tokio::time::timeout(Duration::from_secs(30), async {
+            loop {
+                let mut metadata_absent = true;
+                for pool in &store.pools {
+                    metadata_absent &= pool
+                        .get_disks_by_key(object)
+                        .load_file_info_versions_exact(bucket, object)
+                        .await
+                        .expect("aggregate-error cleanup metadata should remain readable")
+                        .is_none();
+                }
+                if metadata_absent && backend.remove_count().await == 1 {
+                    return;
+                }
+                tokio::time::sleep(Duration::from_millis(10)).await;
+            }
+        })
+        .await
+        .expect("aggregate failure must not suppress committed receipt dispatch");
         assert_eq!(backend.object_count().await, 0, "the shared remote object should be removed exactly once");
         store
             .delete_bucket(bucket, &DeleteBucketOptions::default())
@@ -17347,55 +16864,6 @@ mod tests {
         assert_eq!(body, original_body);
     }
 
-    #[cfg(feature = "test-util")]
-    #[tokio::test]
-    #[serial_test::serial(storage_class_env)]
-    async fn object_lock_snapshot_uses_read_quorum_bucket_existence_probe() {
-        let temp = tempfile::tempdir().expect("create degraded snapshot store dir");
-        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
-            temp.path(),
-            "degraded-object-lock-snapshot",
-            &[(2, 12)],
-            CancellationToken::new(),
-            None,
-        ))
-        .await;
-        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
-
-        let bucket = format!("degraded-ol-{}", uuid::Uuid::new_v4());
-        store
-            .make_bucket(&bucket, &MakeBucketOptions::default())
-            .await
-            .expect("create snapshot bucket");
-        let expected_incarnation = store
-            .bucket_incarnation_id(&bucket)
-            .await
-            .expect("read bucket incarnation before degrading sets");
-
-        let mut offline_disks = Vec::new();
-        for set in store.all_set_disks() {
-            offline_disks.push(force_set_disk_range_offline_for_test(&set, 6..12).await);
-        }
-
-        let snapshot = store
-            .object_lock_config_snapshot(&bucket)
-            .await
-            .expect("read-quorum bucket existence should admit guarded Object Lock snapshot");
-        assert!(matches!(
-            snapshot.state(),
-            crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent
-        ));
-        assert!(snapshot.is_valid_for_destructive_put(store.id, &bucket, expected_incarnation));
-
-        let current_incarnation = crate::bucket::metadata_sys::get_object_lock_config_and_incarnation_from_disk_in(&ctx, &bucket)
-            .await
-            .expect("authoritative metadata read should also survive at read quorum")
-            .1;
-        assert_eq!(current_incarnation, expected_incarnation);
-
-        drop(offline_disks);
-    }
-
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
     async fn force_create_existing_bucket_preserves_incarnation_and_inflight_request() {
@@ -19130,34 +18598,27 @@ mod tests {
                             .await
                             .expect("transition metadata should be readable");
                         let mut metadata = FileMeta::load(&encoded).expect("transition metadata should decode");
-                        let (version_index, mut transitioned) = metadata
-                            .find_version(history.version_id)
+                        let mut transitioned = metadata
+                            .get_all_file_info_versions(bucket, object, true)
+                            .expect("transitioned versions should decode")
+                            .versions
+                            .into_iter()
+                            .find(|version| version.version_id == history.version_id)
                             .expect("transitioned history should exist");
-                        // Rewrite the serialized record to model legacy metadata;
-                        // ordinary writes preserve an already reconciled state.
-                        rustfs_utils::http::metadata_compat::remove_bytes(
-                            &mut transitioned.object.as_mut().expect("history should be an object").meta_sys,
+                        transitioned.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
+                        rustfs_utils::http::metadata_compat::remove_str(
+                            &mut transitioned.metadata,
                             rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
                         );
-                        metadata.versions[version_index] = rustfs_filemeta::FileMetaShallowVersion::try_from(transitioned)
-                            .expect("legacy history should re-encode");
+                        metadata
+                            .add_version(transitioned)
+                            .expect("unknown state should replace the transitioned version");
                         tokio::fs::write(
                             &metadata_path,
                             metadata.marshal_msg().expect("unknown transition metadata should encode"),
                         )
                         .await
                         .expect("unknown transition metadata should be written");
-                        let encoded = tokio::fs::read(&metadata_path)
-                            .await
-                            .expect("legacy transition metadata should be readable");
-                        let legacy = FileMeta::load(&encoded)
-                            .expect("legacy transition metadata should decode")
-                            .find_version(history.version_id)
-                            .expect("legacy history should exist")
-                            .1
-                            .into_fileinfo(bucket, object, true)
-                            .expect("legacy history should decode");
-                        assert_eq!(legacy.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
                     }
                     let lifecycle_event = crate::bucket::lifecycle::lifecycle::Event {
                         action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,

crates/ecstore/src/store/list_objects.rs

@@ -164,16 +164,6 @@ pub fn max_keys_plus_one(max_keys: i32, add_one: bool) -> i32 {
     max_keys
 }
 
-fn list_versions_scan_limit(max_keys: i32, has_version_marker: bool) -> i32 {
-    if max_keys <= 0 {
-        return 0;
-    }
-
-    // The marker object's versions may all be filtered out after gathering.
-    // Reserve its raw entry in addition to the next-page lookahead entry.
-    max_keys_plus_one(max_keys, true) + i32::from(has_version_marker)
-}
-
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
 enum GatherResultsState {
     LimitReached,
@@ -2149,19 +2139,15 @@ fn build_list_versions_next_marker(
         // here; advertise it as the literal `null` marker so a resumed listing
         // parses it back to `VersionMarker::Null` instead of a nil UUID that
         // `find_version_index` can never match (issue #6745).
-        let version_marker = if last.is_dir && last.mod_time.is_none() {
-            // A CommonPrefix has no version to resume; a version marker would
-            // make the next page include this same prefix again.
-            None
-        } else {
+        (
+            Some(append_list_cache_id_to_marker(last.name.clone(), cache_id)),
             Some(
                 last.version_id
                     .filter(|v| !v.is_nil())
                     .map(|v| v.to_string())
                     .unwrap_or_else(|| "null".to_string()),
-            )
-        };
-        (Some(append_list_cache_id_to_marker(last.name.clone(), cache_id)), version_marker)
+            ),
+        )
     } else if let Some(last_prefix) = prefixes.last() {
         (Some(append_list_cache_id_to_marker(last_prefix.clone(), cache_id)), None)
     } else {
@@ -2880,20 +2866,6 @@ fn listing_entries_supplement_target(
         return None;
     }
 
-    if let Some(directory) = entries.0.iter().flatten().find(|entry| entry.is_dir()) {
-        let directory_copies = entries
-            .0
-            .iter()
-            .flatten()
-            .filter(|entry| entry.is_dir() && entry.name == directory.name)
-            .count();
-        // A committed child may have some of its directory copies only on
-        // fallback disks, just like object metadata in a partial primary sample.
-        if directory_copies < resolver.dir_quorum {
-            return Some(directory.name.clone());
-        }
-    }
-
     for (idx, entry) in entries.0.iter().enumerate() {
         let Some(entry) = entry.as_ref().filter(|entry| entry.is_object()) else {
             continue;
@@ -4046,7 +4018,8 @@ impl ECStore {
             None
         };
 
-        let effective_max_keys = list_versions_scan_limit(max_keys, has_version_marker);
+        let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) };
+        // Always request max_keys + 1 to detect if there are more results
         let mut opts = ListPathOptions {
             bucket: bucket.to_owned(),
             prefix: prefix.to_owned(),
@@ -5352,7 +5325,7 @@ impl Sets {
             None
         };
 
-        let effective_max_keys = list_versions_scan_limit(max_keys, has_version_marker);
+        let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) };
         let mut opts = ListPathOptions {
             bucket: bucket.to_owned(),
             prefix: prefix.to_owned(),
@@ -6061,7 +6034,7 @@ impl SetDisks {
 
         let has_version_marker = version_marker.is_some();
         let version_marker = version_marker.map(parse_version_marker).transpose()?;
-        let effective_max_keys = list_versions_scan_limit(max_keys, has_version_marker);
+        let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) };
         let mut opts = ListPathOptions {
             bucket: bucket.to_owned(),
             prefix: prefix.to_owned(),
@@ -6275,7 +6248,7 @@ impl SetDisks {
             None
         };
 
-        let effective_max_keys = list_versions_scan_limit(max_keys, has_version_marker);
+        let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) };
         let mut opts = ListPathOptions {
             bucket: bucket.to_owned(),
             prefix: prefix.to_owned(),
@@ -7468,153 +7441,6 @@ mod test {
         assert!(cancel.is_cancelled());
     }
 
-    #[test]
-    fn list_versions_pagination_scan_limit_boundaries() {
-        for has_version_marker in [false, true] {
-            assert_eq!(super::list_versions_scan_limit(-1, has_version_marker), 0);
-            assert_eq!(super::list_versions_scan_limit(0, has_version_marker), 0);
-            let marker_slot = i32::from(has_version_marker);
-            assert_eq!(super::list_versions_scan_limit(1, has_version_marker), 2 + marker_slot);
-            assert_eq!(super::list_versions_scan_limit(MAX_OBJECT_LIST, has_version_marker), 1001 + marker_slot);
-            assert_eq!(super::list_versions_scan_limit(i32::MAX, has_version_marker), 1001 + marker_slot);
-        }
-    }
-
-    #[tokio::test]
-    async fn list_versions_pagination_does_not_require_an_empty_final_page() {
-        use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
-        use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
-
-        let (dirs, store) = isolated_store_over_temp_disks().await;
-        let bucket = "version-pagination-bucket";
-        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
-        store
-            .make_bucket(bucket, &MakeBucketOptions::default())
-            .await
-            .expect("pagination bucket should be created");
-        let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
-
-        for kind in ["objects", "deletes", "null", "mixed", "delimiter"] {
-            let count = if kind == "mixed" { 5 } else { 10 };
-            let mut expected = Vec::new();
-            for index in 0..count {
-                let name = if kind == "delimiter" && index % 2 == 1 {
-                    format!("{kind}/testobject-{index:02}/child")
-                } else {
-                    format!("{kind}/testobject-{index:02}")
-                };
-                let entry = match kind {
-                    "deletes" => test_delete_marker_meta_entry(&name, mod_time),
-                    "null" => test_object_meta_entry(&name),
-                    "mixed" => test_object_with_delete_marker_meta_entry(&name, mod_time, mod_time + time::Duration::SECOND),
-                    _ => test_object_meta_entry_with_erasure_versions(&name, &[(mod_time, "etag", 2, 2)]),
-                };
-                for dir in &dirs {
-                    let object_dir = dir.path().join(bucket).join(&name);
-                    tokio::fs::create_dir_all(&object_dir)
-                        .await
-                        .expect("pagination object directory should be created");
-                    tokio::fs::write(object_dir.join(STORAGE_FORMAT_FILE), &entry.metadata)
-                        .await
-                        .expect("pagination metadata should be written");
-                }
-                if kind == "delimiter" && index % 2 == 1 {
-                    expected.push((name.trim_end_matches("child").to_owned(), None, false));
-                } else {
-                    let versions = entry.file_info_versions(bucket).expect("fixture versions should decode");
-                    expected.extend(
-                        versions
-                            .versions
-                            .iter()
-                            .map(|version| (name.clone(), version.version_id, version.deleted)),
-                    );
-                }
-            }
-            let prefix = format!("{kind}/");
-            let delimiter = (kind == "delimiter").then(|| "/".to_owned());
-            // Exercise each public/internal entry point with the reported page size.
-            // The store entry point also covers exact and one-over limit boundaries.
-            for (layer, max_keys) in [(0, 0), (0, 1), (0, 5), (0, 9), (0, 10), (0, 11), (1, 5), (2, 5), (3, 5)] {
-                if layer == 3 && delimiter.is_some() {
-                    continue;
-                }
-                let mut marker = None;
-                let mut version_marker = None;
-                let expected_pages = if max_keys == 0 {
-                    1
-                } else {
-                    10usize.div_ceil(usize::try_from(max_keys).expect("positive page size"))
-                };
-                let mut actual = Vec::new();
-                for page in 0..expected_pages {
-                    let result = match layer {
-                        0 => {
-                            store
-                                .clone()
-                                .inner_list_object_versions(bucket, &prefix, marker, version_marker, delimiter.clone(), max_keys)
-                                .await
-                        }
-                        1 => {
-                            store.pools[0]
-                                .clone()
-                                .inner_list_object_versions(bucket, &prefix, marker, version_marker, delimiter.clone(), max_keys)
-                                .await
-                        }
-                        2 => {
-                            store.pools[0].disk_set[0]
-                                .clone()
-                                .inner_list_object_versions(bucket, &prefix, marker, version_marker, delimiter.clone(), max_keys)
-                                .await
-                        }
-                        _ => {
-                            store.pools[0].disk_set[0]
-                                .clone()
-                                .inner_list_object_versions_for_recursive_delete(
-                                    bucket,
-                                    &prefix,
-                                    marker,
-                                    version_marker,
-                                    max_keys,
-                                )
-                                .await
-                        }
-                    }
-                    .expect("version page should list successfully");
-                    let page_size = usize::try_from(max_keys).expect("nonnegative page size");
-                    assert_eq!(result.objects.len() + result.prefixes.len(), (10 - page * page_size).min(page_size));
-                    let has_more = page + 1 < expected_pages;
-                    assert_eq!(result.is_truncated, has_more, "{kind}, layer {layer}, max_keys {max_keys}, page {page}");
-                    assert_eq!(
-                        result.next_marker.is_some(),
-                        has_more,
-                        "key marker must exist only when another page exists"
-                    );
-                    if !has_more {
-                        assert!(
-                            result.next_version_idmarker.is_none(),
-                            "the final page must not advertise a version marker"
-                        );
-                    }
-                    actual.extend(
-                        result
-                            .objects
-                            .into_iter()
-                            .map(|object| (object.name, object.version_id, object.delete_marker)),
-                    );
-                    actual.extend(result.prefixes.into_iter().map(|prefix| (prefix, None, false)));
-                    marker = result.next_marker;
-                    version_marker = result.next_version_idmarker;
-                }
-                // Objects and CommonPrefixes are serialized separately; compare their
-                // identities without relying on their relative position in the response.
-                actual.sort();
-                let mut expected = if max_keys == 0 { Vec::new() } else { expected.clone() };
-                expected.sort();
-                assert_eq!(actual, expected, "{kind}, layer {layer}, max_keys {max_keys}");
-            }
-        }
-    }
-
     #[test]
     fn version_marker_is_applied_only_when_key_marker_entry_is_present() {
         let version_marker = Some(VersionMarker::Null);
@@ -9622,71 +9448,6 @@ mod test {
         assert!(supplemented.is_latest_delete_marker());
     }
 
-    #[tokio::test]
-    async fn latest_listing_supplement_checks_fallback_disks_for_common_prefix_quorum() {
-        let mut fallback_disks = Vec::new();
-        let mut fallback_tempdirs = Vec::new();
-        for index in 0..4 {
-            let tempdir = tempfile::tempdir().expect("fallback tempdir should be created");
-            let endpoint = Endpoint::try_from(tempdir.path().to_str().expect("fallback path should be utf8"))
-                .expect("fallback endpoint should parse");
-            let disk = new_disk(
-                &endpoint,
-                &DiskOption {
-                    cleanup: false,
-                    health_check: false,
-                },
-            )
-            .await
-            .expect("fallback disk should be created");
-            disk.make_volume("bucket").await.expect("fallback bucket should be created");
-            for copies in [3, 4] {
-                if index < copies {
-                    let object = format!("quux-{copies}/thud");
-                    let entry = test_object_meta_entry(&object);
-                    disk.write_all("bucket", &format!("{object}/{STORAGE_FORMAT_FILE}"), bytes::Bytes::from(entry.metadata))
-                        .await
-                        .expect("fallback child metadata should be written");
-                }
-            }
-            fallback_disks.push(disk);
-            fallback_tempdirs.push(tempdir);
-        }
-        let supplement = ListingSupplement::new(
-            ListingSupplementOptions {
-                bucket: "bucket".to_owned(),
-                path: String::new(),
-                recursive: false,
-                incl_deleted: false,
-                skip_hidden_prefix_check: false,
-                filter_prefix: None,
-                forward_to: None,
-                per_disk_limit: 100,
-                skip_total_timeout: true,
-                walkdir_timeout: None,
-                walkdir_stall_timeout: None,
-            },
-            Arc::new(fallback_disks),
-            FallbackClaimTracker::default(),
-        );
-        // A 16-drive EC:4 set asks 12 primary disks. A committed write may
-        // exist on eight primary disks and all four remaining fallback disks.
-        let resolver = list_metadata_resolution_params("bucket".to_owned(), 4, 12, false, 0);
-        for fallback_copies in [3, 4] {
-            let prefix = format!("quux-{fallback_copies}/");
-            let mut primary = vec![Some(test_dir_meta_entry(&prefix)); 8];
-            primary.extend([None, None, None, None]);
-            let entry =
-                resolve_listing_entries_with_supplement(MetaCacheEntries(primary), resolver.clone(), true, supplement.clone())
-                    .await;
-            assert_eq!(
-                entry.map(|entry| entry.name),
-                (fallback_copies == 4).then_some(prefix),
-                "the common prefix needs all twelve copies, including fallback disks"
-            );
-        }
-    }
-
     #[test]
     fn latest_listing_supplement_keeps_a_subquorum_delete_marker_hidden() {
         let object_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");

crates/ecstore/src/store/object.rs

@@ -50,7 +50,7 @@ use crate::services::notification_sys::{
 use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata};
 use crate::set_disk::{
     SetDisks, get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold,
-    is_lock_optimization_enabled, is_object_lock_diag_enabled,
+    is_lock_optimization_enabled, is_object_lock_diag_enabled, same_distributed_lock_domain,
 };
 use crate::storage_api_contracts::{
     list::ListOperations as _,
@@ -859,78 +859,9 @@ async fn delete_recursive_prefix_with_tier_delete_journal(
             }
         }
     }
-    // A trailing slash selects a directory, not the object at its parent key.
-    // Raw filesystem recursion would also remove that object's metadata and
-    // data. Preserve it by purging the selected keys individually when they
-    // share this physical directory. The bucket write lock covers both scans.
-    if object.ends_with('/') && !is_meta_bucketname(bucket) {
-        let parent = object.strip_suffix('/').unwrap_or(object);
-        for pool in &store.pools {
-            for set in &pool.disk_set {
-                let page = set
-                    .clone()
-                    .inner_list_object_versions_for_recursive_delete(bucket, parent, None, None, 1)
-                    .await?;
-                if page.objects.iter().any(|info| info.name == parent) {
-                    return delete_directory_keys_with_tier_delete_journal(store, bucket, object, opts, tier_journal_api).await;
-                }
-            }
-        }
-    }
     delete_prefix_with_tier_delete_journal(store, bucket, object, opts, tier_journal_api).await
 }
 
-async fn delete_directory_keys_with_tier_delete_journal(
-    store: &ECStore,
-    bucket: &str,
-    prefix: &str,
-    opts: &ObjectOptions,
-    tier_journal_api: Option<&Arc<ECStore>>,
-) -> Result<()> {
-    for pool in &store.pools {
-        for set in &pool.disk_set {
-            let mut previous_keys = std::collections::BTreeSet::new();
-            loop {
-                // Restart after each bounded batch: its version markers have
-                // been deleted, and the bucket write lock excludes new keys.
-                let page = set
-                    .clone()
-                    .inner_list_object_versions_for_recursive_delete(
-                        bucket,
-                        prefix,
-                        None,
-                        None,
-                        RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE,
-                    )
-                    .await?;
-                let keys = page
-                    .objects
-                    .into_iter()
-                    .map(|info| info.name)
-                    .filter(|key| key.starts_with(prefix))
-                    .collect::<std::collections::BTreeSet<_>>();
-                if keys.is_empty() {
-                    break;
-                }
-                if keys == previous_keys {
-                    return Err(Error::other("directory deletion did not advance"));
-                }
-                for key in &keys {
-                    let encoded_key = encode_dir_object(key);
-                    let mut exact_opts = opts.clone();
-                    exact_opts.delete_prefix_object = true;
-                    let _guard = store
-                        .acquire_object_write_lock_if_needed("delete_object", bucket, &encoded_key, &mut exact_opts)
-                        .await?;
-                    delete_prefix_with_tier_delete_journal(store, bucket, &encoded_key, &exact_opts, tier_journal_api).await?;
-                }
-                previous_keys = keys;
-            }
-        }
-    }
-    Ok(())
-}
-
 /// A GET whose object identity has been resolved while its namespace read lock
 /// remains held, but whose body reader has not been constructed yet.
 ///
@@ -3148,7 +3079,12 @@ impl ECStore {
         let store = Arc::clone(self);
         let write = async move {
             let object = "buckets/.scanner-pause-backlog.json";
-            let mut opts = ObjectOptions::default();
+            let mut opts = ObjectOptions {
+                max_parity: true,
+                http_preconditions: Some(preconditions),
+                write_completion: crate::object_api::WriteCompletion::TailDrained,
+                ..Default::default()
+            };
             // Match migration: fixed object namespace -> durable pool metadata ->
             // actual replica namespace. The replica need not be the hash-routed set.
             let object_guard = if store.single_pool() {
@@ -3174,14 +3110,9 @@ impl ECStore {
             } else {
                 None
             };
-            let result = crate::data_movement::scanner_backlog::persist_native_scanner_pause_backlog_replica(
-                set,
-                data,
-                preconditions,
-                opts,
-                "publish",
-            )
-            .await;
+            let result = set
+                .put_object(RUSTFS_META_BUCKET, object, &mut PutObjReader::from_vec(data), &opts)
+                .await;
             drop(capacity_guard);
             drop(object_guard);
             result
@@ -3509,15 +3440,10 @@ impl ECStore {
 
         for pool in &self.pools {
             let hashed_set = pool.get_disks_by_key(object);
-            let mut lock_domain_already_held = !distributed;
-            if !lock_domain_already_held {
-                for locked_set in &locked_sets {
-                    if locked_set.shares_namespace_lock_domain(&hashed_set).await {
-                        lock_domain_already_held = true;
-                        break;
-                    }
-                }
-            }
+            let lock_domain_already_held = !distributed
+                || locked_sets
+                    .iter()
+                    .any(|locked_set| same_distributed_lock_domain(&locked_set.lockers, &hashed_set.lockers));
             if lock_domain_already_held {
                 continue;
             }
@@ -3574,15 +3500,10 @@ impl ECStore {
         let mut locked_sets = vec![fixed_set];
         for pool in &self.pools {
             for set in &pool.disk_set {
-                let mut lock_domain_already_held = !distributed;
-                if !lock_domain_already_held {
-                    for locked_set in &locked_sets {
-                        if locked_set.shares_namespace_lock_domain(set).await {
-                            lock_domain_already_held = true;
-                            break;
-                        }
-                    }
-                }
+                let lock_domain_already_held = !distributed
+                    || locked_sets
+                        .iter()
+                        .any(|locked_set| same_distributed_lock_domain(&locked_set.lockers, &set.lockers));
                 if lock_domain_already_held {
                     continue;
                 }
@@ -3660,15 +3581,10 @@ impl ECStore {
         let mut locked_sets = vec![fixed_set];
         for pool in &self.pools {
             for set in &pool.disk_set {
-                let mut lock_domain_already_held = !distributed;
-                if !lock_domain_already_held {
-                    for locked_set in &locked_sets {
-                        if locked_set.shares_namespace_lock_domain(set).await {
-                            lock_domain_already_held = true;
-                            break;
-                        }
-                    }
-                }
+                let lock_domain_already_held = !distributed
+                    || locked_sets
+                        .iter()
+                        .any(|locked_set| same_distributed_lock_domain(&locked_set.lockers, &set.lockers));
                 if lock_domain_already_held {
                     continue;
                 }
@@ -3751,15 +3667,11 @@ impl ECStore {
                 .get(pool_idx)
                 .ok_or_else(|| Error::other(format!("invalid data movement publication pool {pool_idx}")))?;
             let set = pool.get_disks_by_key(object);
-            let mut lock_domain_already_held = !locked_sets.is_empty() && !distributed;
-            if !lock_domain_already_held {
-                for locked_set in &locked_sets {
-                    if locked_set.shares_namespace_lock_domain(&set).await {
-                        lock_domain_already_held = true;
-                        break;
-                    }
-                }
-            }
+            let lock_domain_already_held = !locked_sets.is_empty()
+                && (!distributed
+                    || locked_sets.iter().any(|locked_set: &Arc<crate::set_disk::SetDisks>| {
+                        same_distributed_lock_domain(&locked_set.lockers, &set.lockers)
+                    }));
             if lock_domain_already_held {
                 continue;
             }
@@ -3835,37 +3747,29 @@ impl ECStore {
         opts: &ObjectOptions,
         no_lock: bool,
     ) -> Result<usize> {
-        let capacity_owner = DecommissionCapacityOwner::from_options(opts);
         match self
             .get_pool_info_existing_with_opts(bucket, object, &data_movement_pool_lookup_opts(opts, no_lock))
             .await
         {
-            Ok((pinfo, _)) => {
-                if let Some(owner) = capacity_owner {
-                    if self.is_decommission_capacity_target_reserved(owner, pinfo.index).await? {
-                        return Ok(pinfo.index);
-                    }
-                } else {
-                    return Ok(pinfo.index);
-                }
-            }
+            Ok((pinfo, _)) => Ok(pinfo.index),
             Err(err) => {
                 if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
                     return Err(err);
                 }
+
+                if let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
+                    let expected_data_bytes = opts
+                        .capacity_expected_data_bytes()
+                        .or_else(|| usize::try_from(size).ok())
+                        .unwrap_or_default();
+                    return self
+                        .select_decommission_capacity_target_pool(owner, expected_data_bytes)
+                        .await;
+                }
+
+                self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull)
             }
         }
-        if let Some(owner) = capacity_owner {
-            let expected_data_bytes = opts
-                .capacity_expected_data_bytes()
-                .or_else(|| usize::try_from(size).ok())
-                .unwrap_or_default();
-            return self
-                .select_decommission_capacity_target_pool(owner, expected_data_bytes)
-                .await;
-        }
-
-        self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull)
     }
 
     async fn find_data_movement_target_info(
@@ -4347,10 +4251,8 @@ impl ECStore {
     }
 
     /// Return metadata for DELETE preflight, including an explicitly addressed
-    /// delete marker. GET/HEAD may also use this metadata-only lookup to enrich
-    /// an already failed read with marker headers, never to serve marker data.
-    /// Normal reads must keep using `get_object_info`; authorization and Object
-    /// Lock enforcement still belong to the caller and locked delete.
+    /// delete marker. Read APIs must keep using `get_object_info`; authorization
+    /// and Object Lock enforcement still belong to the caller and locked delete.
     #[instrument(level = "trace", skip_all)]
     pub async fn get_object_info_for_delete(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
         self.get_object_info_snapshot(bucket, object, opts, true).await
@@ -4751,14 +4653,13 @@ impl ECStore {
             return Err(Error::other("lifecycle delete-all requires namespace locking"));
         }
 
-        let _bucket_lifecycle_guard =
-            if is_meta_bucketname(bucket) || (opts.delete_prefix && opts.bucket_lifecycle_lock_fence.is_some()) {
-                None
-            } else if opts.delete_prefix {
-                Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
-            } else {
-                Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
-            };
+        let _bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
+            None
+        } else if opts.delete_prefix {
+            Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
+        } else {
+            Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
+        };
         let object = if opts.delete_prefix && !opts.delete_prefix_object {
             object.to_owned()
         } else {
@@ -5806,7 +5707,6 @@ mod tests {
         GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource, clear_get_object_body_cache_hook,
         lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
     };
-    use crate::set_disk::same_distributed_lock_domain;
     use crate::set_disk::{SetDisks, disk_call_counters};
     use crate::storage_api_contracts::bucket::MakeBucketOptions;
     use crate::storage_api_contracts::lifecycle::TransitionedObject;

crates/filemeta/src/filemeta.rs

@@ -50,9 +50,6 @@ use tracing::{error, warn};
 use uuid::Uuid;
 use xxhash_rust::xxh64;
 
-mod transition_reconcile;
-pub use transition_reconcile::TransitionStateReconcileTarget;
-
 // XL header specifies the format
 pub static XL_FILE_HEADER: [u8; 4] = *b"XL2 ";
 // pub static XL_FILE_VERSION_CURRENT: [u8; 4] = [0; 4];
@@ -394,16 +391,6 @@ impl FileMeta {
 
                     if ver_vid == fi_vid {
                         let mut ver = FileMetaVersion::try_from(version.meta.as_slice())?;
-                        let previous = ver
-                            .object
-                            .as_ref()
-                            .is_some_and(|object| {
-                                rustfs_utils::http::contains_key_bytes(
-                                    &object.meta_sys,
-                                    rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
-                                )
-                            })
-                            .then(|| ver.clone());
 
                         if let Some(ref mut obj) = ver.object {
                             if replace_user_metadata {
@@ -460,9 +447,6 @@ impl FileMeta {
                             }
                         }
 
-                        if let Some(previous) = previous {
-                            transition_reconcile::preserve_reconciled_transition(&previous, &mut ver)?;
-                        }
                         // Update
                         version.header = ver.header();
                         version.meta = ver.marshal_msg()?;
@@ -508,7 +492,7 @@ impl FileMeta {
         Ok(())
     }
 
-    pub fn add_version_filemata(&mut self, mut version: FileMetaVersion) -> Result<()> {
+    pub fn add_version_filemata(&mut self, version: FileMetaVersion) -> Result<()> {
         if !version.valid() {
             return Err(Error::other("file meta version invalid"));
         }
@@ -528,7 +512,6 @@ impl FileMeta {
             if existing.free_version() != version.free_version() {
                 return Err(Error::other("cannot replace a free version with a non-free version"));
             }
-            transition_reconcile::preserve_reconciled_transition(&existing, &mut version)?;
             return self.set_idx(fidx, version);
         }
 

crates/filemeta/src/filemeta/transition_reconcile.rs

@@ -1,384 +0,0 @@
-// Copyright 2026 RustFS Team
-// SPDX-License-Identifier: Apache-2.0
-
-use super::{FileMeta, FileMetaVersion};
-use crate::{Error, Result, TRANSITION_COMPLETE, TransitionVersionState};
-use rustfs_utils::http::metadata_compat::{
-    SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, contains_key_bytes,
-    get_consistent_bytes, insert_bytes, remove_bytes,
-};
-use serde::{Deserialize, Serialize};
-use uuid::Uuid;
-
-const RECONCILE_SUFFIXES: [&str; 3] = [
-    SUFFIX_TRANSITIONED_VERSION_STATE,
-    SUFFIX_TRANSITIONED_VERSION_ID,
-    SUFFIX_TRANSITION_TIER_DESTINATION_ID,
-];
-
-/// The only fields a legacy transition repair is allowed to persist.
-#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct TransitionStateReconcileTarget {
-    pub state: TransitionVersionState,
-    pub remote_version: Option<String>,
-    pub destination_id: String,
-}
-
-impl TransitionStateReconcileTarget {
-    pub fn validate(&self) -> Result<()> {
-        let valid_version = match self.state {
-            TransitionVersionState::KnownDisabled => self.remote_version.is_none(),
-            TransitionVersionState::SuspendedNull => self.remote_version.as_deref() == Some("null"),
-            TransitionVersionState::Exact => self.remote_version.as_deref().is_some_and(|value| {
-                !value.is_empty()
-                    && value.len() <= 1024
-                    && value != "null"
-                    && !value.chars().any(char::is_control)
-                    && !Uuid::parse_str(value).is_ok_and(|id| id.is_nil())
-            }),
-            TransitionVersionState::Unknown => false,
-        };
-        if !valid_version
-            || self.destination_id.len() != 64
-            || !self
-                .destination_id
-                .bytes()
-                .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
-        {
-            return Err(Error::FileCorrupt);
-        }
-        Ok(())
-    }
-}
-
-impl FileMeta {
-    /// Canonical identity of every version and inline byte, excluding only the
-    /// three repairable suffixes on the selected version. It survives a repair
-    /// and encoding-order changes, while detecting unrelated metadata changes.
-    pub fn transition_reconcile_generation(&self, version_id: Option<Uuid>) -> Result<Vec<u8>> {
-        if self
-            .versions
-            .iter()
-            .filter(|version| version.header.version_id.unwrap_or_default() == version_id.unwrap_or_default())
-            .count()
-            != 1
-        {
-            return Err(Error::FileCorrupt);
-        }
-        let (selected, _) = self.find_version(version_id)?;
-        let mut versions = Vec::with_capacity(self.versions.len());
-        for index in 0..self.versions.len() {
-            let mut version = self.get_idx(index)?;
-            if index == selected {
-                let object = version.object.as_mut().ok_or(Error::FileCorrupt)?;
-                for suffix in RECONCILE_SUFFIXES {
-                    remove_bytes(&mut object.meta_sys, suffix);
-                }
-            }
-            versions.push(version);
-        }
-        versions.sort_by_key(|version| version.get_version_id().unwrap_or_default());
-        let mut value = serde_json::to_value((&versions, &self.data)).map_err(|_| Error::FileCorrupt)?;
-        value.sort_all_objects();
-        serde_json::to_vec(&value).map_err(|_| Error::FileCorrupt)
-    }
-
-    /// Returns false for an already converged record. Callers must serialize
-    /// the read/check/commit and compare the observed metadata generation.
-    pub fn reconcile_transition_state(
-        &mut self,
-        version_id: Option<Uuid>,
-        target: &TransitionStateReconcileTarget,
-    ) -> Result<bool> {
-        target.validate()?;
-        let (index, mut version) = self.find_version(version_id)?;
-        let info = version.into_fileinfo("", "", true)?;
-        info.validate_for_metadata_read()?;
-        if info.transition_status != TRANSITION_COMPLETE
-            || info.transition_tier.is_empty()
-            || info.transitioned_objname.is_empty()
-        {
-            return Err(Error::FileCorrupt);
-        }
-        let object = version.object.as_mut().ok_or(Error::FileCorrupt)?;
-        let destination = get_consistent_bytes(&object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID);
-        if contains_key_bytes(&object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
-            && destination != Some(target.destination_id.as_bytes())
-        {
-            return Err(Error::FileCorrupt);
-        }
-        if contains_key_bytes(&object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE) {
-            if info.transition_version_state == target.state
-                && info.transition_version == target.remote_version
-                && destination == Some(target.destination_id.as_bytes())
-            {
-                return Ok(false);
-            }
-            return Err(Error::FileCorrupt);
-        }
-        if info.transition_version_state != TransitionVersionState::Unknown
-            || info
-                .transition_version
-                .as_deref()
-                .filter(|value| !value.is_empty())
-                .is_some_and(|value| Some(value) != target.remote_version.as_deref())
-        {
-            return Err(Error::FileCorrupt);
-        }
-        insert_bytes(
-            &mut object.meta_sys,
-            SUFFIX_TRANSITIONED_VERSION_STATE,
-            target.state.as_str().as_bytes().to_vec(),
-        );
-        remove_bytes(&mut object.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID);
-        if let Some(version) = &target.remote_version {
-            insert_bytes(&mut object.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, version.as_bytes().to_vec());
-        }
-        insert_bytes(
-            &mut object.meta_sys,
-            SUFFIX_TRANSITION_TIER_DESTINATION_ID,
-            target.destination_id.as_bytes().to_vec(),
-        );
-        version.into_fileinfo("", "", true)?.validate_for_metadata_read()?;
-        self.set_idx(index, version)?;
-        Ok(true)
-    }
-}
-
-/// A stale healer or metadata writer may carry the original absent fields.
-/// Preserve a proven binding for the same immutable transition, or reject an
-/// attempted change of meaning. A new payload/version or a delete is separate.
-pub(super) fn preserve_reconciled_transition(previous: &FileMetaVersion, next: &mut FileMetaVersion) -> Result<()> {
-    let (Some(previous_object), Some(next_object)) = (&previous.object, &mut next.object) else {
-        return Ok(());
-    };
-    if !contains_key_bytes(&previous_object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
-        || !contains_key_bytes(&previous_object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE)
-        || previous_object.data_dir != next_object.data_dir
-    {
-        return Ok(());
-    }
-    let previous_info = previous.into_fileinfo("", "", true)?;
-    if previous_info.transition_version_state == TransitionVersionState::Unknown
-        || previous_info.transition_status != TRANSITION_COMPLETE
-    {
-        return Ok(());
-    }
-    let next_info = next.into_fileinfo("", "", true)?;
-    if previous_info.transition_tier != next_info.transition_tier
-        || previous_info.transitioned_objname != next_info.transitioned_objname
-        || previous_info.transition_status != next_info.transition_status
-        || previous_info.size != next_info.size
-        || previous_info.metadata.get("etag") != next_info.metadata.get("etag")
-    {
-        return Err(Error::FileCorrupt);
-    }
-    let destination =
-        get_consistent_bytes(&previous_object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID).ok_or(Error::FileCorrupt)?;
-    previous_info.validate_for_metadata_read()?;
-    TransitionStateReconcileTarget {
-        state: previous_info.transition_version_state,
-        remote_version: previous_info.transition_version.clone(),
-        destination_id: std::str::from_utf8(destination).map_err(|_| Error::FileCorrupt)?.to_string(),
-    }
-    .validate()?;
-    let next_object = next.object.as_mut().ok_or(Error::FileCorrupt)?;
-    if next_info
-        .transition_version
-        .as_ref()
-        .is_some_and(|version| Some(version) != previous_info.transition_version.as_ref())
-    {
-        return Err(Error::FileCorrupt);
-    }
-    if contains_key_bytes(&next_object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE) {
-        if previous_info.transition_version_state != next_info.transition_version_state
-            || previous_info.transition_version != next_info.transition_version
-        {
-            return Err(Error::FileCorrupt);
-        }
-    } else if next_info.transition_version_state != TransitionVersionState::Unknown {
-        return Err(Error::FileCorrupt);
-    }
-    if contains_key_bytes(&next_object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
-        && get_consistent_bytes(&next_object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID) != Some(destination)
-    {
-        return Err(Error::FileCorrupt);
-    }
-    for suffix in RECONCILE_SUFFIXES {
-        remove_bytes(&mut next_object.meta_sys, suffix);
-        if let Some(value) = get_consistent_bytes(&previous_object.meta_sys, suffix) {
-            insert_bytes(&mut next_object.meta_sys, suffix, value.to_vec());
-        }
-    }
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::{ErasureInfo, FileInfo, ObjectPartInfo};
-
-    fn legacy() -> (FileMeta, FileInfo) {
-        let info = FileInfo {
-            version_id: Some(Uuid::from_u128(1)),
-            data_dir: Some(Uuid::from_u128(2)),
-            mod_time: Some(time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixture time")),
-            size: 7,
-            parts: vec![ObjectPartInfo {
-                number: 1,
-                size: 7,
-                actual_size: 7,
-                ..Default::default()
-            }],
-            erasure: ErasureInfo {
-                algorithm: "ReedSolomon".to_string(),
-                data_blocks: 2,
-                parity_blocks: 2,
-                block_size: 1024 * 1024,
-                index: 1,
-                distribution: vec![1, 2, 3, 4],
-                ..Default::default()
-            },
-            transition_status: TRANSITION_COMPLETE.to_string(),
-            transition_tier: "WARM".to_string(),
-            transitioned_objname: "remote-object".to_string(),
-            metadata: std::collections::HashMap::from([("etag".to_string(), "source-etag".to_string())]),
-            data: Some(bytes::Bytes::from_static(b"payload")),
-            ..Default::default()
-        };
-        let mut metadata = FileMeta::new();
-        metadata.add_version(info.clone()).expect("legacy fixture");
-        (metadata, info)
-    }
-
-    fn target(state: TransitionVersionState) -> TransitionStateReconcileTarget {
-        TransitionStateReconcileTarget {
-            state,
-            remote_version: match state {
-                TransitionVersionState::Exact => Some("opaque-version".to_string()),
-                TransitionVersionState::SuspendedNull => Some("null".to_string()),
-                _ => None,
-            },
-            destination_id: "ab".repeat(32),
-        }
-    }
-
-    #[test]
-    fn transition_reconcile_preserves_payload_and_generation_and_is_idempotent() {
-        for state in [
-            TransitionVersionState::KnownDisabled,
-            TransitionVersionState::SuspendedNull,
-            TransitionVersionState::Exact,
-        ] {
-            let (mut metadata, info) = legacy();
-            let mut other = info.clone();
-            other.version_id = Some(Uuid::from_u128(3));
-            other.data_dir = Some(Uuid::from_u128(4));
-            other.transition_status.clear();
-            other.transition_tier.clear();
-            other.transitioned_objname.clear();
-            other.data = Some(bytes::Bytes::from_static(b"other!!"));
-            metadata.add_version(other.clone()).expect("unrelated inline version");
-            let other_before = metadata.find_version(other.version_id).expect("unrelated version").1;
-            let original_data = metadata.data.clone();
-            let generation = metadata
-                .transition_reconcile_generation(info.version_id)
-                .expect("initial generation");
-            let target = target(state);
-            assert!(metadata.reconcile_transition_state(info.version_id, &target).expect("repair"));
-            let bytes = metadata.marshal_msg().expect("encode repair");
-            let mut reloaded = FileMeta::load(&bytes).expect("reload repair");
-            assert_eq!(reloaded.data, original_data);
-            assert_eq!(
-                reloaded
-                    .find_version(other.version_id)
-                    .expect("preserved unrelated version")
-                    .1,
-                other_before
-            );
-            assert_eq!(
-                reloaded
-                    .transition_reconcile_generation(info.version_id)
-                    .expect("repaired generation"),
-                generation
-            );
-            assert!(!reloaded.reconcile_transition_state(info.version_id, &target).expect("retry"));
-            let (_, version) = reloaded.find_version(info.version_id).expect("selected version");
-            let repaired = version.into_fileinfo("", "", true).expect("decode explicit state");
-            assert_eq!(repaired.transition_version_state, state);
-            assert_eq!(repaired.transition_version, target.remote_version);
-            assert_eq!(repaired.parts, info.parts);
-            for prefix in [
-                rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
-                rustfs_utils::http::MINIO_INTERNAL_PREFIX,
-            ] {
-                assert_eq!(
-                    version
-                        .object
-                        .as_ref()
-                        .expect("object")
-                        .meta_sys
-                        .get(&format!("{prefix}{SUFFIX_TRANSITIONED_VERSION_STATE}")),
-                    Some(&state.as_str().as_bytes().to_vec())
-                );
-            }
-        }
-    }
-
-    #[test]
-    fn transition_reconcile_generation_detects_unrelated_metadata_and_inline_changes() {
-        let (mut metadata, info) = legacy();
-        let original = metadata.transition_reconcile_generation(info.version_id).expect("generation");
-        let mut updated = info.clone();
-        updated.metadata.insert("user-tag".to_string(), "changed".to_string());
-        metadata.update_object_version(updated).expect("update unrelated field");
-        assert_ne!(metadata.transition_reconcile_generation(info.version_id).expect("generation"), original);
-        let (mut metadata, mut info) = legacy();
-        info.data = Some(bytes::Bytes::from_static(b"changed"));
-        metadata.add_version(info.clone()).expect("change inline bytes");
-        assert_ne!(metadata.transition_reconcile_generation(info.version_id).expect("generation"), original);
-    }
-
-    #[test]
-    fn transition_reconcile_binding_survives_stale_heal_and_metadata_writes() {
-        let (mut metadata, mut stale) = legacy();
-        let target = target(TransitionVersionState::Exact);
-        metadata
-            .reconcile_transition_state(stale.version_id, &target)
-            .expect("repair");
-        metadata
-            .add_version(stale.clone())
-            .expect("stale heal must preserve the binding");
-        stale.metadata.insert("user-tag".to_string(), "updated".to_string());
-        metadata
-            .update_object_version(stale.clone())
-            .expect("ordinary metadata update");
-        assert!(
-            !metadata
-                .reconcile_transition_state(stale.version_id, &target)
-                .expect("binding remains exact")
-        );
-    }
-
-    #[test]
-    fn transition_reconcile_rejects_explicit_unknown_and_conflicting_binding() {
-        let (mut metadata, mut info) = legacy();
-        rustfs_utils::http::insert_str(&mut info.metadata, SUFFIX_TRANSITIONED_VERSION_STATE, "unknown".to_string());
-        metadata.add_version(info.clone()).expect("explicit unknown fixture");
-        assert!(
-            metadata
-                .reconcile_transition_state(info.version_id, &target(TransitionVersionState::Exact))
-                .is_err()
-        );
-        let (mut metadata, mut stale) = legacy();
-        metadata
-            .reconcile_transition_state(stale.version_id, &target(TransitionVersionState::Exact))
-            .expect("repair");
-        stale.transition_version_state = TransitionVersionState::KnownDisabled;
-        assert!(
-            metadata.add_version(stale).is_err(),
-            "an explicit state cannot be replaced with a different model"
-        );
-    }
-}

crates/heal/Cargo.toml

@@ -31,7 +31,6 @@ workspace = true
 
 [features]
 default = []
-test-util = []
 hotpath = [
     "hotpath/hotpath",
     "hotpath/tokio",
@@ -105,7 +104,6 @@ walkdir = { workspace = true }
 http = { workspace = true }
 temp-env = { workspace = true, features = ["async_closure"] }
 tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
-chrono = { workspace = true }
 
 [lib]
 doctest = false