fix(ecstore): recover prepared batches with missing versions

Merge frozen main acd26afd8b and include
compatibility fixes and measured Darwin/Linux membership corrections
across six files.

Validation on candidate b4c3ab560a:
- All 20 local gates passed.
- Native smoke: 218 passed. Native full: 661 passed.
- Full passed on the same-head failed-only rerun; the initial failure
  remains recorded.
- Gateway remains report-only: 38 passed, 180 failed, under the existing
  exemption.

CI: https://github.com/rustfs/rustfs/actions/runs/38285788334

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=1520cd3027c3a6924afc6a204eddd5475bc30f0109d4c2847456a66177263244
-sha256-linux=26155b7f1c5bf6bf596ff24a4136e6767ea4ee6461c32db7fbe1400791da2c28
+sha256-darwin=ad81c65dcca3a3994c7d62562f477777d7d07fa5aa9daa799741365ac9e5e886
+sha256-linux=a5f013b21fed4162e6bc33077841f0f07b3ffbbbb665ddbce4e77a2513c34d76

.config/e2e-nightly-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=347a0d4e5e547d5388703062ce66a66167a845b6e57ec591ccb4c732a567c68d
-sha256-linux=4f9efa47194279185bb68249e82c788fa147abe944b6c764050e3d47f31e7926
+sha256-darwin=ce7b419faf1ff79840dfc6325d31ee8d1ca5eab85125b94d0bcd58cdf07f0f76
+sha256-linux=5e9e3d701ee873c13fb9094575f6889779c6078b1107624bb57663bef9b9d21f

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=ba8f392b617cf9f2338142233a417e43927b28096635e628233ce8a60cf49d10
+sha256=bf77250d94604db7ba094c5d63bf8a284beb872983871bcfd0dc02e46ff53326

.config/make/lint-fmt.mak

@@ -70,6 +70,11 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
 	@echo "📦 Checking s3s footprint ratchet..."
 	./scripts/check_s3s_footprint.sh
 
+.PHONY: test-count-ratchet-check
+test-count-ratchet-check: ## Check no workspace crate lost tests or gained #[ignore] against the committed baseline
+	@echo "🧪 Checking per-crate test-count ratchet..."
+	./scripts/check_test_count_ratchet.sh
+
 .PHONY: fips-wording-check
 fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities
 	@echo "📣 Checking cryptographic capability wording guard..."

.config/make/pre-commit.mak

@@ -20,11 +20,11 @@ planning-docs-check: ## Check that no planning-type documents are committed
 	./scripts/check_no_planning_docs.sh
 
 .PHONY: pre-commit
-pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
+pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
 	@echo "✅ All pre-commit checks passed!"
 
 .PHONY: pre-pr
-pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
+pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
 	@echo "✅ All pre-PR checks passed!"
 
 .PHONY: dev-check

.config/make/tests.mak

@@ -50,10 +50,10 @@ script-tests: ## Run shell script tests
 	./scripts/test_fuzz_runner.sh
 	./scripts/test_python_bin.sh
 	./scripts/check_embedded_secrets.sh --self-test
+	./scripts/check_s3s_footprint.sh --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
-	$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
-	$(RUSTFS_PYTHON_BIN) ./scripts/test_release_glibc.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_upgrade_workflow.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_migration_gate_evidence.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test

.config/nextest.toml

@@ -88,11 +88,6 @@ threads-required = "num-test-threads"
 filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)'
 setup = 'ecstore-large-stack'
 
-# These cross-crate real-storage fixtures need the same bounded stack as ECStore tests.
-[[profile.default.scripts]]
-filter = '(package(rustfs) & test(/^(admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
-setup = 'ecstore-base-stack'
-
 [[profile.default.scripts]]
 filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
 setup = 'ecstore-base-stack'
@@ -344,11 +339,6 @@ threads-required = "num-test-threads"
 filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)'
 setup = 'ecstore-large-stack'
 
-# These cross-crate real-storage fixtures need the same bounded stack as ECStore tests.
-[[profile.ci.scripts]]
-filter = '(package(rustfs) & test(/^(admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
-setup = 'ecstore-base-stack'
-
 [[profile.ci.scripts]]
 filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
 setup = 'ecstore-base-stack'
@@ -599,6 +589,14 @@ test-group = 'ecstore-serial-flaky'
 # the PR-lane e2e coverage for scanner usage snapshots consumed by quota and
 # admin surfaces. It uses the same single-node, random-port, isolated-temp-dir
 # fixture as the existing smoke modules.
+#
+# Dual-stack differential (rustfs/backlog#2740): `dual_stack_differential::`
+# starts two single-node servers from the same binary, one per
+# RUSTFS_S3_STACK value, on random ports with isolated temp dirs, and runs one
+# SDK script covering every operation of `impl S3 for FS` against both. Its
+# comparator unit tests need no server. It meets the admission criteria
+# unchanged; the module clause admits a new differential test without another
+# filter edit.
 [profile.e2e-smoke]
 default-filter = """
     package(e2e_test) & (
@@ -606,6 +604,7 @@ default-filter = """
         | test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
         | test(/^reliant::lifecycle::/)
         | test(/^reliant::tiering::/)
+        | test(/^dual_stack_differential::/)
         | test(/^upgrade_compatibility_test::upgrade_write_readiness_tests::/)
         | test(/^on_demand_migration::(get_basic_test::(get_miss_pulls_inline_and_serves_locally_afterwards|head_miss_answers_from_the_source_without_persisting)|interaction_test::test_odm_admin_config_is_redacted_and_status_counts_match_the_source)$/)
     )

.config/s3s-edge-allowlist.txt

@@ -0,0 +1,37 @@
+# Files that may still reference s3s (`s3s::` or `s3s_sigv4`) when Phase 1 of
+# the gateway migration ends (rustfs/backlog#2734 task T0.6, rustfs/backlog#2741).
+# Consumed only by `scripts/check_s3s_footprint.sh --mode allowlist`.
+#
+# One entry per line: a repository-relative file path or a git `:(glob)`
+# pathspec. `**` matches across directories, `*` stays within one path
+# component, and a directory entry matches everything under it. Blank lines
+# and lines starting with `#` are ignored; an entry may not contain whitespace,
+# so a note belongs on its own comment line.
+#
+# Every Rust file listed by `git grep -l '\bs3s\(::\|_sigv4\)' -- '*.rs'` that
+# no entry matches fails the allowlist mode. Task T1.9 switches CI to that
+# mode; until then `--dry-run` prints the files still outside this list, which
+# is the Phase 1 tracking list.
+#
+# Never add crates/** other than crates/e2e_test: the engine crates leave s3s
+# entirely, and the script rejects such an entry. Only the s3s-facing edge of
+# the rustfs binary and the e2e oracle keep it.
+
+# rustfs binary: the s3s-facing edge that survives Phase 1
+rustfs/src/storage/ecfs.rs
+rustfs/src/storage/s3_compat/**
+rustfs/src/storage/access.rs
+rustfs/src/auth.rs
+rustfs/src/error.rs
+rustfs/src/storage_api.rs
+rustfs/src/app/storage_api.rs
+rustfs/src/admin/router.rs
+rustfs/src/app/metadata_route.rs
+rustfs/src/server/http.rs
+rustfs/src/server/layer.rs
+rustfs/src/server/hybrid.rs
+rustfs/src/protocols/client.rs
+
+# e2e oracle: the test target that verifies S3 behaviour against s3s
+crates/e2e_test/src/fake_s3_target/**
+crates/e2e_test/src/**

.config/s3s-footprint-baseline.txt

@@ -0,0 +1,17 @@
+# Exact baseline for the direct s3s footprint, consumed only by
+# scripts/check_s3s_footprint.sh (rustfs/backlog#1677; rustfs/backlog#2734 T0.4).
+#
+# Each value is the exact measurement of the tree, not a ceiling. CI fails when a
+# measured counter differs from its line here in either direction:
+#   - a larger measurement means new code widened the s3s surface: route it
+#     through the gateway abstractions instead; never raise a value here;
+#   - a smaller measurement means the PR shrank the surface: lower the matching
+#     value to the measured number in the same PR.
+#
+# files           Rust files referencing s3s paths, repo-wide minus crates/e2e_test
+# s3_error_lines  lines holding s3_error!, same scope, minus those the script
+#                 header's rule clears (rustfs_s3_types's macro); fails closed
+# ecstore_files   Rust files referencing s3s paths under crates/ecstore/src
+files=194
+s3_error_lines=1566
+ecstore_files=36

.config/test-count-baseline.txt

@@ -0,0 +1,63 @@
+# Per-crate test-count baseline consumed by scripts/check_test_count_ratchet.sh
+# (rustfs/backlog#2734 section 1 rule 3; T0.3 in rustfs/backlog#2738).
+#
+# One row per workspace member: <package>, tests, ignored, tab-separated.
+# tests counts #[test], #[tokio::test], #[tokio::test(...)] and #[test_case(...)]
+# attribute lines under the package directory; ignored counts #[ignore] and
+# #[ignore = "..."]. Both columns are exact: CI fails when a crate's tests fall
+# below or rise above its row, or when its ignored count changes. Raise a row in
+# the PR that adds tests; lowering one is a reviewed statement that tests were
+# intentionally removed. Regenerate with:
+#   scripts/check_test_count_ratchet.sh --update
+e2e_test	898	18
+rustfs	5585	63
+rustfs-audit	70	0
+rustfs-checksums	26	0
+rustfs-common	25	0
+rustfs-concurrency	27	0
+rustfs-config	43	0
+rustfs-connect-symbol-catalog	6	0
+rustfs-credentials	24	0
+rustfs-crypto	67	0
+rustfs-data-usage	54	0
+rustfs-ecstore	6443	1
+rustfs-extension-schema	23	0
+rustfs-filemeta	306	0
+rustfs-heal	839	3
+rustfs-heal-contracts	6	0
+rustfs-iam	388	0
+rustfs-io-core	49	0
+rustfs-io-metrics	125	0
+rustfs-keystone	29	0
+rustfs-kms	715	25
+rustfs-license	2	0
+rustfs-lifecycle	152	0
+rustfs-lock	135	0
+rustfs-log-analyzer	89	0
+rustfs-madmin	209	0
+rustfs-notify	136	0
+rustfs-object-capacity	92	0
+rustfs-object-data-cache	140	0
+rustfs-obs	366	0
+rustfs-policy	681	0
+rustfs-protocols	736	1
+rustfs-protos	63	0
+rustfs-replication	200	0
+rustfs-rio	200	0
+rustfs-rio-v2	34	6
+rustfs-s3-client	65	0
+rustfs-s3-ops	8	0
+rustfs-s3-types	97	0
+rustfs-s3select-api	206	0
+rustfs-s3select-query	188	2
+rustfs-scanner	997	20
+rustfs-scanner-metrics	81	0
+rustfs-security-governance	20	0
+rustfs-signer	31	0
+rustfs-storage-api	31	0
+rustfs-targets	603	26
+rustfs-test-utils	1	0
+rustfs-tls-runtime	43	0
+rustfs-trusted-proxies	68	0
+rustfs-utils	217	0
+rustfs-zip	30	0

.docker/observability/grafana/dashboards/rustfs.json

@@ -1748,7 +1748,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id=\"$storage_cluster\", job=~\"$job\", bucket=~\"$bucket\"})",
+          "expr": "max by (job, bucket) (rustfs_cluster_usage_buckets_objects_count{job=~\"$job\", bucket=~\"$bucket\"})",
           "legendFormat": "{{bucket}}",
           "range": true,
           "refId": "A"
@@ -1848,7 +1848,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id=\"$storage_cluster\", job=~\"$job\", bucket=~\"$bucket\"})",
+          "expr": "max by (job, bucket) (rustfs_cluster_usage_buckets_total_bytes{job=~\"$job\", bucket=~\"$bucket\"})",
           "legendFormat": "{{bucket}}",
           "range": true,
           "refId": "A"

.docker/observability/prometheus.yml

@@ -61,7 +61,6 @@ scrape_configs:
 otlp:
   promote_resource_attributes:
     - service.instance.id
-    - process.pid
     - service.name
     - service.namespace
     - cloud.availability_zone

.docker/observability/tests/bucket-usage.test.yml

@@ -1,48 +0,0 @@
-evaluation_interval: 1m
-tests:
-- name: bucket usage keeps clusters separate and deduplicates reporting nodes
-  interval: 1m
-  input_series:
-  - series: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",job="rustfs",bucket="shared",service_instance_id="a1"}
-    values: '4096 1024'
-  - series: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",job="rustfs",bucket="shared",service_instance_id="a2"}
-    values: '4096 1024'
-  - series: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="b",job="rustfs",bucket="shared",service_instance_id="b1"}
-    values: '65536 65536'
-  - series: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",job="rustfs",bucket="empty",service_instance_id="a1"}
-    values: '0 0'
-  - series: rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="a",job="rustfs",bucket="shared",service_instance_id="a1"}
-    values: '3 1'
-  - series: rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="a",job="rustfs",bucket="shared",service_instance_id="a2"}
-    values: '3 1'
-  - series: rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="b",job="rustfs",bucket="shared",service_instance_id="b1"}
-    values: '70 70'
-  - series: rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="a",job="rustfs",bucket="empty",service_instance_id="a1"}
-    values: '0 0'
-  promql_expr_test:
-  - expr: max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",job=~"rustfs",bucket=~".*"})
-    eval_time: 1m
-    exp_samples:
-    - labels: '{rustfs_cluster_id="a",job="rustfs",bucket="shared"}'
-      value: 1024
-    - labels: '{rustfs_cluster_id="a",job="rustfs",bucket="empty"}'
-      value: 0
-  - expr: max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="a",job=~"rustfs",bucket=~".*"})
-    eval_time: 1m
-    exp_samples:
-    - labels: '{rustfs_cluster_id="a",job="rustfs",bucket="shared"}'
-      value: 1
-    - labels: '{rustfs_cluster_id="a",job="rustfs",bucket="empty"}'
-      value: 0
-  - expr: max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_total_bytes)
-    eval_time: 1m
-    exp_samples:
-    - labels: '{rustfs_cluster_id="a",job="rustfs",bucket="shared"}'
-      value: 1024
-    - labels: '{rustfs_cluster_id="a",job="rustfs",bucket="empty"}'
-      value: 0
-    - labels: '{rustfs_cluster_id="b",job="rustfs",bucket="shared"}'
-      value: 65536
-  - expr: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",bucket="unknown"}
-    eval_time: 1m
-    exp_samples: []

.github/actions/quick-checks/action.yml

@@ -86,6 +86,10 @@ runs:
       shell: bash
       run: ./scripts/check_s3s_footprint.sh
 
+    - name: Check per-crate test-count ratchet
+      shell: bash
+      run: ./scripts/check_test_count_ratchet.sh
+
     - name: Check cryptographic capability wording
       shell: bash
       run: ./scripts/check_fips_wording.sh

.github/actions/setup-docker-compose/action.yml

@@ -1,39 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: Set up Docker Compose
-description: Install the verified Compose CLI used by native Linux x86_64 Connect acceptance
-
-runs:
-  using: composite
-  steps:
-    - name: Install verified Docker Compose CLI plugin
-      shell: bash
-      run: |
-        set -euo pipefail
-        [[ $(uname -s) == Linux && $(uname -m) == x86_64 ]]
-        plugin_dir="${DOCKER_CONFIG:-$HOME/.docker}/cli-plugins"
-        mkdir -p "$plugin_dir"
-        compose_download=$(mktemp "${RUNNER_TEMP:?}/connect-compose.XXXXXX")
-        compose_staged=$(mktemp "$plugin_dir/.docker-compose.XXXXXX")
-        trap 'rm -f "$compose_download" "$compose_staged"' EXIT
-        curl --fail --location --retry 3 \
-          https://github.com/docker/compose/releases/download/v2.40.3/docker-compose-linux-x86_64 \
-          --output "$compose_download"
-        printf '%s  %s\n' \
-          dba9d98e1ba5bfe11d88c99b9bd32fc4a0624a30fafe68eea34d61a3e42fd372 \
-          "$compose_download" | sha256sum --check --strict
-        install -m 0755 "$compose_download" "$compose_staged"
-        mv -f "$compose_staged" "$plugin_dir/docker-compose"
-        [[ $(docker compose version --short) == 2.40.3 ]]

.github/actions/setup/action.yml

@@ -36,16 +36,6 @@ inputs:
     description: "Install cross-compilation tools"
     required: false
     default: "false"
-  zig-version:
-    description: >-
-      Zig version for cargo-zigbuild builds. Pinned so the `<target>.<glibc>`
-      baseline semantics cannot drift with an untested zig release.
-    required: false
-    default: "0.17.0"
-  cargo-zigbuild-version:
-    description: "cargo-zigbuild version installed for cross builds."
-    required: false
-    default: "0.23.4"
   target:
     description: "Target architecture to add"
     required: false
@@ -109,14 +99,10 @@ runs:
     - name: Install Zig
       if: inputs.install-cross-tools == 'true'
       uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2
-      with:
-        version: ${{ inputs.zig-version }}
 
     - name: Install cargo-zigbuild
       if: inputs.install-cross-tools == 'true'
       uses: taiki-e/install-action@a21ae4029b089b9ddc45704028756f51ab8abe48 # cargo-zigbuild
-      with:
-        tool: cargo-zigbuild@${{ inputs.cargo-zigbuild-version }}
 
     - name: Install cargo-nextest
       if: inputs.install-test-tools == 'true'
@@ -140,5 +126,5 @@ runs:
         # normalises, so cargo fingerprints stay valid.
         cache-all-crates: false
         cache-on-failure: true
-        shared-key: ${{ inputs.cache-shared-key }}${{ inputs.install-cross-tools == 'true' && format('-zig-{0}-zigbuild-{1}', inputs.zig-version, inputs.cargo-zigbuild-version) || '' }}
+        shared-key: ${{ inputs.cache-shared-key }}
         save-if: ${{ inputs.cache-save-if }}

.github/workflows/build.yml

@@ -213,12 +213,12 @@ jobs:
           fi
 
           all='{"include":[
-            {"target_id":"linux-x86_64-musl","os":"sm-standard-4","target":"x86_64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
-            {"target_id":"linux-aarch64-musl","os":"sm-standard-4-arm","target":"aarch64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
-            {"target_id":"linux-x86_64-gnu","os":"sm-standard-4","target":"x86_64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
-            {"target_id":"linux-aarch64-gnu","os":"sm-standard-4-arm","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
-            {"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope","glibc":""},
-            {"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":"","features":"","glibc":""}
+            {"target_id":"linux-x86_64-musl","os":"sm-standard-4","target":"x86_64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":""},
+            {"target_id":"linux-aarch64-musl","os":"sm-standard-4-arm","target":"aarch64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":""},
+            {"target_id":"linux-x86_64-gnu","os":"sm-standard-4","target":"x86_64-unknown-linux-gnu","cross":false,"platform":"linux","rustflags":"","features":"pyroscope"},
+            {"target_id":"linux-aarch64-gnu","os":"sm-standard-4-arm","target":"aarch64-unknown-linux-gnu","cross":false,"platform":"linux","rustflags":"","features":"pyroscope"},
+            {"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope"},
+            {"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":"","features":""}
           ]}'
 
           if [[ "${selected}" == "all" ]]; then
@@ -246,8 +246,7 @@ jobs:
           echo "Selected platforms: ${selected}"
 
   build-rustfs:
-    # Acceptance workflows select artifacts by this stable platform name.
-    name: Build RustFS (${{ matrix.target_id }})
+    name: Build RustFS
     needs: [ build-check, prepare-platform-matrix ]
     if: needs.build-check.outputs.should_build == 'true' && needs.prepare-platform-matrix.result == 'success'
     runs-on: ${{ matrix.os }}
@@ -274,7 +273,7 @@ jobs:
         with:
           rust-version: stable
           target: ${{ matrix.target }}
-          cache-shared-key: build-${{ matrix.target }}${{ matrix.glibc && format('-glibc-{0}', matrix.glibc) || '' }}
+          cache-shared-key: build-${{ matrix.target }}
           # main only. A cache saved on refs/tags/X is scoped to that tag: no
           # other tag, no main run and no PR can restore it, so every release
           # cycle wrote up to 12 entries of 1-2GB (preview tag plus final tag,
@@ -411,53 +410,35 @@ jobs:
         shell: bash
         env:
           BUILD_TARGET: ${{ matrix.target }}
-          BUILD_OS: ${{ matrix.platform }}-${{ startsWith(matrix.target, 'aarch64-') && 'aarch64' || 'x86_64' }}
         run: |
           # Force rebuild by touching build.rs
           touch rustfs/build.rs
 
+          BINARY_NAMES=(rustfs)
+          if [[ "${{ matrix.platform }}" == "linux" ]]; then
+            BINARY_NAMES+=(rustfs-cli)
+          fi
           FEATURE_ARGS=()
           if [[ -n "${{ matrix.features }}" ]]; then
             FEATURE_ARGS=(--features "${{ matrix.features }}")
           fi
 
-          # gnu targets pin an explicit glibc baseline (`<target>.<glibc>` for
-          # cargo-zigbuild) so release binaries link no symbols newer than it
-          # and run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
-          TARGET_ARG="$BUILD_TARGET"
-          if [[ -n "${{ matrix.glibc }}" ]]; then
-            TARGET_ARG="${TARGET_ARG}.${{ matrix.glibc }}"
-          fi
-
-          PROFILE_ARGS=()
-          if [[ "$BUILD_TARGET" == *-linux-gnu && "${{ matrix.features }}" == "pyroscope" ]]; then
-            # The in-process profiler resolves names from the final executable.
-            # Retain function symbols without enabling DWARF source information.
-            PROFILE_ARGS=(--config 'profile.release.strip="none"')
-          fi
-          if [[ "${{ matrix.cross }}" == "true" ]]; then
-            cargo zigbuild --release --locked --target "$TARGET_ARG" "${FEATURE_ARGS[@]}" "${PROFILE_ARGS[@]}" -p rustfs --bin rustfs
-          else
-            cargo build --release --locked --target "$TARGET_ARG" "${FEATURE_ARGS[@]}" "${PROFILE_ARGS[@]}" -p rustfs --bin rustfs
-          fi
-
-          # All release targets use same-architecture runners, including GNU
-          # builds that use Zig only to select the glibc ABI floor.
-          bin="target/${{ matrix.target }}/release/rustfs"
-          if [[ -f "${bin}.exe" ]]; then
-            bin="${bin}.exe"
-          fi
-          version_output=$("$bin" --version)
-          if ! grep -Fxq "build target : ${BUILD_TARGET}" <<< "$version_output" || \
-             ! grep -Fxq "build os     : ${BUILD_OS}" <<< "$version_output" || \
-             ! grep -Fxq "rust channel : stable-${BUILD_TARGET}" <<< "$version_output"; then
-            echo "release binary reports an unexpected compile target" >&2
-            exit 1
-          fi
-          if [[ -n "${{ matrix.glibc }}" ]]; then
-            python3 scripts/release/check_glibc.py "${{ matrix.glibc }}" \
-              "target/${{ matrix.target }}/release/rustfs"
-          fi
+          # Build one binary at a time so release LTO links cannot exhaust a
+          # self-hosted runner by running concurrently.
+          for binary in "${BINARY_NAMES[@]}"; do
+            PROFILE_ARGS=()
+            if [[ "$binary" == "rustfs" && "$BUILD_TARGET" == *-linux-gnu && "${{ matrix.features }}" == "pyroscope" ]]; then
+              # The in-process profiler resolves names from the final executable.
+              # Retain function symbols without enabling DWARF source information.
+              PROFILE_ARGS=(--config 'profile.release.strip="none"')
+            fi
+            if [[ "${{ matrix.cross }}" == "true" ]]; then
+              # All cross targets in the matrix are Linux; zigbuild handles them.
+              cargo zigbuild --release --target ${{ matrix.target }} "${FEATURE_ARGS[@]}" "${PROFILE_ARGS[@]}" -p rustfs --bin "$binary"
+            else
+              cargo build --release --target ${{ matrix.target }} "${FEATURE_ARGS[@]}" "${PROFILE_ARGS[@]}" -p rustfs --bin "$binary"
+            fi
+          done
 
       - name: Verify macOS binary has no dynamic liblzma dependency
         if: matrix.platform == 'macos'
@@ -573,6 +554,9 @@ jobs:
           fi
 
           BINARY_FILES=("$BINARY_NAME")
+          if [[ "${{ matrix.platform }}" == "linux" ]]; then
+            BINARY_FILES+=("rustfs-cli")
+          fi
 
           if [[ "$TARGET" == *-linux-gnu && "${{ matrix.features }}" == "pyroscope" ]]; then
             BINARY_FILES+=(rustfs.cpu-symbol-catalog.json rustfs.cpu-symbol-catalog.sha256)
@@ -687,7 +671,7 @@ jobs:
           echo "🔧 Build type: ${BUILD_TYPE}"
           echo "📊 Version: ${VERSION}"
 
-      - name: Verify packaged Linux binary
+      - name: Verify packaged Linux binaries
         if: matrix.platform == 'linux'
         shell: bash
         run: |
@@ -695,7 +679,7 @@ jobs:
 
           package_file="${{ steps.package.outputs.package_file }}"
           archive_members="$(unzip -Z1 "$package_file" | sort)"
-          expected_members='rustfs'
+          expected_members=$'rustfs\nrustfs-cli'
           target="${{ matrix.target }}"
           features="${{ matrix.features }}"
           if [[ "$target" == *-linux-gnu && "$features" == "pyroscope" ]]; then
@@ -707,14 +691,13 @@ jobs:
             exit 1
           fi
 
-          # musl targets build natively; the gnu legs are same-arch
-          # cargo-zigbuild builds pinned to the release glibc baseline. Either
-          # way the binaries run on their runner, so executing them here
+          # Every Linux target builds natively, so executing the binaries here
           # proves each artifact matches its runner architecture.
-          binary_dir="$(mktemp -d)"
-          trap 'rm -rf "$binary_dir"' EXIT
-          unzip -q "$package_file" rustfs -d "$binary_dir"
-          "$binary_dir/rustfs" --version
+          cli_dir="$(mktemp -d)"
+          trap 'rm -rf "$cli_dir"' EXIT
+          unzip -q "$package_file" rustfs rustfs-cli -d "$cli_dir"
+          "$cli_dir/rustfs" --version
+          "$cli_dir/rustfs-cli" --help >/dev/null
 
       - name: Verify packaged console
         if: matrix.target == 'x86_64-unknown-linux-gnu' || matrix.target == 'aarch64-unknown-linux-gnu'

.github/workflows/ci.yml

@@ -16,7 +16,9 @@ name: Continuous Integration
 
 on:
   push:
-    branches: [ main ]
+    # gateway/integration receives the daily merge of main as a direct push
+    # (rustfs/backlog#2734), so its pushes must prove the result like main's.
+    branches: [ main, gateway/integration ]
     paths-ignore:
       - "**.md"
       - "docs/**"
@@ -748,6 +750,8 @@ jobs:
     needs: [ build-rustfs-debug-binary ]
     runs-on: sm-standard-2
     timeout-minutes: 30
+    env:
+      RUSTFS_S3_STACK: legacy
     steps:
       - name: Checkout repository
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -866,18 +870,105 @@ jobs:
           path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
           retention-days: 3
 
+  # The same e2e-smoke suite against the same debug binary with the gateway
+  # S3 stack selected (rustfs/backlog#2734). Report-only: it is outside the
+  # Test and Lint aggregate and becomes required at T2.10. The archive and run
+  # steps must stay identical to e2e-tests; scripts/ci_gate.py enforces that.
+  e2e-tests-gateway:
+    name: e2e-smoke (gateway, report-only until T2.10)
+    needs: [ build-rustfs-debug-binary ]
+    runs-on: sm-standard-2
+    timeout-minutes: 30
+    continue-on-error: true
+    env:
+      RUSTFS_S3_STACK: gateway
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-dev
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
+
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+
+      - name: Install awscurl
+        run: |
+          python3 -m pip install --user --upgrade pip "awscurl==0.44"
+          echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
+
+      - name: Verify awscurl
+        run: test -x "$AWSCURL_PATH"
+
+      - name: Download debug binary
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: rustfs-debug-binary
+          path: target/debug
+
+      - name: Make binary executable
+        run: chmod +x ./target/debug/rustfs
+
+      - name: Archive e2e smoke test binaries
+        env:
+          NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
+          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
+        run: |
+          cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}"
+          cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}"
+          python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
+          ./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
+
+      - name: Run e2e smoke suite
+        env:
+          NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
+          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
+          SAMPLER_INTERVAL_SECS: "15"
+        run: |
+          ./scripts/ci/resource_sampler.sh start e2e-smoke
+          trap './scripts/ci/resource_sampler.sh stop' EXIT
+          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
+            --status-level all --final-status-level all --failure-output final
+
+      - name: Upload e2e smoke diagnostics
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: e2e-smoke-gateway-diagnostics-${{ github.run_number }}
+          path: |
+            ${{ runner.temp }}/rustfs-e2e-smoke-logs/
+            ${{ runner.temp }}/rustfs-e2e-smoke-list.json
+          if-no-files-found: warn
+
+      - name: Upload e2e smoke JUnit report
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: e2e-smoke-gateway-junit-${{ github.run_number }}
+          path: target/nextest/e2e-smoke/junit.xml
+          if-no-files-found: warn
+
   e2e-full:
     name: End-to-End Tests (full merge gate)
     # Merge gate only (backlog#1149 ci-5): the never-automated user-visible
     # suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
     # security-boundary, ... — via the e2e-full nextest profile. Too heavy for
-    # every PR, so it is gated to main pushes, the merge queue, and manual
-    # dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
-    # owned by other lanes (see .config/nextest.toml profile.e2e-full).
+    # every PR, so it is gated to main and gateway/integration pushes, the merge
+    # queue, and manual dispatch. protocols / the 7 cluster suites / replication
+    # / #[ignore] are owned by other lanes (see .config/nextest.toml profile.e2e-full).
     if: >-
       github.event_name == 'workflow_dispatch' ||
       github.event_name == 'merge_group' ||
-      (github.event_name == 'push' && github.ref == 'refs/heads/main')
+      (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/gateway/integration'))
     needs: [ build-rustfs-debug-binary ]
     runs-on: sm-standard-2
     timeout-minutes: 55

.github/workflows/connect-diagnostic-schedule-acceptance.yml

@@ -57,14 +57,6 @@ jobs:
           ref: ${{ inputs.connect_sha }}
           token: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
-      - name: Set up Docker Compose
-        uses: ./.github/actions/setup-docker-compose
-
-      - name: Set up Python
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
-        with:
-          python-version: "3.14"
-
       - name: Set up Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
@@ -102,7 +94,7 @@ jobs:
           ' <<<"$run" >/dev/null
           jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
           jq -e '
-            [.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu\\)$"))] as $matches
+            [.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu, [a-z0-9-]+, x86_64-unknown-linux-gnu, false, linux, pyroscope\\)$"))] as $matches
             | ($matches | length) == 1 and $matches[0].conclusion == "success"
           ' <<<"$jobs" >/dev/null
           artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
@@ -145,7 +137,7 @@ jobs:
               assert outer.testzip() is None
               with zipfile.ZipFile(io.BytesIO(outer.read(package))) as inner:
                   entries = inner.infolist()
-                  assert {item.filename for item in entries} == {'rustfs', 'rustfs.cpu-symbol-catalog.json', 'rustfs.cpu-symbol-catalog.sha256'} and len(entries) == 3
+                  assert {item.filename for item in entries} == {'rustfs', 'rustfs-cli'} and len(entries) == 2
                   assert all(regular(item) and 0 < item.file_size <= 1073741824 for item in entries)
                   assert inner.testzip() is None
                   pathlib.Path('binary').mkdir()
@@ -170,7 +162,7 @@ jobs:
           connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
           make -C connect-harness e2e-connected-dispatch-check
           (cd connect-harness && ./tests/connectivity/diagnostic-schedule-evidence.test.sh)
-          TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
             RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
             CONNECT_E2E_DIAGNOSTIC_SCHEDULE_EVIDENCE="$GITHUB_WORKSPACE/diagnostic-schedule-evidence.json" \
             make -C connect-harness e2e-connected E2E_SCENARIO=diagnostic-schedule

.github/workflows/connect-health-service-acceptance.yml

@@ -57,14 +57,6 @@ jobs:
           ref: ${{ inputs.connect_sha }}
           token: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
-      - name: Set up Docker Compose
-        uses: ./.github/actions/setup-docker-compose
-
-      - name: Set up Python
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
-        with:
-          python-version: "3.14"
-
       - name: Set up Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
@@ -135,7 +127,7 @@ jobs:
           ' <<<"$run" >/dev/null
           jobs=$(gh api --paginate --slurp "repos/rustfs/rustfs/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
           jq -e '
-            [.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu\\)$"))] as $matches
+            [.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu, [a-z0-9-]+, x86_64-unknown-linux-gnu, false, linux, pyroscope\\)$"))] as $matches
             | ($matches | length) == 1 and $matches[0].conclusion == "success"
           ' <<<"$jobs" >/dev/null
           artifact=$(gh api "repos/rustfs/rustfs/actions/artifacts/${ARTIFACT_ID}")
@@ -180,7 +172,7 @@ jobs:
               assert outer.testzip() is None
               with zipfile.ZipFile(io.BytesIO(outer.read(package))) as inner:
                   entries = inner.infolist()
-                  assert {entry.filename for entry in entries} == {'rustfs', 'rustfs.cpu-symbol-catalog.json', 'rustfs.cpu-symbol-catalog.sha256'} and len(entries) == 3
+                  assert {entry.filename for entry in entries} == {'rustfs', 'rustfs-cli', 'rustfs.cpu-symbol-catalog.json', 'rustfs.cpu-symbol-catalog.sha256'} and len(entries) == 4
                   assert all(regular(entry) and 0 < entry.file_size <= 1073741824 for entry in entries)
                   assert inner.testzip() is None
                   pathlib.Path('binary').mkdir()
@@ -207,7 +199,7 @@ jobs:
           connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
           make -C connect-harness e2e-connected-dispatch-check
           (cd connect-harness && ./tests/connectivity/health-service-evidence.test.sh)
-          TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
             RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
             CONNECT_E2E_HEALTH_SERVICE_EVIDENCE="$GITHUB_WORKSPACE/health-service-evidence.json" \
             make -C connect-harness e2e-connected E2E_SCENARIO=health-service

.github/workflows/connect-performance-drive-acceptance.yml

@@ -73,14 +73,6 @@ jobs:
           ref: ${{ inputs.connect_sha }}
           token: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
-      - name: Set up Docker Compose
-        uses: ./.github/actions/setup-docker-compose
-
-      - name: Set up Python
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
-        with:
-          python-version: "3.14"
-
       - name: Set up Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
@@ -115,7 +107,7 @@ jobs:
           [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
           [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
 
-          expected_job='Build RustFS (linux-x86_64-gnu)'
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-4, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
           jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
           jq -e --arg name "$expected_job" '
             [.[].jobs[] | select(.name == $name)] as $matches
@@ -162,7 +154,7 @@ jobs:
           npm --prefix connect-harness/web ci
           connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
           make -C connect-harness e2e-connected-dispatch-check
-          TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
             RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
             CONNECT_E2E_PERFORMANCE_DRIVE_EVIDENCE="$GITHUB_WORKSPACE/performance-drive-service-job-evidence.json" \
             make -C connect-harness e2e-connected E2E_SCENARIO=performance-drive

.github/workflows/connect-performance-network-acceptance.yml

@@ -73,14 +73,6 @@ jobs:
           ref: ${{ inputs.connect_sha }}
           token: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
-      - name: Set up Docker Compose
-        uses: ./.github/actions/setup-docker-compose
-
-      - name: Set up Python
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
-        with:
-          python-version: "3.14"
-
       - name: Set up Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
@@ -115,7 +107,7 @@ jobs:
           [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
           [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
 
-          expected_job='Build RustFS (linux-x86_64-gnu)'
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-4, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
           jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
           jq -e --arg name "$expected_job" '
             [.[].jobs[] | select(.name == $name)] as $matches
@@ -162,7 +154,7 @@ jobs:
           npm --prefix connect-harness/web ci
           connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
           make -C connect-harness e2e-connected-dispatch-check
-          TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
             RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
             CONNECT_E2E_PERFORMANCE_NETWORK_EVIDENCE="$GITHUB_WORKSPACE/performance-network-service-job-evidence.json" \
             make -C connect-harness e2e-connected E2E_SCENARIO=performance-network

.github/workflows/connect-profile-cpu-acceptance.yml

@@ -77,14 +77,6 @@ jobs:
           ref: ${{ inputs.connect_sha }}
           token: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
-      - name: Set up Docker Compose
-        uses: ./.github/actions/setup-docker-compose
-
-      - name: Set up Python
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
-        with:
-          python-version: "3.14"
-
       - name: Set up Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
@@ -126,7 +118,7 @@ jobs:
           [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
           [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
 
-          expected_job='Build RustFS (linux-x86_64-gnu)'
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-4, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
           jobs=$(github_api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
           jq -e --arg name "$expected_job" '
             [.jobs[] | select(.name == $name)] as $matches
@@ -181,7 +173,7 @@ jobs:
           npm --prefix connect-harness/web ci
           connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
           make -C connect-harness e2e-connected-dispatch-check
-          TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
             RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
             CONNECT_E2E_CPU_CATALOG="$GITHUB_WORKSPACE/binary/rustfs.cpu-symbol-catalog.json" \
             CONNECT_E2E_CPU_CATALOG_SHA256="$CATALOG_SHA256" \

.github/workflows/connect-profile-memory-acceptance.yml

@@ -57,14 +57,6 @@ jobs:
           ref: ${{ inputs.connect_sha }}
           token: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
-      - name: Set up Docker Compose
-        uses: ./.github/actions/setup-docker-compose
-
-      - name: Set up Python
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
-        with:
-          python-version: "3.14"
-
       - name: Set up Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
@@ -135,7 +127,7 @@ jobs:
           ' <<<"$run" >/dev/null
           jobs=$(gh api --paginate --slurp "repos/rustfs/rustfs/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
           jq -e '
-            [.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu\\)$"))] as $matches
+            [.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu, [a-z0-9-]+, x86_64-unknown-linux-gnu, false, linux, pyroscope\\)$"))] as $matches
             | ($matches | length) == 1 and $matches[0].conclusion == "success"
           ' <<<"$jobs" >/dev/null
           artifact=$(gh api "repos/rustfs/rustfs/actions/artifacts/${ARTIFACT_ID}")
@@ -180,7 +172,7 @@ jobs:
               assert outer.testzip() is None
               with zipfile.ZipFile(io.BytesIO(outer.read(package))) as inner:
                   entries = inner.infolist()
-                  expected = {'rustfs', 'rustfs.cpu-symbol-catalog.json', 'rustfs.cpu-symbol-catalog.sha256'}
+                  expected = {'rustfs', 'rustfs-cli', 'rustfs.cpu-symbol-catalog.json', 'rustfs.cpu-symbol-catalog.sha256'}
                   assert {entry.filename for entry in entries} == expected and len(entries) == len(expected)
                   assert all(regular(entry) and 0 < entry.file_size <= 1073741824 for entry in entries)
                   assert inner.testzip() is None
@@ -206,7 +198,7 @@ jobs:
           connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
           make -C connect-harness e2e-connected-dispatch-check
           (cd connect-harness && ./tests/connectivity/profile-memory-evidence.test.sh)
-          TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
             RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
             CONNECT_E2E_PROFILE_MEMORY_EVIDENCE="$GITHUB_WORKSPACE/profile-memory-evidence.json" \
             make -C connect-harness e2e-connected E2E_SCENARIO=profile-memory

.github/workflows/connect-profile-threads-acceptance.yml

@@ -81,14 +81,6 @@ jobs:
           ref: ${{ inputs.connect_sha }}
           token: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
-      - name: Set up Docker Compose
-        uses: ./.github/actions/setup-docker-compose
-
-      - name: Set up Python
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
-        with:
-          python-version: "3.14"
-
       - name: Set up Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
@@ -156,7 +148,7 @@ jobs:
           [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
           [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
 
-          expected_job='Build RustFS (linux-x86_64-gnu)'
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-4, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
           jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
           jq -e --arg name "$expected_job" '
             [.[].jobs[] | select(.name == $name)] as $matches
@@ -209,7 +201,7 @@ jobs:
           npm --prefix connect-harness/web ci
           connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
           make -C connect-harness e2e-connected-dispatch-check
-          TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
             RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
             CONNECT_E2E_PROFILE_THREADS_EVIDENCE="$GITHUB_WORKSPACE/profile-threads-service-job-evidence.json" \
             make -C connect-harness e2e-connected E2E_SCENARIO=profile-threads

.github/workflows/connect-top-api-acceptance.yml

@@ -73,14 +73,6 @@ jobs:
           ref: ${{ inputs.connect_sha }}
           token: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
-      - name: Set up Docker Compose
-        uses: ./.github/actions/setup-docker-compose
-
-      - name: Set up Python
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
-        with:
-          python-version: "3.14"
-
       - name: Set up Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
@@ -115,7 +107,7 @@ jobs:
           [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
           [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
 
-          expected_job='Build RustFS (linux-x86_64-gnu)'
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-4, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
           jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
           jq -e --arg name "$expected_job" '
             [.[].jobs[] | select(.name == $name)] as $matches
@@ -162,7 +154,7 @@ jobs:
           npm --prefix connect-harness/web ci
           connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
           make -C connect-harness e2e-connected-dispatch-check
-          TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
             RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
             CONNECT_E2E_TOP_API_EVIDENCE="$GITHUB_WORKSPACE/top-api-service-job-evidence.json" \
             make -C connect-harness e2e-connected E2E_SCENARIO=top-api

.github/workflows/connect-top-disk-acceptance.yml

@@ -68,14 +68,6 @@ jobs:
           ref: ${{ inputs.connect_sha }}
           token: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
-      - name: Set up Docker Compose
-        uses: ./.github/actions/setup-docker-compose
-
-      - name: Set up Python
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
-        with:
-          python-version: "3.14"
-
       - name: Set up Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
@@ -146,7 +138,7 @@ jobs:
           ' <<<"$run" >/dev/null
           jobs=$(gh api --paginate --slurp "repos/rustfs/rustfs/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
           jq -e '
-            [.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu\\)$"))] as $matches
+            [.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu, [a-z0-9-]+, x86_64-unknown-linux-gnu, false, linux, pyroscope\\)$"))] as $matches
             | ($matches | length) == 1 and $matches[0].conclusion == "success"
           ' <<<"$jobs" >/dev/null
           artifact=$(gh api "repos/rustfs/rustfs/actions/artifacts/${ARTIFACT_ID}")
@@ -191,7 +183,7 @@ jobs:
               assert outer.testzip() is None
               with zipfile.ZipFile(io.BytesIO(outer.read(package))) as inner:
                   entries = inner.infolist()
-                  expected = {'rustfs', 'rustfs.cpu-symbol-catalog.json', 'rustfs.cpu-symbol-catalog.sha256'}
+                  expected = {'rustfs', 'rustfs-cli', 'rustfs.cpu-symbol-catalog.json', 'rustfs.cpu-symbol-catalog.sha256'}
                   assert {entry.filename for entry in entries} == expected and len(entries) == len(expected)
                   assert all(regular(entry) and 0 < entry.file_size <= 1073741824 for entry in entries)
                   assert inner.testzip() is None
@@ -221,7 +213,7 @@ jobs:
           npm --prefix connect-harness/web ci
           connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
           make -C connect-harness e2e-connected-dispatch-check
-          TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
             RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
             CONNECT_E2E_TOP_DISK_EVIDENCE="$GITHUB_WORKSPACE/top-disk-service-job-evidence.json" \
             make -C connect-harness e2e-connected E2E_SCENARIO=top-disk

.github/workflows/connect-top-locks-acceptance.yml

@@ -73,14 +73,6 @@ jobs:
           ref: ${{ inputs.connect_sha }}
           token: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
-      - name: Set up Docker Compose
-        uses: ./.github/actions/setup-docker-compose
-
-      - name: Set up Python
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
-        with:
-          python-version: "3.14"
-
       - name: Set up Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
@@ -115,7 +107,7 @@ jobs:
           [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
           [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
 
-          expected_job='Build RustFS (linux-x86_64-gnu)'
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-4, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
           jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
           jq -e --arg name "$expected_job" '
             [.[].jobs[] | select(.name == $name)] as $matches
@@ -162,7 +154,7 @@ jobs:
           npm --prefix connect-harness/web ci
           connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
           make -C connect-harness e2e-connected-dispatch-check
-          TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
             RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
             CONNECT_E2E_TOP_LOCKS_EVIDENCE="$GITHUB_WORKSPACE/top-locks-service-job-evidence.json" \
             make -C connect-harness e2e-connected E2E_SCENARIO=top-locks

.github/workflows/connect-top-rpc-acceptance.yml

@@ -73,14 +73,6 @@ jobs:
           ref: ${{ inputs.connect_sha }}
           token: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
-      - name: Set up Docker Compose
-        uses: ./.github/actions/setup-docker-compose
-
-      - name: Set up Python
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
-        with:
-          python-version: "3.14"
-
       - name: Set up Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
@@ -115,7 +107,7 @@ jobs:
           [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
           [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
 
-          expected_job='Build RustFS (linux-x86_64-gnu)'
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-4, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
           jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
           jq -e --arg name "$expected_job" '
             [.[].jobs[] | select(.name == $name)] as $matches
@@ -162,7 +154,7 @@ jobs:
           npm --prefix connect-harness/web ci
           connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
           make -C connect-harness e2e-connected-dispatch-check
-          TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
             RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
             CONNECT_E2E_TOP_RPC_EVIDENCE="$GITHUB_WORKSPACE/top-rpc-service-job-evidence.json" \
             make -C connect-harness e2e-connected E2E_SCENARIO=top-rpc

.github/workflows/nightly-gnu.yml

@@ -80,24 +80,13 @@ jobs:
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
-          cache-shared-key: build-x86_64-unknown-linux-gnu-glibc-2.31
+          cache-shared-key: build-x86_64-unknown-linux-gnu
           cache-save-if: 'false'
-          install-cross-tools: 'true'
-          target: x86_64-unknown-linux-gnu
           install-build-packaging-tools: 'false'
           install-test-tools: 'false'
 
       - name: Build RustFS
-        # Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
-        # nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
-        run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
-
-      - name: Verify glibc symbol baseline
-        run: |
-          python3 scripts/release/check_glibc.py 2.31 \
-            target/x86_64-unknown-linux-gnu/release/rustfs \
-            target/x86_64-unknown-linux-gnu/release/rustfs-cli
-          target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
+        run: cargo build --release --locked --target x86_64-unknown-linux-gnu -p rustfs --bins
 
       - name: Build DEB package
         id: deb

.gitignore

@@ -90,4 +90,3 @@ worktrees/*
 # insta scratch files; the accepted .snap files ARE the assertions and are committed
 *.snap.new
 *.pending-snap
-.mimosa

ARCHITECTURE.md

@@ -155,7 +155,7 @@ default build (lifecycle:
    - ⚠️ PARTIALLY VIOLATED: serving-side `s3s` references remain in ecstore
      (bucket metadata/replication/lifecycle DTOs and error mapping). The
      count is ratcheted shrink-only by `scripts/check_s3s_footprint.sh`
-     (`S3S_ECSTORE_FILES_BASELINE`; the `object_lock` module was converted to
+     (the `ecstore_files` line of `.config/s3s-footprint-baseline.txt`; the `object_lock` module was converted to
      storage-level types as the first ratchet step). Target state: the
      baseline reaches zero and ecstore's `Cargo.toml` drops `s3s`.
 

Cargo.toml

@@ -48,7 +48,7 @@ members = [
     "crates/replication", # Replication contracts and wire formats
     "crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
     "crates/s3-client", # S3 client for engine-side consumption of remote S3 endpoints (tiering, transition targets)
-    "crates/s3-types", # S3 event type definitions
+    "crates/s3-types", # S3 event and error type definitions
     "crates/s3-ops", # S3 operation definitions and mapping
     "crates/s3select-api", # S3 Select API interface
     "crates/s3select-query", # S3 Select query engine
@@ -74,7 +74,7 @@ edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
 rust-version = "1.99.0"
-version = "1.0.2"
+version = "1.0.1"
 homepage = "https://rustfs.com"
 description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
 keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
@@ -91,56 +91,56 @@ redundant_clone = "warn"
 
 [workspace.dependencies]
 # RustFS Internal Crates
-rustfs = { path = "./rustfs", version = "1.0.2" }
-rustfs-heal = { path = "crates/heal", version = "1.0.2" }
-rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.2" }
-rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.2" }
-rustfs-audit = { path = "crates/audit", version = "1.0.2" }
-rustfs-checksums = { path = "crates/checksums", version = "1.0.2" }
-rustfs-common = { path = "crates/common", version = "1.0.2" }
-rustfs-data-usage = { path = "crates/data-usage", version = "1.0.2" }
-rustfs-config = { path = "./crates/config", version = "1.0.2" }
-rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.2" }
-rustfs-credentials = { path = "crates/credentials", version = "1.0.2" }
-rustfs-crypto = { path = "crates/crypto", version = "1.0.2" }
-rustfs-ecstore = { path = "crates/ecstore", version = "1.0.2" }
-rustfs-filemeta = { path = "crates/filemeta", version = "1.0.2" }
-rustfs-iam = { path = "crates/iam", version = "1.0.2" }
-rustfs-keystone = { path = "crates/keystone", version = "1.0.2" }
-rustfs-license = { path = "crates/license", version = "1.0.2" }
-rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.2" }
-rustfs-kms = { path = "crates/kms", version = "1.0.2" }
-rustfs-lock = { path = "crates/lock", version = "1.0.2" }
-rustfs-madmin = { path = "crates/madmin", version = "1.0.2" }
-rustfs-notify = { path = "crates/notify", version = "1.0.2" }
-rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.2" }
-rustfs-io-core = { path = "crates/io-core", version = "1.0.2" }
-rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.2" }
-rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.2", default-features = false }
-rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.2" }
-rustfs-obs = { path = "crates/obs", version = "1.0.2" }
-rustfs-policy = { path = "crates/policy", version = "1.0.2" }
-rustfs-protos = { path = "crates/protos", version = "1.0.2" }
-rustfs-protocols = { path = "crates/protocols", version = "1.0.2" }
-rustfs-replication = { path = "crates/replication", version = "1.0.2" }
-rustfs-rio = { path = "crates/rio", version = "1.0.2" }
-rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.2" }
-rustfs-s3-client = { path = "crates/s3-client", version = "1.0.2" }
-rustfs-s3-types = { path = "crates/s3-types", version = "1.0.2" }
-rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.2" }
-rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.2" }
-rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.2" }
-rustfs-scanner = { path = "crates/scanner", version = "1.0.2" }
-rustfs-security-governance = { path = "crates/security-governance", version = "1.0.2" }
-rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.2" }
-rustfs-signer = { path = "crates/signer", version = "1.0.2" }
-rustfs-storage-api = { path = "crates/storage-api", version = "1.0.2" }
-rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.2" }
-rustfs-targets = { path = "crates/targets", version = "1.0.2" }
-rustfs-test-utils = { path = "crates/test-utils", version = "1.0.2" }
-rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.2" }
-rustfs-utils = { path = "crates/utils", version = "1.0.2" }
-rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
+rustfs = { path = "./rustfs", version = "1.0.1" }
+rustfs-heal = { path = "crates/heal", version = "1.0.1" }
+rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.1" }
+rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.1" }
+rustfs-audit = { path = "crates/audit", version = "1.0.1" }
+rustfs-checksums = { path = "crates/checksums", version = "1.0.1" }
+rustfs-common = { path = "crates/common", version = "1.0.1" }
+rustfs-data-usage = { path = "crates/data-usage", version = "1.0.1" }
+rustfs-config = { path = "./crates/config", version = "1.0.1" }
+rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.1" }
+rustfs-credentials = { path = "crates/credentials", version = "1.0.1" }
+rustfs-crypto = { path = "crates/crypto", version = "1.0.1" }
+rustfs-ecstore = { path = "crates/ecstore", version = "1.0.1" }
+rustfs-filemeta = { path = "crates/filemeta", version = "1.0.1" }
+rustfs-iam = { path = "crates/iam", version = "1.0.1" }
+rustfs-keystone = { path = "crates/keystone", version = "1.0.1" }
+rustfs-license = { path = "crates/license", version = "1.0.1" }
+rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.1" }
+rustfs-kms = { path = "crates/kms", version = "1.0.1" }
+rustfs-lock = { path = "crates/lock", version = "1.0.1" }
+rustfs-madmin = { path = "crates/madmin", version = "1.0.1" }
+rustfs-notify = { path = "crates/notify", version = "1.0.1" }
+rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.1" }
+rustfs-io-core = { path = "crates/io-core", version = "1.0.1" }
+rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.1" }
+rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.1", default-features = false }
+rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.1" }
+rustfs-obs = { path = "crates/obs", version = "1.0.1" }
+rustfs-policy = { path = "crates/policy", version = "1.0.1" }
+rustfs-protos = { path = "crates/protos", version = "1.0.1" }
+rustfs-protocols = { path = "crates/protocols", version = "1.0.1" }
+rustfs-replication = { path = "crates/replication", version = "1.0.1" }
+rustfs-rio = { path = "crates/rio", version = "1.0.1" }
+rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.1" }
+rustfs-s3-client = { path = "crates/s3-client", version = "1.0.1" }
+rustfs-s3-types = { path = "crates/s3-types", version = "1.0.1" }
+rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.1" }
+rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.1" }
+rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.1" }
+rustfs-scanner = { path = "crates/scanner", version = "1.0.1" }
+rustfs-security-governance = { path = "crates/security-governance", version = "1.0.1" }
+rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.1" }
+rustfs-signer = { path = "crates/signer", version = "1.0.1" }
+rustfs-storage-api = { path = "crates/storage-api", version = "1.0.1" }
+rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.1" }
+rustfs-targets = { path = "crates/targets", version = "1.0.1" }
+rustfs-test-utils = { path = "crates/test-utils", version = "1.0.1" }
+rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.1" }
+rustfs-utils = { path = "crates/utils", version = "1.0.1" }
+rustfs-zip = { path = "./crates/zip", version = "1.0.1" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
@@ -156,9 +156,8 @@ futures-lite = "2.6.1"
 futures-util = "0.3.34"
 pollster = "1.0.1"
 pulsar = { default-features = false, version = "6.9.0" }
-lapin = { default-features = false, version = "4.12.2" }
+lapin = { default-features = false, version = "4.12.0" }
 hyper = { version = "1.12.0" }
-h2 = { version = "0.4.20" }
 hyper-rustls = { default-features = false, version = "0.27.10" }
 hyper-util = { version = "0.1.21" }
 http = "1.5.0"
@@ -169,11 +168,11 @@ minlz = "1.2.3"
 reqwest = "0.13.5"
 rustfs-kafka-async = { version = "1.4.0" }
 socket2 = { version = "0.6.5" }
-tokio = { version = "1.53.3" }
+tokio = { version = "1.53.2" }
 tokio-rustls = { default-features = false, version = "0.26.6" }
 tokio-stream = { version = "0.1.19" }
 tokio-test = "0.4.6"
-tokio-util = { version = "0.7.20" }
+tokio-util = { version = "0.7.19" }
 tonic = { version = "0.14.6" }
 tonic-prost = { version = "0.14.6" }
 tonic-prost-build = { version = "0.14.6" }
@@ -181,7 +180,7 @@ tower = { version = "0.5.3" }
 tower-http = { version = "0.7.1" }
 h3 = "0.0.8"
 h3-quinn = "0.0.10"
-quinn = { version = "0.11.12", default-features = false, features = [
+quinn = { version = "0.11.11", default-features = false, features = [
       "futures-io",
       "runtime-tokio",
       "rustls-aws-lc-rs",
@@ -201,7 +200,7 @@ rmp-serde = { version = "1.3.1" }
 serde = { version = "1.0.229" }
 serde_ignored = { version = "0.1" }
 serde_with = { version = "3", default-features = false, features = ["macros", "std"] }
-serde_json = { version = "1.0.154" }
+serde_json = { version = "1.0.151" }
 serde_urlencoded = "0.7.1"
 
 # Cryptography and Security
@@ -225,7 +224,7 @@ rustls-native-certs = "0.8"
 rustls-pki-types = "1.15.1"
 x509-parser = "0.18.1"
 sha1 = "0.11.0"
-sha2 = "0.11.1"
+sha2 = "0.11.0"
 subtle = "2.6"
 zeroize = { version = "1.9.1" }
 proptest = "1"
@@ -269,7 +268,7 @@ crossbeam-queue = "0.3.14"
 crossbeam-channel = "0.5.17"
 crossbeam-deque = "0.8.8"
 crossbeam-utils = "0.8.23"
-datafusion = { default-features = false, version = "55.2.0" }
+datafusion = { default-features = false, version = "55.1.0" }
 derive_builder = "0.20.2"
 ed25519-dalek = "3.0.0"
 enumset = "1.1.14"
@@ -320,18 +319,23 @@ rumqttc = { package = "rumqttc-next", version = "0.34.0" }
 redis = { version = "1.7.1" }
 rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.5" }
-rust-embed = { version = "8.13.0" }
+rust-embed = { version = "8.12.0" }
 rustc-hash = { version = "2.1.3" }
 s3s = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
 s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" }
+# The same s3s pin under the name the legacy edge of the gateway migration uses
+# (crates/s3-types, feature `compat-s3s`). Keep `rev` identical to `s3s` above.
+# DELETE BY T4.2 (rustfs/backlog#2784).
+legacy-s3s = { package = "s3s", git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
 serial_test = "4.0.1"
 shadow-rs = { default-features = false, version = "2.0.0" }
 siphasher = "1.0.4"
-smallvec = { version = "1.16.3" }
+smallvec = { version = "1.16.2" }
 compact_str = "0.10.0"
 snap = "1.1.2"
 starshard = { version = "2.4.0" }
 strum = { version = "0.28.0" }
+sync_wrapper = "1.0.2"
 sysinfo = "0.39.6"
 temp-env = "0.3.6"
 tempfile = "3.27.0"
@@ -341,12 +345,12 @@ tracing = { version = "0.1.44" }
 tracing-appender = "0.2.5"
 tracing-core = "0.1.36"
 tracing-error = "0.2.1"
-tracing-opentelemetry = { version = "0.34.0" }
+tracing-opentelemetry = { version = "0.34" }
 tracing-subscriber = { version = "0.3.23" }
 transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
-uuid = { version = "1.28.0" }
+uuid = { version = "1.27.0" }
 vaultrs = { version = "0.8.0" }
 tar = "0.4.46"
 walkdir = "2.5.0"
@@ -354,26 +358,26 @@ winapi-util = "0.1.11"
 windows = { version = "0.62.2" }
 windows-sys = "0.61.2"
 xxhash-rust = { version = "0.8.19" }
-zip = "9.0.3"
-zstd = "0.14.2"
+zip = "8.6.0"
+zstd = "0.14.0"
 
 # Observability and Metrics
 metrics = "0.24.6"
 metrics-util = "0.20"
-dial9 = { version = "0.5.4", default-features = false }
-opentelemetry = { version = "0.33.1" }
-opentelemetry-appender-tracing = { version = "0.33.1" }
-opentelemetry-otlp = { version = "0.33.1" }
-opentelemetry-proto = { version = "0.33.1", default-features = false, features = ["metrics", "gen-tonic-messages"] }
-opentelemetry_sdk = { version = "0.33.1" }
-opentelemetry-semantic-conventions = { version = "0.33.1" }
-opentelemetry-stdout = { version = "0.33.1" }
-pyroscope = { version = "2.1.2" }
+dial9 = { version = "0.5.3", default-features = false }
+opentelemetry = { version = "0.33.0" }
+opentelemetry-appender-tracing = { version = "0.33.0" }
+opentelemetry-otlp = { version = "0.33.0" }
+opentelemetry-proto = { version = "0.33.0", default-features = false, features = ["metrics", "gen-tonic-messages"] }
+opentelemetry_sdk = { version = "0.33.0" }
+opentelemetry-semantic-conventions = { version = "0.33.0" }
+opentelemetry-stdout = { version = "0.33.0" }
+pyroscope = { version = "2.1.1" }
 
 # FTP and SFTP
 libunftp = { version = "0.23.1" }
 unftp-core = "0.1.0"
-suppaftp = { version = "12.2.0" }
+suppaftp = { version = "12.1.1" }
 rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
 russh = { version = "0.64.1" }
 russh-sftp = "3.0.1"
@@ -384,7 +388,7 @@ dav-server = "0.11.0"
 # Performance Analysis and Memory Profiling
 rustfs-mimalloc = { version = "0.5.6" }
 # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
-hotpath = { version = "0.28.6", default-features = false }
+hotpath = { version = "0.28.5", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.49" }
 

_typos.toml

@@ -51,8 +51,6 @@ unparseable = "unparseable"
 # Disaster-recovery objectives: recovery time and recovery point.
 RTO = "RTO"
 rto = "rto"
-# rustix OFlags variant (POSIX O_WRONLY equivalent).
-WRONLY = "WRONLY"
 
 [files]
 extend-exclude = []

crates/config/src/observability/mod.rs

@@ -51,11 +51,6 @@ pub const ENV_OBS_PROFILING_EXPORT_ENABLED: &str = "RUSTFS_OBS_PROFILING_EXPORT_
 /// PUT records multiple timers and histograms when attribution is active.
 pub const ENV_OBS_PUT_STAGE_METRICS_ENABLED: &str = "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED";
 
-/// Enables sampled in-process GetObject stage summaries without OTLP histograms.
-pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED";
-pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE";
-pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS";
-
 pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
 pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED";
 pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
@@ -164,15 +159,6 @@ mod tests {
         assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED");
         assert_eq!(ENV_OBS_PROFILING_EXPORT_ENABLED, "RUSTFS_OBS_PROFILING_EXPORT_ENABLED");
         assert_eq!(ENV_OBS_PUT_STAGE_METRICS_ENABLED, "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED");
-        assert_eq!(ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED, "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED");
-        assert_eq!(
-            ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE,
-            "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE"
-        );
-        assert_eq!(
-            ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS,
-            "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS"
-        );
         // Test log cleanup related env keys
         assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES");
         assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES");

crates/e2e_test/src/distributed/harness.rs

@@ -860,12 +860,10 @@ pub(crate) async fn start_rebalance(cluster: &RustFSTestClusterEnvironment) -> T
         if status.is_success() {
             break response;
         }
-        // The init stage answers 503 + Retry-After while the cluster capability
-        // proof is still missing; older binaries surfaced the same condition as
-        // a 500 carrying the proof marker.
-        let waiting_for_readiness = status == StatusCode::SERVICE_UNAVAILABLE
-            || (status == StatusCode::INTERNAL_SERVER_ERROR && response.contains("requires a live fleet capability proof"));
-        if waiting_for_readiness && Instant::now() < deadline {
+        if status == StatusCode::INTERNAL_SERVER_ERROR
+            && response.contains("requires a live fleet capability proof")
+            && Instant::now() < deadline
+        {
             sleep(Duration::from_secs(1)).await;
             continue;
         }

crates/e2e_test/src/dual_stack_differential/compare.rs

@@ -0,0 +1,512 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! What one stack answered, and how two answers are compared.
+//!
+//! An [`Observation`] keeps only what a client acts on: the HTTP status, the S3 error code, the
+//! allowlisted response headers and a few typed fields decoded from the response. The error
+//! message is kept for diagnostics and never compared: message wording is judged by the wire
+//! corpus, not here. The comparator is pure so it can be tested without a server.
+
+use serde::Serialize;
+use std::collections::{BTreeMap, BTreeSet};
+use std::path::{Path, PathBuf};
+
+/// Response headers compared exactly (after [`normalized_header_value`]).
+const HEADER_ALLOWLIST: [&str; 7] = [
+    "content-type",
+    "x-amz-version-id",
+    "etag",
+    "x-amz-delete-marker",
+    "content-range",
+    "accept-ranges",
+    "x-amz-request-charged",
+];
+
+/// Every response header with this prefix is compared as well.
+const HEADER_ALLOWLIST_PREFIX: &str = "x-amz-server-side-encryption";
+
+/// Placeholder for a field one side did not send.
+const ABSENT: &str = "<absent>";
+
+/// The lowercase header name if the comparator looks at this header.
+pub(crate) fn compared_header(name: &str) -> Option<String> {
+    let name = name.to_ascii_lowercase();
+    (HEADER_ALLOWLIST.contains(&name.as_str()) || name.starts_with(HEADER_ALLOWLIST_PREFIX)).then_some(name)
+}
+
+/// A version ID is minted per process, so its value never matches across two servers; only
+/// whether it is the literal `null` (an unversioned write) carries meaning.
+fn normalized_header_value(name: &str, value: &str) -> String {
+    if name == "x-amz-version-id" && value != "null" {
+        "<version-id>".to_owned()
+    } else {
+        value.to_owned()
+    }
+}
+
+/// One request's outcome as a client sees it.
+#[derive(Clone, Debug, Serialize)]
+pub(crate) struct Observation {
+    /// The S3 operation, named as in the `s3s::S3` trait (`GetObject`).
+    pub op: &'static str,
+    /// Which request of the script this was (`normal`, `negative`, or a setup label).
+    pub case: &'static str,
+    /// HTTP status; 0 when no response arrived.
+    pub status: u16,
+    /// The S3 error code, or the SDK's error kind when there was no service error.
+    pub error_code: Option<String>,
+    /// Allowlisted headers only, lowercase names, normalized values.
+    pub headers: BTreeMap<String, String>,
+    /// Typed values decoded from the response body, never free text.
+    pub fields: BTreeMap<String, String>,
+    /// Diagnostic only; never compared.
+    #[serde(skip)]
+    pub message: Option<String>,
+}
+
+impl Observation {
+    /// Builds an observation, keeping only the allowlisted headers.
+    pub(crate) fn new<'h>(
+        op: &'static str,
+        case: &'static str,
+        status: u16,
+        error_code: Option<String>,
+        headers: impl IntoIterator<Item = (&'h str, &'h str)>,
+    ) -> Self {
+        let headers = headers
+            .into_iter()
+            .filter_map(|(name, value)| compared_header(name).map(|name| (name, value)))
+            .map(|(name, value)| {
+                let value = normalized_header_value(&name, value);
+                (name, value)
+            })
+            .collect();
+        Self {
+            op,
+            case,
+            status,
+            error_code,
+            headers,
+            fields: BTreeMap::new(),
+            message: None,
+        }
+    }
+
+    pub(crate) fn with_field(mut self, name: &str, value: impl Into<String>) -> Self {
+        self.fields.insert(name.to_owned(), value.into());
+        self
+    }
+
+    pub(crate) fn is_success(&self) -> bool {
+        (200..300).contains(&self.status)
+    }
+}
+
+/// One field on which the two stacks answered differently.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct Difference {
+    pub op: String,
+    pub case: String,
+    pub field: String,
+    pub a: String,
+    pub b: String,
+}
+
+/// Every difference between two transcripts of the same script, matched by `(op, case)`.
+pub(crate) fn compare(a: &[Observation], b: &[Observation]) -> Vec<Difference> {
+    let mut differences = Vec::new();
+    let (index_a, duplicates_a) = index(a);
+    let (index_b, duplicates_b) = index(b);
+    for (op, case) in duplicates_a.union(&duplicates_b) {
+        let count = |side: &[Observation]| {
+            side.iter()
+                .filter(|observation| (observation.op, observation.case) == (*op, *case))
+                .count()
+                .to_string()
+        };
+        differences.push(difference(op, case, "duplicate_case", count(a), count(b)));
+    }
+    let keys: BTreeSet<_> = index_a.keys().chain(index_b.keys()).copied().collect();
+    for key @ (op, case) in keys {
+        match (index_a.get(&key), index_b.get(&key)) {
+            (Some(a), Some(b)) => compare_one(a, b, &mut differences),
+            (a, b) => {
+                let presence = |side: Option<&&Observation>| if side.is_some() { "present" } else { ABSENT }.to_owned();
+                differences.push(difference(op, case, "observation", presence(a), presence(b)));
+            }
+        }
+    }
+    differences
+}
+
+type Key = (&'static str, &'static str);
+
+fn index(side: &[Observation]) -> (BTreeMap<Key, &Observation>, BTreeSet<Key>) {
+    let mut index = BTreeMap::new();
+    let mut duplicates = BTreeSet::new();
+    for observation in side {
+        let key = (observation.op, observation.case);
+        if index.insert(key, observation).is_some() {
+            duplicates.insert(key);
+        }
+    }
+    (index, duplicates)
+}
+
+fn difference(op: &str, case: &str, field: &str, a: String, b: String) -> Difference {
+    Difference {
+        op: op.to_owned(),
+        case: case.to_owned(),
+        field: field.to_owned(),
+        a,
+        b,
+    }
+}
+
+fn compare_one(a: &Observation, b: &Observation, differences: &mut Vec<Difference>) {
+    let mut field = |name: String, value_a: Option<&str>, value_b: Option<&str>| {
+        if value_a != value_b {
+            let show = |value: Option<&str>| value.unwrap_or(ABSENT).to_owned();
+            differences.push(difference(a.op, a.case, &name, show(value_a), show(value_b)));
+        }
+    };
+    field(
+        "status".to_owned(),
+        Some(a.status.to_string().as_str()),
+        Some(b.status.to_string().as_str()),
+    );
+    field("error_code".to_owned(), a.error_code.as_deref(), b.error_code.as_deref());
+    let headers: BTreeSet<_> = a.headers.keys().chain(b.headers.keys()).collect();
+    for name in headers {
+        field(
+            format!("header.{name}"),
+            a.headers.get(name).map(String::as_str),
+            b.headers.get(name).map(String::as_str),
+        );
+    }
+    let fields: BTreeSet<_> = a.fields.keys().chain(b.fields.keys()).collect();
+    for name in fields {
+        field(
+            format!("body.{name}"),
+            a.fields.get(name).map(String::as_str),
+            b.fields.get(name).map(String::as_str),
+        );
+    }
+}
+
+/// Operations observed on both sides.
+pub(crate) fn compared_ops(a: &[Observation], b: &[Observation]) -> BTreeSet<&'static str> {
+    let ops_b: BTreeSet<_> = b.iter().map(|observation| observation.op).collect();
+    a.iter()
+        .map(|observation| observation.op)
+        .filter(|op| ops_b.contains(op))
+        .collect()
+}
+
+/// `(op, case)` pairs observed on both sides.
+pub(crate) fn compared_cases(a: &[Observation], b: &[Observation]) -> usize {
+    let cases_b: BTreeSet<_> = b.iter().map(|observation| (observation.op, observation.case)).collect();
+    a.iter()
+        .map(|observation| (observation.op, observation.case))
+        .collect::<BTreeSet<_>>()
+        .intersection(&cases_b)
+        .count()
+}
+
+/// The JSON report written for every run.
+#[derive(Debug, Serialize)]
+pub(crate) struct Report {
+    pub schema: u32,
+    pub run: String,
+    pub stack_a: String,
+    pub stack_b: String,
+    pub region_a: Option<String>,
+    pub region_b: Option<String>,
+    pub compared_ops: usize,
+    pub compared_cases: usize,
+    pub differences: Vec<Difference>,
+}
+
+impl Report {
+    /// Writes the report as `<directory>/<run>.json` and returns its path.
+    pub(crate) fn write_to(&self, directory: &Path) -> std::io::Result<PathBuf> {
+        std::fs::create_dir_all(directory)?;
+        let path = directory.join(format!("{}.json", self.run));
+        let json = serde_json::to_vec_pretty(self).map_err(std::io::Error::other)?;
+        std::fs::write(&path, json)?;
+        Ok(path)
+    }
+}
+
+/// One line per difference, for a failing assertion.
+pub(crate) fn render(differences: &[Difference]) -> String {
+    differences
+        .iter()
+        .map(|d| format!("({}, {}, {}, a={:?}, b={:?})", d.op, d.case, d.field, d.a, d.b))
+        .collect::<Vec<_>>()
+        .join("\n")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn ok(op: &'static str) -> Observation {
+        Observation::new(op, "normal", 200, None, [("content-type", "application/xml")])
+    }
+
+    fn fields(differences: &[Difference]) -> Vec<&str> {
+        differences.iter().map(|d| d.field.as_str()).collect()
+    }
+
+    #[test]
+    fn identical_transcripts_have_no_difference() {
+        let a = vec![
+            ok("HeadBucket"),
+            ok("GetBucketLocation").with_field("location_constraint", "us-east-1"),
+        ];
+        let b = a.clone();
+        assert_eq!(compare(&a, &b), Vec::new());
+        assert_eq!(compared_ops(&a, &b).len(), 2);
+    }
+
+    #[test]
+    fn a_status_difference_is_reported() {
+        let a = vec![ok("PutBucketAcl")];
+        let b = vec![Observation::new(
+            "PutBucketAcl",
+            "normal",
+            403,
+            None,
+            [("content-type", "application/xml")],
+        )];
+        assert_eq!(
+            compare(&a, &b),
+            vec![Difference {
+                op: "PutBucketAcl".into(),
+                case: "normal".into(),
+                field: "status".into(),
+                a: "200".into(),
+                b: "403".into(),
+            }]
+        );
+    }
+
+    #[test]
+    fn an_error_code_difference_is_reported() {
+        let a = vec![Observation::new("GetObject", "negative", 404, Some("NoSuchKey".into()), [])];
+        let b = vec![Observation::new(
+            "GetObject",
+            "negative",
+            404,
+            Some("NoSuchBucket".into()),
+            [],
+        )];
+        let differences = compare(&a, &b);
+        assert_eq!(fields(&differences), ["error_code"]);
+        assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("NoSuchKey", "NoSuchBucket"));
+    }
+
+    #[test]
+    fn an_error_code_on_one_side_only_is_reported() {
+        let a = vec![Observation::new("HeadBucket", "negative", 404, Some("NotFound".into()), [])];
+        let b = vec![Observation::new("HeadBucket", "negative", 404, None, [])];
+        let differences = compare(&a, &b);
+        assert_eq!(fields(&differences), ["error_code"]);
+        assert_eq!(differences[0].b, ABSENT);
+    }
+
+    #[test]
+    fn an_allowlisted_header_difference_is_reported() {
+        let a = vec![Observation::new("HeadObject", "normal", 200, None, [("ETag", "\"aa\"")])];
+        let b = vec![Observation::new("HeadObject", "normal", 200, None, [("etag", "\"bb\"")])];
+        assert_eq!(fields(&compare(&a, &b)), ["header.etag"]);
+    }
+
+    #[test]
+    fn a_header_sent_by_one_side_only_is_reported() {
+        let a = vec![Observation::new(
+            "DeleteObject",
+            "normal",
+            204,
+            None,
+            [("x-amz-delete-marker", "true")],
+        )];
+        let b = vec![Observation::new("DeleteObject", "normal", 204, None, [])];
+        let differences = compare(&a, &b);
+        assert_eq!(fields(&differences), ["header.x-amz-delete-marker"]);
+        assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("true", ABSENT));
+    }
+
+    #[test]
+    fn every_server_side_encryption_header_is_compared() {
+        let header = "x-amz-server-side-encryption-customer-key-md5";
+        let a = vec![Observation::new("PutObject", "sse-c", 200, None, [(header, "one")])];
+        let b = vec![Observation::new("PutObject", "sse-c", 200, None, [(header, "two")])];
+        assert_eq!(fields(&compare(&a, &b)), [format!("header.{header}")]);
+    }
+
+    #[test]
+    fn headers_outside_the_allowlist_are_not_compared() {
+        let a = vec![Observation::new(
+            "ListBuckets",
+            "normal",
+            200,
+            None,
+            [
+                ("date", "Mon"),
+                ("x-amz-request-id", "1"),
+                ("server", "a"),
+                ("content-length", "10"),
+            ],
+        )];
+        let b = vec![Observation::new(
+            "ListBuckets",
+            "normal",
+            200,
+            None,
+            [
+                ("date", "Tue"),
+                ("x-amz-request-id", "2"),
+                ("server", "b"),
+                ("content-length", "11"),
+            ],
+        )];
+        assert_eq!(compare(&a, &b), Vec::new());
+        assert!(a[0].headers.is_empty(), "non-allowlisted headers were kept: {:?}", a[0].headers);
+    }
+
+    #[test]
+    fn distinct_version_ids_are_not_a_difference() {
+        let a = vec![Observation::new(
+            "PutObject",
+            "normal",
+            200,
+            None,
+            [("x-amz-version-id", "0b1c")],
+        )];
+        let b = vec![Observation::new(
+            "PutObject",
+            "normal",
+            200,
+            None,
+            [("x-amz-version-id", "9f8e")],
+        )];
+        assert_eq!(compare(&a, &b), Vec::new());
+    }
+
+    #[test]
+    fn a_null_version_id_against_a_real_one_is_reported() {
+        let a = vec![Observation::new(
+            "PutObject",
+            "normal",
+            200,
+            None,
+            [("x-amz-version-id", "null")],
+        )];
+        let b = vec![Observation::new(
+            "PutObject",
+            "normal",
+            200,
+            None,
+            [("x-amz-version-id", "9f8e")],
+        )];
+        assert_eq!(fields(&compare(&a, &b)), ["header.x-amz-version-id"]);
+    }
+
+    #[test]
+    fn a_body_field_difference_is_reported() {
+        let a = vec![ok("GetBucketLocation").with_field("location_constraint", "us-east-1")];
+        let b = vec![ok("GetBucketLocation").with_field("location_constraint", "eu-west-1")];
+        let differences = compare(&a, &b);
+        assert_eq!(fields(&differences), ["body.location_constraint"]);
+        assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("us-east-1", "eu-west-1"));
+    }
+
+    #[test]
+    fn a_body_field_on_one_side_only_is_reported() {
+        let a = vec![ok("ListObjectsV2").with_field("keys", "[]")];
+        let b = vec![ok("ListObjectsV2")];
+        assert_eq!(fields(&compare(&a, &b)), ["body.keys"]);
+    }
+
+    #[test]
+    fn error_messages_are_never_compared() {
+        let mut a = Observation::new("GetObject", "negative", 404, Some("NoSuchKey".into()), []);
+        let mut b = a.clone();
+        a.message = Some("The specified key does not exist.".into());
+        b.message = Some("Object not found".into());
+        assert_eq!(compare(&[a], &[b]), Vec::new());
+    }
+
+    #[test]
+    fn a_case_observed_on_one_side_only_is_reported() {
+        let a = vec![ok("HeadBucket"), ok("ListBuckets")];
+        let b = vec![ok("HeadBucket")];
+        let differences = compare(&a, &b);
+        assert_eq!(fields(&differences), ["observation"]);
+        assert_eq!(differences[0].op, "ListBuckets");
+        assert_eq!(compared_ops(&a, &b).into_iter().collect::<Vec<_>>(), ["HeadBucket"]);
+    }
+
+    #[test]
+    fn a_case_recorded_twice_is_reported() {
+        let a = vec![ok("HeadBucket"), ok("HeadBucket")];
+        let b = vec![ok("HeadBucket")];
+        assert_eq!(fields(&compare(&a, &b)), ["duplicate_case"]);
+    }
+
+    #[test]
+    fn compared_ops_counts_operations_not_cases() {
+        let negative = Observation::new("HeadBucket", "negative", 404, Some("NotFound".into()), []);
+        let a = vec![ok("HeadBucket"), negative, ok("ListBuckets")];
+        let b = a.clone();
+        assert_eq!(compared_ops(&a, &b).len(), 2);
+    }
+
+    #[test]
+    fn the_report_lists_each_difference_as_op_field_a_b() {
+        let directory = std::env::temp_dir().join(format!("dual-stack-report-{}", uuid::Uuid::new_v4()));
+        let report = Report {
+            schema: 1,
+            run: "unit".into(),
+            stack_a: "legacy".into(),
+            stack_b: "legacy".into(),
+            region_a: None,
+            region_b: Some("eu-west-1".into()),
+            compared_ops: 1,
+            compared_cases: 1,
+            differences: compare(
+                &[ok("GetBucketLocation").with_field("location_constraint", "us-east-1")],
+                &[ok("GetBucketLocation").with_field("location_constraint", "eu-west-1")],
+            ),
+        };
+        let path = report.write_to(&directory).expect("write the report");
+        let json: serde_json::Value = serde_json::from_slice(&std::fs::read(&path).expect("read the report")).expect("parse");
+        let _ = std::fs::remove_dir_all(&directory);
+        assert_eq!(path.file_name().and_then(|name| name.to_str()), Some("unit.json"));
+        assert_eq!(json["compared_ops"], 1);
+        assert_eq!(
+            json["differences"],
+            serde_json::json!([{
+                "op": "GetBucketLocation",
+                "case": "normal",
+                "field": "body.location_constraint",
+                "a": "us-east-1",
+                "b": "eu-west-1",
+            }])
+        );
+    }
+}

crates/e2e_test/src/dual_stack_differential/mod.rs

@@ -0,0 +1,197 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Dual-stack differential: one `rustfs` binary, two processes, one SDK script (rustfs/backlog#2740).
+//!
+//! Stack A always runs `RUSTFS_S3_STACK=legacy`. Stack B runs the value of `RUSTFS_E2E_STACK_B`
+//! (default `legacy` until the gateway stack is assembled), and `RUSTFS_E2E_STACK_B_REGION`, when
+//! set, starts B in another region so that a difference can be injected by hand. Both servers
+//! answer the script in `scenarios`, which covers every operation of `impl S3 for FS`; `compare`
+//! reports each `(op, field, a, b)` on which the answers differ, and every run writes its report
+//! to `<target>/dual-stack/<run>.json`. Two processes, not two embedded servers: bucket metadata
+//! is process-global, so two stacks in one process would read each other's buckets.
+
+mod compare;
+mod scenarios;
+
+use crate::common::{RustFSTestEnvironment, init_logging, workspace_root};
+use compare::{Difference, Observation, Report, compare, compared_cases, compared_ops, render};
+use std::path::PathBuf;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+type BoxError = Box<dyn std::error::Error + Send + Sync>;
+
+const STACK_A: &str = "legacy";
+const ENV_STACK_B: &str = "RUSTFS_E2E_STACK_B";
+const ENV_STACK_B_REGION: &str = "RUSTFS_E2E_STACK_B_REGION";
+/// Becomes `gateway` once rustfs/backlog#2734 T2.5 assembles the gateway stack.
+const DEFAULT_STACK_B: &str = "legacy";
+/// The region the negative control gives stack A explicitly, so an inherited `RUSTFS_REGION`
+/// cannot erase the injected difference.
+const CONTROL_REGION_A: &str = "us-east-1";
+const CONTROL_REGION_B: &str = "eu-west-1";
+/// `normal` cases the legacy stack must answer with a 2xx. The comparison alone cannot tell a
+/// script that exercises real behavior from one in which both stacks fail the same way. Five of
+/// the 73 fail on legacy by design of this script: PutBucketReplication (no registered remote
+/// target), GetBucketReplication (so no configuration), PutBucketNotificationConfiguration (no
+/// notification target configured), RestoreObject (the object is not archived) and
+/// GetObjectTorrent (legacy answers `NoSuchKey`).
+const MIN_NORMAL_SUCCESSES: usize = 68;
+
+struct PairConfig {
+    stack_b: String,
+    region_a: Option<String>,
+    region_b: Option<String>,
+}
+
+impl PairConfig {
+    fn from_env() -> Self {
+        Self {
+            stack_b: std::env::var(ENV_STACK_B).unwrap_or_else(|_| DEFAULT_STACK_B.to_owned()),
+            region_a: None,
+            region_b: std::env::var(ENV_STACK_B_REGION).ok(),
+        }
+    }
+}
+
+/// Starts one server on `stack`, overriding its region only when asked.
+async fn start_stack(stack: &str, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
+    let mut env = RustFSTestEnvironment::new().await?;
+    let mut vars = vec![("RUSTFS_S3_STACK", stack)];
+    if let Some(region) = region {
+        vars.push(("RUSTFS_REGION", region));
+    }
+    env.start_rustfs_server_without_cleanup_with_env(&vars).await?;
+    Ok(env)
+}
+
+/// Starts stack A and stack B side by side, each with its own port and data directory.
+async fn spawn_pair(config: &PairConfig) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment), BoxError> {
+    tokio::try_join!(
+        start_stack(STACK_A, config.region_a.as_deref()),
+        start_stack(&config.stack_b, config.region_b.as_deref())
+    )
+}
+
+fn report_directory() -> PathBuf {
+    let target = std::env::var_os("CARGO_TARGET_DIR").map_or_else(|| PathBuf::from("target"), PathBuf::from);
+    let target = if target.is_absolute() {
+        target
+    } else {
+        workspace_root().join(target)
+    };
+    target.join("dual-stack")
+}
+
+struct Run {
+    report: Report,
+    path: PathBuf,
+    a: Vec<Observation>,
+}
+
+/// Runs the script against both stacks concurrently and writes the report.
+async fn run_differential(label: &str, config: &PairConfig) -> Run {
+    let (server_a, server_b) = spawn_pair(config).await.expect("start stack A and stack B");
+    let (a, b) = tokio::join!(Box::pin(scenarios::run(&server_a)), Box::pin(scenarios::run(&server_b)));
+    let millis = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map_or(0, |elapsed| elapsed.as_millis());
+    let report = Report {
+        schema: 1,
+        run: format!("{label}-{millis}-{}", std::process::id()),
+        stack_a: STACK_A.to_owned(),
+        stack_b: config.stack_b.clone(),
+        region_a: config.region_a.clone(),
+        region_b: config.region_b.clone(),
+        compared_ops: compared_ops(&a, &b).len(),
+        compared_cases: compared_cases(&a, &b),
+        differences: compare(&a, &b),
+    };
+    let path = report.write_to(&report_directory()).expect("write the dual-stack report");
+    Run { report, path, a }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn both_stacks_answer_every_operation_identically() {
+        init_logging();
+        let config = PairConfig::from_env();
+        let run = run_differential("self-comparison", &config).await;
+        let report = &run.report;
+
+        let mut observed: Vec<_> = run.a.iter().map(|observation| observation.op).collect();
+        observed.sort_unstable();
+        observed.dedup();
+        assert_eq!(
+            observed,
+            scenarios::OPERATIONS,
+            "the script must drive exactly the operations of impl S3 for FS"
+        );
+        assert_eq!(report.compared_ops, scenarios::OPERATIONS.len(), "report: {}", run.path.display());
+        assert!(
+            report.differences.is_empty(),
+            "stack {} and stack {} differ on {} of {} cases (report: {}):\n{}",
+            report.stack_a,
+            report.stack_b,
+            report.differences.len(),
+            report.compared_cases,
+            run.path.display(),
+            render(&report.differences)
+        );
+
+        let not_4xx: Vec<_> = run
+            .a
+            .iter()
+            .filter(|observation| observation.case == "negative" && !(400..500).contains(&observation.status))
+            .collect();
+        assert!(not_4xx.is_empty(), "negative cases must fail with a 4xx on {STACK_A}: {not_4xx:#?}");
+        let successes = run
+            .a
+            .iter()
+            .filter(|observation| observation.case == "normal" && observation.is_success())
+            .count();
+        assert!(
+            successes >= MIN_NORMAL_SUCCESSES,
+            "only {successes} normal cases succeeded on {STACK_A}: {:#?}",
+            run.a
+        );
+    }
+
+    #[tokio::test]
+    async fn differential_detects_an_injected_difference() {
+        init_logging();
+        let config = PairConfig {
+            stack_b: STACK_A.to_owned(),
+            region_a: Some(CONTROL_REGION_A.to_owned()),
+            region_b: Some(CONTROL_REGION_B.to_owned()),
+        };
+        let run = run_differential("injected-region", &config).await;
+        let expected = Difference {
+            op: "GetBucketLocation".to_owned(),
+            case: "normal".to_owned(),
+            field: "body.location_constraint".to_owned(),
+            a: CONTROL_REGION_A.to_owned(),
+            b: CONTROL_REGION_B.to_owned(),
+        };
+        assert!(
+            run.report.differences.contains(&expected),
+            "B started in {CONTROL_REGION_B} but GetBucketLocation was not reported (report: {}):\n{}",
+            run.path.display(),
+            render(&run.report.differences)
+        );
+    }
+}

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -1969,16 +1969,11 @@ mod tests {
             let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
             let status: serde_json::Value = serde_json::from_str(&status_body)
                 .map_err(|err| format!("pre-interrupt background heal status is not JSON ({err}): {status_body}"))?;
-            let admin_active = background_rejoin_heal_evidence
-                || (status["state"].as_str() == Some("active")
-                    && status["healOperations"]["activeBySource"]["admin"].as_u64() == Some(1));
-            if admin_active && (background_enabled || only_admin_heal_is_active(&status)) {
+            if background_enabled || only_admin_heal_is_active(&status) {
                 break status;
             }
             if Instant::now() >= partial_deadline {
-                return Err(
-                    format!("heal tasks did not reach the required interruption state before the deadline: {status}").into(),
-                );
+                return Err(format!("non-admin heal did not drain before the interruption deadline: {status}").into());
             }
             sleep(Duration::from_millis(50)).await;
         };
@@ -2638,157 +2633,4 @@ mod tests {
         };
         assert!(!outage_candidate_targets_replacement(&insufficient_peer_shards, 12, replacement_set_slot));
     }
-    async fn two_node_heal_status(
-        cluster: &RustFSTestClusterEnvironment,
-        bucket: Option<&str>,
-        recreate: bool,
-        scan_mode: u8,
-    ) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
-        let url = format!("{}/rustfs/admin/v3/heal/{}", cluster.nodes[0].url, bucket.unwrap_or_default());
-        let options = serde_json::json!({
-            "recursive": true, "dryRun": false, "remove": false,
-            "recreate": recreate, "scanMode": scan_mode, "updateParity": false, "nolock": false,
-        });
-        let started =
-            start_root_heal_when_control_ready(&url, &options.to_string(), &cluster.access_key, &cluster.secret_key).await?;
-        let started: serde_json::Value = serde_json::from_str(&started)?;
-        let token = started["clientToken"]
-            .as_str()
-            .ok_or_else(|| format!("missing heal token: {started}"))?;
-        let status_url = format!("{url}?clientToken={token}");
-        let deadline = Instant::now() + Duration::from_secs(240);
-        loop {
-            let body = timeout(
-                Duration::from_secs(20),
-                signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key),
-            )
-            .await??;
-            let status: serde_json::Value = serde_json::from_str(&body)?;
-            match status["summary"].as_str() {
-                Some("finished" | "stopped") => return Ok(status),
-                Some("running") if Instant::now() < deadline => sleep(Duration::from_millis(250)).await,
-                _ => return Err(format!("heal did not reach a terminal state: {status}").into()),
-            }
-        }
-    }
-
-    async fn verify_two_node_directory_recovery(
-        remove_bucket: bool,
-        scan_mode: u8,
-        bucket_scoped: bool,
-        recreate: bool,
-    ) -> Result<(), Box<dyn Error + Send + Sync>> {
-        init_logging();
-        let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
-        // Isolate explicit administrator recovery from scanner, MRF, and disk healing.
-        for key in [
-            "RUSTFS_HEAL_AUTO_HEAL_ENABLE",
-            "RUSTFS_HEAL_MRF_ENABLE",
-            "RUSTFS_SCANNER_ENABLED",
-            "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE",
-        ] {
-            cluster.set_env(key, "false");
-        }
-        cluster.start().await?;
-        let source = cluster.create_s3_client(0)?;
-        let target = cluster.create_s3_client(1)?;
-        let bucket = "two-node-directory-recovery";
-        source.create_bucket().bucket(bucket).send().await?;
-        let objects = [
-            ("empty", Vec::new()),
-            ("nested/inline", vec![0x3c; 1024]),
-            ("nested/external", (0_u8..=255).cycle().take(1024 * 1024).collect::<Vec<_>>()),
-        ];
-        for (key, bytes) in &objects {
-            source
-                .put_object()
-                .bucket(bucket)
-                .key(*key)
-                .body(ByteStream::from(bytes.clone()))
-                .send()
-                .await?;
-        }
-        let target_bucket = Path::new(&cluster.nodes[1].data_dir).join(bucket);
-        let original = objects
-            .iter()
-            .map(|(key, _)| census_object_version_on_disk(Path::new(&cluster.nodes[1].data_dir), bucket, key, None))
-            .collect::<Result<Vec<_>, _>>()?;
-        for census in &original {
-            assert!(census.has_xl_meta, "fixture must have a target metadata copy: {census:?}");
-        }
-        assert!(
-            !original[2].present_part_fingerprints.is_empty(),
-            "large fixture must exercise external parts"
-        );
-        let removed = Path::new(&cluster.temp_dir).join("removed");
-        tokio::fs::create_dir(&removed).await?;
-        if remove_bucket {
-            tokio::fs::rename(&target_bucket, removed.join(bucket)).await?;
-        } else {
-            for (index, (key, _)) in objects.iter().enumerate() {
-                tokio::fs::rename(target_bucket.join(key), removed.join(index.to_string())).await?;
-            }
-        }
-        let status = two_node_heal_status(&cluster, bucket_scoped.then_some(bucket), recreate, scan_mode).await?;
-        assert_eq!(status["summary"], "finished", "administrator recovery should complete: {status}");
-        assert_eq!(status["outcome"]["execution"]["state"], "completed");
-        assert_eq!(status["outcome"]["coverage"], "complete");
-        for ((key, _), before) in objects.iter().zip(&original) {
-            let after = census_object_version_on_disk(Path::new(&cluster.nodes[1].data_dir), bucket, key, None)?;
-            assert!(after.has_xl_meta, "metadata must be restored before GET: {key}");
-            assert_eq!(
-                after.present_part_fingerprints, before.present_part_fingerprints,
-                "external shard bytes must match: {key}"
-            );
-            assert_eq!(
-                after.inline_data_fingerprint, before.inline_data_fingerprint,
-                "inline shard bytes must match: {key}"
-            );
-        }
-        cluster.stop_node(0)?;
-        for (key, expected) in &objects {
-            let actual = timeout(Duration::from_secs(30), async {
-                target
-                    .get_object()
-                    .bucket(bucket)
-                    .key(*key)
-                    .send()
-                    .await?
-                    .body
-                    .collect()
-                    .await
-                    .map(|body| body.into_bytes())
-                    .map_err(Box::<dyn Error + Send + Sync>::from)
-            })
-            .await??;
-            assert_eq!(actual.len(), expected.len(), "target-only read length: {key}");
-            assert_eq!(sha256_hex(&actual), sha256_hex(expected), "target-only content: {key}");
-        }
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_two_node_bucket_discovery_and_explicit_recreate() -> Result<(), Box<dyn Error + Send + Sync>> {
-        verify_two_node_directory_recovery(true, 2, true, true).await
-    }
-
-    #[tokio::test]
-    async fn test_two_node_missing_bucket_default_cluster_recovery() -> Result<(), Box<dyn Error + Send + Sync>> {
-        verify_two_node_directory_recovery(true, 2, false, false).await
-    }
-
-    #[tokio::test]
-    async fn test_two_node_missing_bucket_default_scoped_recovery() -> Result<(), Box<dyn Error + Send + Sync>> {
-        verify_two_node_directory_recovery(true, 2, true, false).await
-    }
-
-    #[tokio::test]
-    async fn test_two_node_object_directory_recovery_normal() -> Result<(), Box<dyn Error + Send + Sync>> {
-        verify_two_node_directory_recovery(false, 1, false, false).await
-    }
-
-    #[tokio::test]
-    async fn test_two_node_object_directory_recovery_deep() -> Result<(), Box<dyn Error + Send + Sync>> {
-        verify_two_node_directory_recovery(false, 2, false, false).await
-    }
 }

crates/e2e_test/src/lib.rs

@@ -31,6 +31,12 @@ pub mod chaos;
 #[cfg(test)]
 pub mod fake_s3_target;
 
+// Dual-stack differential (rustfs/backlog#2740): one binary, two processes on
+// different RUSTFS_S3_STACK values, the same SDK script for every S3 operation,
+// compared on status, error code, allowlisted headers and typed body fields.
+#[cfg(test)]
+mod dual_stack_differential;
+
 // On-demand migration (backlog#2147): shared two-server environment, admin
 // wrappers, and the harness self-test (backlog#2151). Behavior scenarios are
 // added by later ODM tasks.

crates/e2e_test/src/namespace_lock_quorum_test.rs

@@ -195,91 +195,6 @@ async fn wait_for_quorum_read_admission(clients: &[Client], bucket: &str) -> Tes
     }
 }
 
-#[tokio::test]
-async fn test_cold_degraded_object_mutations_require_only_storage_quorum() -> TestResult {
-    crate::common::init_logging();
-    let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
-    cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2");
-    cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
-    cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
-    cluster.set_env("RUSTFS_GET_OBJECT_METADATA_CACHE_DISTRIBUTED_ENABLE", "false");
-    cluster.start().await?;
-    let clients = cluster
-        .create_all_clients()?
-        .into_iter()
-        .map(|client| {
-            Client::from_conf(
-                client
-                    .config()
-                    .to_builder()
-                    .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
-                    .build(),
-            )
-        })
-        .collect::<Vec<_>>();
-    let bucket = "cold-degraded-mutations";
-    clients[0].create_bucket().bucket(bucket).send().await?;
-    // No object mutation may warm a process-wide configuration probe before
-    // the missing peer is introduced. Three shards still satisfy EC:2 writes.
-    cluster.stop_node(3)?;
-    let admission_deadline = Instant::now() + Duration::from_secs(30);
-    for client in &clients[..3] {
-        loop {
-            match client
-                .get_object()
-                .bucket(bucket)
-                .key("never-created-readiness-probe")
-                .send()
-                .await
-            {
-                Err(error) if error.as_service_error().and_then(|error| error.meta().code()) == Some("NoSuchKey") => break,
-                Err(error)
-                    if error.raw_response().is_some_and(|response| response.status().as_u16() == 503)
-                        && Instant::now() < admission_deadline =>
-                {
-                    tokio::time::sleep(Duration::from_millis(100)).await;
-                }
-                Err(error) => return Err(error.into()),
-                Ok(_) => panic!("a cold readiness probe must not create an object"),
-            }
-        }
-    }
-    let body = b"first write while one peer is offline";
-    clients[0]
-        .put_object()
-        .bucket(bucket)
-        .key("object")
-        .body(Bytes::from_static(body).into())
-        .send()
-        .await?;
-    for client in &clients[..3] {
-        assert_quorum_object_body(client, bucket, "object", body).await?;
-    }
-    // This coordinator has not previously committed an object mutation either.
-    clients[1].delete_object().bucket(bucket).key("object").send().await?;
-    let error = clients[2]
-        .get_object()
-        .bucket(bucket)
-        .key("object")
-        .send()
-        .await
-        .expect_err("acknowledged deletion must be visible at another entrypoint");
-    assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("NoSuchKey"));
-
-    cluster.stop_node(2)?;
-    let error = clients[0]
-        .put_object()
-        .bucket(bucket)
-        .key("below-quorum")
-        .body(Bytes::from_static(body).into())
-        .send()
-        .await
-        .expect_err("removing the cache gate must not weaken storage write quorum");
-    assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
-    cluster.stop();
-    Ok(())
-}
-
 #[tokio::test]
 async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult {
     crate::common::init_logging();

crates/e2e_test/src/replication_extension_test.rs

@@ -3945,15 +3945,7 @@ async fn test_bucket_replication_version_purge_of_non_inline_object_releases_sou
         (sse_key, sse_put.version_id().ok_or("SSE PUT omitted version ID")?.to_string()),
         (large_key, large_put.version_id().ok_or("large PUT omitted version ID")?.to_string()),
     ];
-    // Managed-SSE listings may omit ETags. Check the replicated version's body
-    // before deleting it rather than comparing keyless listing fingerprints.
-    for (key, version_id) in &purged {
-        let source_body = get_version_body(&source_client, &source_bucket, key, version_id).await?;
-        let expected_sha256: [u8; 32] = Sha256::digest(&source_body).into();
-        wait_for_replicated_sha256(&target_client, &target_bucket, key, expected_sha256).await?;
-        let target_body = get_version_body(&target_client, &target_bucket, key, version_id).await?;
-        assert_eq!(target_body, source_body, "replicated version body differs for {key}/{version_id}");
-    }
+    assert_replication_converged(&source_client, &source_bucket, &target_client, &target_bucket).await?;
 
     for (key, version_id) in &purged {
         source_client
@@ -3964,6 +3956,10 @@ async fn test_bucket_replication_version_purge_of_non_inline_object_releases_sou
             .send()
             .await?;
     }
+    assert_replication_converged(&source_client, &source_bucket, &target_client, &target_bucket).await?;
+    let target_state = list_replication_state(&target_client, &target_bucket).await?;
+    assert!(target_state.is_empty(), "target retained an explicitly purged version: {target_state:?}");
+
     // The purge state is applied on the source asynchronously after the target
     // acknowledges the delete; only then does the retained version go away and
     // the bucket become deletable. A listing that is empty while DeleteBucket
@@ -3988,9 +3984,6 @@ async fn test_bucket_replication_version_purge_of_non_inline_object_releases_sou
         }
     }
 
-    let target_state = list_replication_state(&target_client, &target_bucket).await?;
-    assert!(target_state.is_empty(), "target retained an explicitly purged version: {target_state:?}");
-
     Ok(())
 }
 

crates/ecstore/Cargo.toml

@@ -183,7 +183,6 @@ sha2 = { workspace = true }
 hex-simd = { workspace = true }
 tempfile.workspace = true
 hyper = { workspace = true, features = ["http2", "http1", "server"] }
-h2 = { workspace = true }
 hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] }
 hyper-rustls = { workspace = true, default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs"] }
 hostname.workspace = true
@@ -201,8 +200,8 @@ md-5.workspace = true
 memmap2 = { workspace = true }
 libc.workspace = true
 # "process" adds getrlimit for the io_uring fd-cache RLIMIT_NOFILE gate
-# (backlog#1178); "thread" provides cleanup affinity; "fs" is a workspace default.
-rustix = { workspace = true, features = ["process", "fs", "thread"] }
+# (backlog#1178); "fs" comes from the workspace default.
+rustix = { workspace = true, features = ["process", "fs"] }
 rustfs-madmin.workspace = true
 reqwest = { workspace = true }
 aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }

crates/ecstore/src/api/mod.rs

@@ -425,11 +425,10 @@ pub mod disk {
         BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
         CheckPartsResp, ConditionalFileUpdate, DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction,
         DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation,
-        DeleteRecoveryRpcReply, DeleteRecoveryState, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption,
-        DiskStore, FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
-        PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
-        STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk,
-        validate_batch_read_version_item_count,
+        DeleteRecoveryRpcReply, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions,
+        FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, PartTransactionAction,
+        RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE,
+        SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, validate_batch_read_version_item_count,
     };
     pub use bytes::Bytes;
     pub use endpoint::Endpoint;

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -2417,16 +2417,6 @@ fn is_slow_down(err: &Error) -> bool {
     matches!(err, Error::SlowDown)
 }
 
-/// Lifecycle expiry is idempotent when the exact object or version has already
-/// disappeared between the scanner listing and the delete admission.
-///
-/// Keep this predicate narrower than bucket/volume "not found" helpers: an
-/// unavailable bucket or disk is an operational failure and must not be
-/// reported as a completed expiry.
-fn is_lifecycle_expiry_already_gone(err: &Error) -> bool {
-    is_err_object_not_found(err) || is_err_version_not_found(err)
-}
-
 /// Resolves the expiry worker count from the single documented knob,
 /// `RUSTFS_MAX_EXPIRY_WORKERS`: a set, parsable, non-zero value wins;
 /// anything else falls back to `min(cpus, 16)`. The historical
@@ -4930,20 +4920,6 @@ async fn expire_transitioned_object_with_lock_lost_signal(
                 //audit_log_lifecycle(*oi, ILMExpiry, tags, traceFn);
                 Ok(dobj)
             }
-            Err(err) if is_lifecycle_expiry_already_gone(&err) => {
-                debug!(
-                    event = EVENT_LIFECYCLE_DELETE_COMPLETED,
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                    bucket = %oi.bucket,
-                    object = %oi.name,
-                    operation = "delete_restored_object",
-                    reason = "already_gone",
-                    "Lifecycle delete target was already absent"
-                );
-                crate::object_api::notify_object_mutation(&oi.bucket, &oi.name).await;
-                Ok(oi.clone())
-            }
             Err(err) => Err(std::io::Error::other(err)),
         };
     }
@@ -4954,8 +4930,6 @@ async fn expire_transitioned_object_with_lock_lost_signal(
     // pointing to a remote tier version that has already been removed. If this
     // only creates a delete marker, remote cleanup must be driven by persisted
     // free-version recovery rather than the visible delete result.
-    // Keep remote transitioned not-found errors visible: local metadata
-    // absence alone does not prove that the tier object was cleaned up.
     let dobj = match api.delete_object(&oi.bucket, &oi.name, opts).await {
         Ok(obj) => obj,
         Err(e) => {
@@ -5828,20 +5802,6 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
         .await
     {
         Ok(dobj) => dobj,
-        Err(err) if is_lifecycle_expiry_already_gone(&err) => {
-            debug!(
-                event = EVENT_LIFECYCLE_DELETE_COMPLETED,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                bucket = %oi.bucket,
-                object = %oi.name,
-                operation = "delete_object",
-                reason = "already_gone",
-                "Lifecycle delete target was already absent"
-            );
-            crate::object_api::notify_object_mutation(&oi.bucket, &oi.name).await;
-            return true;
-        }
         Err(e) => {
             error!(
                 event = EVENT_LIFECYCLE_DELETE_FAILED,
@@ -6081,19 +6041,18 @@ mod tests {
         cleanup_empty_multipart_sha_dirs_on_local_disks, cleanup_stale_multipart_uploads_once_at,
         enqueue_recovered_free_version_with_state, enqueue_transition_for_existing_objects_scoped,
         enqueue_transition_with_lifecycle, enqueue_transition_with_lifecycle_report, eval_action_from_lifecycle,
-        get_lock_acquire_timeout, is_lifecycle_expiry_already_gone, jitter_tier_free_version_recovery_delay,
-        lifecycle_action_blocked_by_replication, lifecycle_delete_all_versions_replication_scan, lifecycle_deleted_object,
-        lifecycle_replication_blocks_action, lifecycle_rule_has_date_expiration, lifecycle_version_delete_target,
-        manual_transition_duration_elapsed, manual_transition_has_more_after_limit, manual_transition_recovery_progress_sink,
-        manual_transition_version_marker, manual_transition_worker_failure_reason,
-        mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate,
-        persist_manual_transition_job_progress_if_owned, persist_manual_transition_page_checkpoint,
-        recover_manual_transition_job, recover_manual_transition_jobs, resolve_tier_free_version_recovery_enabled,
-        resolve_transition_queue_capacity, resolve_transition_queue_send_timeout, resolve_transition_worker_count,
-        resolve_transition_workers_absolute_max, run_tier_free_version_recovery_coordinator, run_tier_free_version_recovery_loop,
-        select_restore_s3_location, set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer,
-        should_defer_date_expiry_for_recent_config_update, transitioned_cleanup_tuple, transitioned_object_delete_opts,
-        wait_for_tier_free_version_recovery,
+        get_lock_acquire_timeout, jitter_tier_free_version_recovery_delay, lifecycle_action_blocked_by_replication,
+        lifecycle_delete_all_versions_replication_scan, lifecycle_deleted_object, lifecycle_replication_blocks_action,
+        lifecycle_rule_has_date_expiration, lifecycle_version_delete_target, manual_transition_duration_elapsed,
+        manual_transition_has_more_after_limit, manual_transition_recovery_progress_sink, manual_transition_version_marker,
+        manual_transition_worker_failure_reason, mark_delete_opts_skip_decommissioned_on_remote_success,
+        merge_stale_multipart_candidate, persist_manual_transition_job_progress_if_owned,
+        persist_manual_transition_page_checkpoint, recover_manual_transition_job, recover_manual_transition_jobs,
+        resolve_tier_free_version_recovery_enabled, resolve_transition_queue_capacity, resolve_transition_queue_send_timeout,
+        resolve_transition_worker_count, resolve_transition_workers_absolute_max, run_tier_free_version_recovery_coordinator,
+        run_tier_free_version_recovery_loop, select_restore_s3_location, set_lifecycle_observability_observer,
+        set_recovered_free_version_enqueue_observer, should_defer_date_expiry_for_recent_config_update,
+        transitioned_cleanup_tuple, transitioned_object_delete_opts, wait_for_tier_free_version_recovery,
     };
     #[cfg(feature = "test-util")]
     use super::{delete_free_version_remote_object_then, encode_dir_object, get_transitioned_object_reader_with_tier_manager};
@@ -10552,26 +10511,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn lifecycle_expiry_already_gone_only_accepts_object_or_version_absence() {
-        assert!(is_lifecycle_expiry_already_gone(&Error::FileNotFound));
-        assert!(is_lifecycle_expiry_already_gone(&Error::ObjectNotFound(
-            "bucket".to_string(),
-            "object".to_string(),
-        )));
-        assert!(is_lifecycle_expiry_already_gone(&Error::FileVersionNotFound));
-        assert!(is_lifecycle_expiry_already_gone(&Error::VersionNotFound(
-            "bucket".to_string(),
-            "object".to_string(),
-            "version".to_string(),
-        )));
-
-        assert!(!is_lifecycle_expiry_already_gone(&Error::VolumeNotFound));
-        assert!(!is_lifecycle_expiry_already_gone(&Error::DiskNotFound));
-        assert!(!is_lifecycle_expiry_already_gone(&Error::ErasureReadQuorum));
-        assert!(!is_lifecycle_expiry_already_gone(&Error::MethodNotAllowed));
-    }
-
     #[tokio::test]
     async fn manual_transition_counts_already_transitioned_object() {
         let lc = latest_transition_lifecycle();
@@ -13446,9 +13385,7 @@ mod tests {
                 .bucket_incarnation_id_from_disk(&bucket)
                 .await
                 .expect("bucket incarnation");
-            // Replaying the same scanner snapshot after the delete is an
-            // already-gone success and must not be reported as a failure.
-            for (attempt_incarnation, expected) in [(Uuid::new_v4(), false), (incarnation, true), (incarnation, true)] {
+            for (attempt_incarnation, expected) in [(Uuid::new_v4(), false), (incarnation, true), (incarnation, false)] {
                 assert_eq!(
                     super::apply_expiry_on_non_transitioned_objects(
                         ecstore.clone(),
@@ -14277,7 +14214,11 @@ mod tests {
     #[tokio::test]
     #[serial]
     async fn tier_free_version_recovery_production_entrypoint_enqueues_seeded_item() {
-        let (disk_paths, ecstore) = setup_test_env().await;
+        let (disk_paths, ecstore) =
+            temp_env::async_with_vars([(super::ENV_TIER_FREE_VERSION_RECOVERY_ENABLED, Some("false"))], async {
+                setup_test_env().await
+            })
+            .await;
         let runtime_state = install_unconsumed_runtime_expiry_worker(&ecstore, 1).await;
         let recovery_rx = {
             let state = runtime_state.read().await;

crates/ecstore/src/bucket/metadata.rs

@@ -439,6 +439,8 @@ pub struct BucketMetadata {
     pub bucket_incarnation_id: Uuid,
     pub(crate) bucket_incarnation_sidecar: bool,
     pub(crate) bucket_creation_committed: bool,
+    // A legacy default is not proof of an interrupted creation.
+    pub(crate) bucket_creation_commit_record_present: bool,
     pub policy_config_json: Vec<u8>,
     pub notification_config_xml: Vec<u8>,
     pub lifecycle_config_xml: Vec<u8>,
@@ -514,6 +516,7 @@ impl Default for BucketMetadata {
             bucket_incarnation_id: Uuid::nil(),
             bucket_incarnation_sidecar: false,
             bucket_creation_committed: false,
+            bucket_creation_commit_record_present: false,
             policy_config_json: Default::default(),
             notification_config_xml: Default::default(),
             lifecycle_config_xml: Default::default(),
@@ -752,7 +755,10 @@ impl BucketMetadata {
                 "TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?,
                 "DurabilityConfigUpdatedAt" => self.durability_config_updated_at = read_msgp_time_value(rd)?,
                 "OnDemandMigrationConfigUpdatedAt" => self.on_demand_migration_config_updated_at = read_msgp_time_value(rd)?,
-                "BucketCreationCommitted" => self.bucket_creation_committed = read_msgp_bool(rd)?,
+                "BucketCreationCommitted" => {
+                    self.bucket_creation_committed = read_msgp_bool(rd)?;
+                    self.bucket_creation_commit_record_present = true;
+                }
                 other => {
                     tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field");
                     skip_msgp_value(rd)?;
@@ -765,8 +771,10 @@ impl BucketMetadata {
 
     /// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
     pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
-        // Map size: MinIO fields (25) + RustFS extensions (22)
-        let map_len: u32 = 47;
+        // Preserve the absence of a legacy commit record across config rewrites.
+        let commit_record_present = self.bucket_creation_commit_record_present || self.bucket_creation_committed;
+        // Map size: MinIO fields (25) + RustFS extensions (21) + optional commit record.
+        let map_len: u32 = 46 + u32::from(commit_record_present);
         rmp::encode::write_map_len(wr, map_len)?;
 
         // MinIO field order (same as Go struct)
@@ -847,8 +855,10 @@ impl BucketMetadata {
         write_msgp_time(wr, self.durability_config_updated_at)?;
         rmp::encode::write_str(wr, "OnDemandMigrationConfigUpdatedAt")?;
         write_msgp_time(wr, self.on_demand_migration_config_updated_at)?;
-        rmp::encode::write_str(wr, "BucketCreationCommitted")?;
-        rmp::encode::write_bool(wr, self.bucket_creation_committed)?;
+        if commit_record_present {
+            rmp::encode::write_str(wr, "BucketCreationCommitted")?;
+            rmp::encode::write_bool(wr, self.bucket_creation_committed)?;
+        }
 
         Ok(())
     }
@@ -1691,6 +1701,13 @@ mod test {
         let mut bm = BucketMetadata::unmarshal(&body[4..]).expect("unmarshal recovered blob");
         assert_eq!(bm.name, "interop");
         assert!(!bm.bucket_creation_committed, "legacy metadata must default to uncommitted");
+        assert!(!bm.bucket_creation_commit_record_present, "legacy metadata has no creation-commit record");
+        let rewritten = BucketMetadata::unmarshal(&bm.marshal_msg().expect("rewrite legacy metadata"))
+            .expect("read rewritten legacy metadata");
+        assert!(
+            !rewritten.bucket_creation_commit_record_present,
+            "rewriting legacy metadata must not fabricate a creation intent"
+        );
         bm.parse_all_configs().expect("parse recovered configs");
         assert!(bm.lifecycle_config.is_some());
     }
@@ -1708,6 +1725,7 @@ mod test {
 
         assert_eq!(bm.name, new.name);
         assert!(new.bucket_creation_committed);
+        assert!(new.bucket_creation_commit_record_present);
         assert!(!bm.bucket_incarnation_id.is_nil());
         assert_eq!(bm.bucket_incarnation_id, new.bucket_incarnation_id);
     }

crates/ecstore/src/bucket/metadata_sys.rs

@@ -178,13 +178,6 @@ pub(crate) fn object_lock_config_state_from_authoritative_metadata(bm: &BucketMe
     Ok(ObjectLockConfigState::ConfirmedAbsent)
 }
 
-#[derive(Debug, Clone, PartialEq)]
-pub(crate) struct CachedQuotaMetadataSnapshot {
-    pub(crate) quota: Option<BucketQuota>,
-    pub(crate) bucket_incarnation: Uuid,
-    pub(crate) quota_revision: OffsetDateTime,
-}
-
 /// Convert the persisted serving-layer configuration into the storage-level
 /// [`DefaultRetention`](crate::bucket::object_lock::types::DefaultRetention)
 /// the WORM evaluation code consumes (rustfs/backlog#1842). A rule without a
@@ -525,36 +518,6 @@ pub(crate) async fn get_bucket_incarnation_id_in(ctx: &crate::runtime::instance:
     sys.get_bucket_incarnation_id_from_disk(bucket).await
 }
 
-/// Directory loss is recoverable only for a committed, non-retired generation.
-/// The caller retains the bucket lifecycle fence through the ensuing reconstruction.
-pub(crate) async fn validate_bucket_reconstruction_in(
-    ctx: &crate::runtime::instance::InstanceContext,
-    bucket: &str,
-    expected: Uuid,
-) -> Result<()> {
-    let api = bucket_metadata_sys_of(ctx)?.read().await.object_store();
-    let lock = api
-        .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
-        .await?;
-    let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
-    let (metadata, persisted) = load_bucket_metadata_parse_with_presence(api.clone(), bucket, true).await?;
-    if !persisted
-        || !metadata.bucket_incarnation_sidecar
-        || expected.is_nil()
-        || metadata.bucket_incarnation_id != expected
-        || !metadata.bucket_creation_committed
-    {
-        return Err(Error::PreconditionFailed);
-    }
-    if crate::bucket::retirement::is_retired(api, bucket, expected).await? {
-        return Err(Error::BucketNotFound(bucket.to_owned()));
-    }
-    if guard.is_lock_lost() {
-        return Err(Error::PreconditionFailed);
-    }
-    Ok(())
-}
-
 /// Validate against disk while retaining an already-held Object Lock fence.
 pub(crate) async fn get_bucket_incarnation_id_for_options_in(
     ctx: &crate::runtime::instance::InstanceContext,
@@ -1487,9 +1450,6 @@ fn quota_config_and_incarnation_from_authority(
         BucketMetadataAuthority::Authoritative(metadata)
             if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
         {
-            if !metadata.quota_config_json.is_empty() && metadata.quota_config.is_none() {
-                return Err(Error::other("persisted bucket quota configuration is invalid"));
-            }
             Ok((
                 metadata.quota_config.clone(),
                 metadata.bucket_incarnation_id,
@@ -1504,15 +1464,6 @@ fn quota_config_and_incarnation_from_authority(
     }
 }
 
-pub(crate) async fn get_cached_quota_config_and_incarnation_in(
-    ctx: &crate::runtime::instance::InstanceContext,
-    bucket: &str,
-) -> Result<Option<CachedQuotaMetadataSnapshot>> {
-    let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
-    let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
-    bucket_meta_sys.cached_quota_config_and_incarnation(bucket).await
-}
-
 pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -1649,9 +1600,27 @@ struct MetadataPublishGuard {
     _guard: tokio::sync::OwnedMutexGuard<MetadataPublishLockState>,
 }
 
+#[derive(Debug, Clone)]
+pub(crate) struct PendingBucketCreationProof {
+    pub(crate) generation: Uuid,
+    pub(crate) intent: Vec<u8>,
+    pub(crate) versioning_requested: bool,
+    pub(crate) witnesses: Vec<crate::disk::os::NativeBucketCreationWitness>,
+}
+
+impl PendingBucketCreationProof {
+    pub(crate) fn matches_metadata(&self, metadata: &BucketMetadata) -> Result<bool> {
+        Ok(!metadata.bucket_creation_committed
+            && metadata.bucket_incarnation_sidecar
+            && metadata.bucket_incarnation_id == self.generation
+            && metadata.marshal_msg()? == self.intent)
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct BucketMetadataSys {
     metadata_map: Arc<RwLock<HashMap<String, Arc<BucketMetadata>>>>,
+    pending_bucket_creations: Arc<RwLock<HashMap<String, PendingBucketCreationProof>>>,
     /// Serializes metadata-map commits and their derived cache updates for one
     /// bucket. Namespace locks, when present, are acquired before this lock.
     metadata_publish_locks: Arc<MetadataPublishLockRegistry>,
@@ -1695,6 +1664,7 @@ impl BucketMetadataSys {
     pub fn new(api: Arc<ECStore>) -> Self {
         Self {
             metadata_map: Arc::new(RwLock::new(HashMap::new())),
+            pending_bucket_creations: Arc::new(RwLock::new(HashMap::new())),
             metadata_publish_locks: Arc::new(MetadataPublishLockRegistry {
                 locks: StdMutex::new(HashMap::new()),
             }),
@@ -1725,6 +1695,81 @@ impl BucketMetadataSys {
             .expect("bucket metadata object store should still be live")
     }
 
+    pub(crate) async fn pending_bucket_creation(&self, bucket: &str) -> Option<PendingBucketCreationProof> {
+        self.pending_bucket_creations.read().await.get(bucket).cloned()
+    }
+
+    pub(crate) async fn grant_pending_bucket_creation(&self, bucket: &str, proof: PendingBucketCreationProof) {
+        self.pending_bucket_creations.write().await.insert(bucket.to_owned(), proof);
+    }
+
+    pub(crate) async fn revoke_pending_bucket_creation(&self, bucket: &str) {
+        self.pending_bucket_creations.write().await.remove(bucket);
+    }
+
+    pub(crate) async fn consume_pending_bucket_creation(
+        &self,
+        bucket: &str,
+        metadata: &BucketMetadata,
+        witnesses: &[crate::disk::os::NativeBucketCreationWitness],
+    ) -> Result<bool> {
+        let mut pending = self.pending_bucket_creations.write().await;
+        let Some(proof) = pending.get(bucket) else {
+            return Ok(false);
+        };
+        if !proof.matches_metadata(metadata)?
+            || proof.witnesses.len() != witnesses.len()
+            || !proof
+                .witnesses
+                .iter()
+                .zip(witnesses)
+                .all(|(old, current)| old.matches(current))
+            || !crate::disk::os::consume_native_creation_witnesses(witnesses)
+        {
+            return Ok(false);
+        }
+        pending.remove(bucket);
+        Ok(true)
+    }
+
+    #[cfg(test)]
+    pub(crate) async fn refresh_bucket_for_test(&self, bucket: &str) -> Result<()> {
+        let expected = self.metadata_map.read().await.get(bucket).cloned();
+        let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
+        let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
+        self.load_bucket_under_namespace(bucket, MetadataLoadMode::Refresh, expected.as_ref(), &guard)
+            .await
+    }
+
+    #[cfg(test)]
+    pub(crate) async fn publish_persisted_metadata_for_test(&self, bucket: &str) -> Result<()> {
+        let expected = self.metadata_map.read().await.get(bucket).cloned();
+        let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
+        let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
+        let (metadata, persisted) = await_bucket_namespace_operation(
+            Some(&guard),
+            bucket,
+            "test cached metadata load",
+            load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
+        )
+        .await?;
+        self.publish_if_unchanged(bucket, expected.as_ref(), metadata, persisted, &guard)
+            .await
+    }
+
+    #[cfg(test)]
+    pub(crate) async fn migrate_bucket_creation_commit_for_test(&self, bucket: &str) -> Result<()> {
+        self.migrate_bucket_creation_commit(bucket).await.map(|_| ())
+    }
+
+    #[cfg(test)]
+    pub(crate) async fn legacy_migration_reports_missing_bucket_for_test(&self, bucket: &str) -> Result<bool> {
+        Ok(matches!(
+            self.migrate_legacy_metadata(bucket).await?,
+            BucketMetadataAuthority::MissingBucket
+        ))
+    }
+
     fn object_store_if_live(&self) -> Option<Arc<ECStore>> {
         self.api.upgrade()
     }
@@ -1915,6 +1960,7 @@ impl BucketMetadataSys {
         expected: Option<&Arc<BucketMetadata>>,
         namespace_guard: &rustfs_lock::NamespaceLockGuard,
     ) -> Result<()> {
+        self.revoke_pending_bucket_creation(bucket).await;
         if !await_bucket_namespace_operation(
             Some(namespace_guard),
             bucket,
@@ -1998,6 +2044,7 @@ impl BucketMetadataSys {
         persisted: bool,
         namespace_guard: &rustfs_lock::NamespaceLockGuard,
     ) -> Result<()> {
+        self.revoke_pending_bucket_creation(bucket).await;
         if !persisted {
             let _publish_guard = self
                 .lock_metadata_publish(bucket, namespace_guard, "refreshed bucket metadata absence publish")
@@ -2059,6 +2106,7 @@ impl BucketMetadataSys {
     }
 
     pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
+        self.revoke_pending_bucket_creation(&bucket).await;
         if !is_meta_bucketname(&bucket) {
             let publish_lock = self.metadata_publish_lock(&bucket);
             let _publish_guard = publish_lock.lock().await;
@@ -2080,6 +2128,7 @@ impl BucketMetadataSys {
         if is_meta_bucketname(bucket) {
             return false;
         }
+        self.revoke_pending_bucket_creation(bucket).await;
         let publish_lock = self.metadata_publish_lock(bucket);
         let _publish_guard = publish_lock.lock().await;
         let mut map = self.metadata_map.write().await;
@@ -2097,6 +2146,7 @@ impl BucketMetadataSys {
     }
 
     async fn _reset(&mut self) {
+        self.pending_bucket_creations.write().await.clear();
         let mut map = self.metadata_map.write().await;
         map.clear();
         drop(map);
@@ -2235,6 +2285,7 @@ impl BucketMetadataSys {
     /// server's metadata never leaks into the ambient (first) instance.
     pub(crate) async fn persist_and_set(&self, bm: BucketMetadata) -> Result<()> {
         let mut bm = bm;
+        self.revoke_pending_bucket_creation(&bm.name).await;
         bm.save_with_store(self.object_store()).await?;
 
         self.set(bm.name.clone(), Arc::new(bm)).await;
@@ -2243,6 +2294,7 @@ impl BucketMetadataSys {
     }
 
     async fn persist_new_and_set(&self, mut bm: BucketMetadata) -> Result<()> {
+        self.revoke_pending_bucket_creation(&bm.name).await;
         bm.bucket_creation_committed = true;
         bm.save_with_store_committed(self.object_store()).await?;
         save_bucket_incarnation(self.object_store(), &bm.name, bm.bucket_incarnation_id).await?;
@@ -2252,7 +2304,9 @@ impl BucketMetadataSys {
     }
 
     async fn persist_new_bucket_metadata_intent(&self, mut bm: BucketMetadata) -> Result<()> {
+        self.revoke_pending_bucket_creation(&bm.name).await;
         bm.bucket_creation_committed = false;
+        bm.bucket_creation_commit_record_present = true;
         bm.save_with_store(self.object_store()).await?;
         save_bucket_incarnation(self.object_store(), &bm.name, bm.bucket_incarnation_id).await?;
         bm.bucket_incarnation_sidecar = true;
@@ -2262,6 +2316,7 @@ impl BucketMetadataSys {
     }
 
     async fn commit_bucket_metadata(&self, mut bm: BucketMetadata) -> Result<()> {
+        self.revoke_pending_bucket_creation(&bm.name).await;
         bm.parse_all_configs()?;
         bm.bucket_incarnation_sidecar = true;
         // The caller holds the full bucket creation fence (lifecycle, metadata
@@ -2310,6 +2365,7 @@ impl BucketMetadataSys {
         bucket: &str,
         namespace_guard: &rustfs_lock::NamespaceLockGuard,
     ) -> Result<()> {
+        self.revoke_pending_bucket_creation(bucket).await;
         let expected = self.metadata_map.read().await.get(bucket).cloned();
         if !self
             .bucket_exists(bucket, namespace_guard, "peer bucket metadata existence check")
@@ -2366,6 +2422,7 @@ impl BucketMetadataSys {
             #[cfg(test)]
             self.lazy_disk_loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
 
+            self.revoke_pending_bucket_creation(bucket).await;
             let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
             let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
             #[cfg(test)]
@@ -2674,6 +2731,7 @@ impl BucketMetadataSys {
     /// revalidated at write quorum, so a stale snapshot can neither revert an
     /// acknowledged configuration update nor outlive a delete/recreate.
     async fn migrate_bucket_creation_commit(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
+        self.revoke_pending_bucket_creation(bucket).await;
         let transaction_lock = self
             .object_store()
             .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
@@ -2751,6 +2809,7 @@ impl BucketMetadataSys {
         &self,
         bucket: &str,
     ) -> Result<BucketMetadataAuthority> {
+        self.revoke_pending_bucket_creation(bucket).await;
         #[cfg(test)]
         if self.object_lock_disk_read_errors.write().await.remove(bucket) {
             return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
@@ -3028,27 +3087,6 @@ impl BucketMetadataSys {
         }
     }
 
-    async fn cached_quota_config_and_incarnation(&self, bucket: &str) -> Result<Option<CachedQuotaMetadataSnapshot>> {
-        let metadata = {
-            let metadata_map = self.metadata_map.read().await;
-            metadata_map.get(bucket).cloned()
-        };
-        let Some(metadata) = metadata else {
-            return Ok(None);
-        };
-        if !metadata.bucket_incarnation_sidecar || metadata.bucket_incarnation_id.is_nil() {
-            return Ok(None);
-        }
-        if !metadata.quota_config_json.is_empty() && metadata.quota_config.is_none() {
-            return Err(Error::other("persisted bucket quota configuration is invalid"));
-        }
-        Ok(Some(CachedQuotaMetadataSnapshot {
-            quota: metadata.quota_config.clone(),
-            bucket_incarnation: metadata.bucket_incarnation_id,
-            quota_revision: metadata.quota_config_updated_at,
-        }))
-    }
-
     pub async fn get_replication_config(&self, bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 
@@ -3143,6 +3181,7 @@ mod tests {
     };
     use crate::bucket::target::{BucketTarget, BucketTargetType, Credentials};
     use crate::config::com::read_config;
+    use crate::disk::{DiskAPI as _, RUSTFS_META_TMP_BUCKET};
     use crate::storage_api_contracts::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
     use byteorder::{ByteOrder as _, LittleEndian};
     use serial_test::serial;
@@ -3197,6 +3236,180 @@ mod tests {
         ),
     ];
 
+    #[tokio::test]
+    #[serial(storage_class_env)]
+    async fn persistence_entry_failures_revoke_pending_creation_before_cache_publish() {
+        let (dirs, store) = isolated_store_over_temp_disks().await;
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let sys = bucket_metadata_sys_of(&store.ctx).expect("instance metadata system");
+        let sys = sys.read().await.clone();
+        let options = MakeBucketOptions {
+            lock_enabled: true,
+            ..Default::default()
+        };
+        for entry in ["save", "persist", "persist-new", "commit"] {
+            let bucket = format!("revoke-{entry}-{}", Uuid::new_v4());
+            store.fail_next_bucket_creation_after_intent_for_test(&bucket);
+            store
+                .make_bucket(&bucket, &options)
+                .await
+                .expect_err("stop the real producer before physical publication");
+            sys.pending_bucket_creation(&bucket)
+                .await
+                .expect("the producer must leave a real live creation permission");
+            let (intent, persisted) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
+                .await
+                .expect("read the original producer intent");
+            assert!(persisted, "{entry}: the complete original intent must be persisted");
+            let original_intent = intent.marshal_msg().expect("encode the original intent");
+
+            // Keep the creation fence order used by the real producer.
+            let lifecycle_guard = store
+                .acquire_bucket_lifecycle_write_lock(&bucket)
+                .await
+                .expect("hold the bucket lifecycle fence");
+            let transaction_guard = acquire_bucket_metadata_transaction_lock_in(&store.ctx, &bucket)
+                .await
+                .expect("hold the metadata transaction fence");
+            let namespace_lock = store
+                .new_ns_lock(&bucket, &bucket)
+                .await
+                .expect("create the exact bucket namespace lock");
+            let namespace_guard = namespace_lock
+                .get_write_lock(crate::set_disk::get_lock_acquire_timeout())
+                .await
+                .expect("hold the exact bucket namespace fence");
+
+            // A file at each native staging root causes actual IO to fail before
+            // metadata can reach the cache-publishing set call. Preserve the
+            // original staging directories so no fixture data is discarded.
+            for dir in &dirs {
+                let staging = dir.path().join(RUSTFS_META_TMP_BUCKET);
+                let saved = dir.path().join(".rustfs.sys/revocation-saved-tmp");
+                std::fs::rename(&staging, &saved).expect("preserve the native staging directory");
+                std::fs::write(&staging, b"block metadata staging with a non-directory")
+                    .expect("install an actual non-directory IO obstruction");
+            }
+            let result = match entry {
+                "save" => sys.save(intent).await,
+                "persist" => sys.persist_and_set(intent).await,
+                "persist-new" => sys.persist_new_and_set(intent).await,
+                "commit" => sys.commit_bucket_metadata(intent).await,
+                _ => unreachable!(),
+            };
+            for dir in &dirs {
+                let staging = dir.path().join(RUSTFS_META_TMP_BUCKET);
+                let saved = dir.path().join(".rustfs.sys/revocation-saved-tmp");
+                std::fs::remove_file(&staging).expect("remove only the fixture's staging obstruction");
+                std::fs::rename(&saved, &staging).expect("restore the complete original staging directory");
+            }
+            drop((namespace_guard, transaction_guard, lifecycle_guard));
+            result.expect_err("native persistence must fail before its downstream cache set");
+            assert!(
+                sys.pending_bucket_creation(&bucket).await.is_none(),
+                "{entry}: the entry revoker must retire live permission even when persistence fails"
+            );
+            let (after_failure, present) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
+                .await
+                .expect("read the intent after the failed persistence entry");
+            assert!(present, "{entry}: a failed persistence entry must retain the original metadata");
+            assert_eq!(
+                after_failure
+                    .marshal_msg()
+                    .expect("encode the original intent after persistence failure"),
+                original_intent,
+                "{entry}: native IO failure must not change the original generation or complete intent"
+            );
+            let error = store
+                .make_bucket(&bucket, &options)
+                .await
+                .expect_err("a failed ordinary persistence entry cannot preserve creation permission");
+            assert!(error.to_string().contains("administrator recovery"), "{entry}: {error}");
+            let (after_retry, _) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
+                .await
+                .expect("read the original intent after the rejected retry");
+            assert_eq!(
+                after_retry.marshal_msg().expect("encode the intent after the rejected retry"),
+                original_intent,
+                "{entry}: the rejected retry must preserve the complete original intent"
+            );
+            for disk in store.pools[0].disk_set[0].disks.read().await.iter() {
+                assert!(
+                    matches!(
+                        disk.as_ref().expect("native disk").stat_volume(&bucket).await,
+                        Err(crate::disk::error::DiskError::VolumeNotFound)
+                    ),
+                    "{entry}: failed persistence and the rejected retry must not publish a physical volume"
+                );
+            }
+        }
+    }
+
+    #[tokio::test]
+    #[serial(storage_class_env)]
+    async fn metadata_reset_revokes_real_pending_creation_without_changing_persisted_intent() {
+        let (_dirs, store) = isolated_store_over_temp_disks().await;
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = format!("revoke-reset-{}", Uuid::new_v4());
+        let options = MakeBucketOptions {
+            lock_enabled: true,
+            ..Default::default()
+        };
+        store.fail_next_bucket_creation_after_intent_for_test(&bucket);
+        store
+            .make_bucket(&bucket, &options)
+            .await
+            .expect_err("stop the real producer before physical publication");
+        let sys = bucket_metadata_sys_of(&store.ctx).expect("instance metadata system");
+        let mut sys = sys.read().await.clone();
+        sys.pending_bucket_creation(&bucket)
+            .await
+            .expect("the real producer must leave live creation permission before reset");
+        let (intent, persisted) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
+            .await
+            .expect("read the original intent before cache reset");
+        assert!(persisted, "cache reset starts from a complete real persisted intent");
+        let original_intent = intent.marshal_msg().expect("encode the original reset intent");
+        sys._reset().await;
+        assert!(
+            sys.pending_bucket_creation(&bucket).await.is_none(),
+            "reset must revoke the producer-issued permission shared with the instance metadata system"
+        );
+        let (after_reset, present) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
+            .await
+            .expect("read the original intent after cache reset");
+        assert!(present, "cache reset must retain the persisted intent");
+        assert_eq!(
+            after_reset.marshal_msg().expect("encode the intent after cache reset"),
+            original_intent,
+            "cache reset must leave the original generation and complete persisted intent unchanged"
+        );
+        let error = store
+            .make_bucket(&bucket, &options)
+            .await
+            .expect_err("reset cannot regrant the producer's old creation permission");
+        assert!(error.to_string().contains("administrator recovery"), "{error}");
+        let (after_retry, _) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
+            .await
+            .expect("read the intent after the reset-induced retry rejection");
+        assert_eq!(
+            after_retry
+                .marshal_msg()
+                .expect("encode the intent after the rejected reset retry"),
+            original_intent,
+            "the reset-induced rejection must preserve the original complete intent"
+        );
+        for disk in store.pools[0].disk_set[0].disks.read().await.iter() {
+            assert!(
+                matches!(
+                    disk.as_ref().expect("native disk").stat_volume(&bucket).await,
+                    Err(crate::disk::error::DiskError::VolumeNotFound)
+                ),
+                "cache reset and the rejected retry must not publish a physical volume"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn g_d3_003_new_writer_replication_loads_without_fail_closed_state() {
         let (dirs, store) = isolated_store_over_temp_disks().await;
@@ -4953,43 +5166,6 @@ mod tests {
         );
     }
 
-    #[tokio::test]
-    #[serial]
-    async fn quota_admission_rejects_stale_cached_no_quota() {
-        let (_dirs, store) = isolated_store_over_temp_disks().await;
-        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
-        let sys = bucket_metadata_sys_of(&store.ctx).expect("scoped metadata system");
-        let bucket = "stale-no-quota-admission";
-        store
-            .make_bucket(bucket, &MakeBucketOptions::default())
-            .await
-            .expect("create bucket");
-        let (stale, _) = sys.read().await.get_config(bucket).await.expect("cache initial metadata");
-        store
-            .update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec())
-            .await
-            .expect("publish quota to authoritative storage");
-        // Model a peer that has not received the newer configuration snapshot.
-        sys.read().await.set(bucket.to_owned(), stale).await;
-        assert!(
-            get_cached_quota_config_and_incarnation_in(&store.ctx, bucket)
-                .await
-                .expect("read stale cache")
-                .expect("cached bucket identity")
-                .quota
-                .is_none()
-        );
-
-        temp_env::async_with_vars([("RUSTFS_QUOTA_BEGIN_SAFE_NO_QUOTA_CACHE_FAST_PATH", Some("true"))], async {
-            let error = crate::bucket::quota::reservation::begin(&store.ctx, bucket, "object", &ObjectOptions::default(), 0, 0)
-                .await
-                .err()
-                .expect("persisted quota must reject admission without its snapshot");
-            assert!(matches!(error, Error::PartMissingOrCorrupt), "{error:?}");
-        })
-        .await;
-    }
-
     #[tokio::test]
     #[serial]
     async fn peer_reload_clears_negative_cache_after_remote_bucket_creation() {

crates/ecstore/src/bucket/quota/reservation.rs

@@ -850,11 +850,6 @@ pub(crate) async fn begin(
     let snapshot_admission = opts.quota_admission;
     let data_movement = opts.data_movement;
     if crate::bucket::utils::is_meta_bucketname(bucket) {
-        let meta_bucket_fast_path_started = rustfs_io_metrics::put_stage_timer();
-        rustfs_io_metrics::record_put_object_stage_duration_from(
-            rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH,
-            meta_bucket_fast_path_started,
-        );
         return Ok(QuotaContext {
             store: None,
             bucket: bucket.to_string(),
@@ -911,63 +906,9 @@ pub(crate) async fn begin(
         });
     }
 
-    let metadata_lock_started = rustfs_io_metrics::put_stage_timer();
     let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
-    rustfs_io_metrics::record_put_object_stage_duration_from(
-        rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK,
-        metadata_lock_started,
-    );
-    let cache_lookup_started = rustfs_io_metrics::put_stage_timer();
-    let cached_quota_snapshot = if rustfs_io_metrics::put_stage_metrics_enabled() {
-        match metadata_sys::get_cached_quota_config_and_incarnation_in(ctx, bucket).await {
-            Ok(Some(snapshot)) => {
-                rustfs_io_metrics::record_put_object_quota_cache_decision(
-                    "candidate",
-                    quota_cache_reason(snapshot.quota.as_ref()),
-                );
-                Some(snapshot)
-            }
-            Ok(None) => {
-                rustfs_io_metrics::record_put_object_quota_cache_decision("miss", "no_authoritative_cache");
-                None
-            }
-            Err(_) => {
-                rustfs_io_metrics::record_put_object_quota_cache_decision("miss", "invalid_cached_quota");
-                None
-            }
-        }
-    } else {
-        None
-    };
-    rustfs_io_metrics::record_put_object_stage_duration_from(
-        rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CACHE_LOOKUP,
-        cache_lookup_started,
-    );
-    let config_read_started = rustfs_io_metrics::put_stage_timer();
     let (quota, bucket_incarnation, quota_revision) =
         metadata_sys::get_quota_config_and_incarnation_from_disk_for_options_in(ctx, bucket, opts, &metadata_guard).await?;
-    if let Some(cached) = cached_quota_snapshot.as_ref() {
-        let reason = quota_cache_reason(quota.as_ref());
-        if cached.quota == quota && cached.bucket_incarnation == bucket_incarnation && cached.quota_revision == quota_revision {
-            rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_match", reason);
-        } else {
-            rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch", reason);
-            if cached.quota != quota {
-                rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "quota");
-            }
-            if cached.bucket_incarnation != bucket_incarnation {
-                rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "bucket_incarnation");
-            }
-            if cached.quota_revision != quota_revision {
-                rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "quota_revision");
-            }
-        }
-    }
-    rustfs_io_metrics::record_put_object_stage_duration_from(
-        rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ,
-        config_read_started,
-    );
-    let lock_lost_check_started = rustfs_io_metrics::put_stage_timer();
     if metadata_guard.is_lock_lost() {
         return Err(StorageError::NamespaceLockQuorumUnavailable {
             mode: "quota_config",
@@ -977,11 +918,6 @@ pub(crate) async fn begin(
             achieved: 0,
         });
     }
-    rustfs_io_metrics::record_put_object_stage_duration_from(
-        rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_LOCK_LOST_CHECK,
-        lock_lost_check_started,
-    );
-    let policy_evaluate_started = rustfs_io_metrics::put_stage_timer();
     if quota
         .as_ref()
         .is_some_and(|quota| quota.has_unsupported_reservation_protocol())
@@ -990,11 +926,6 @@ pub(crate) async fn begin(
         return Err(StorageError::PartMissingOrCorrupt);
     }
     let durable_quota = quota.as_ref().filter(|quota| quota.uses_durable_reservations());
-    rustfs_io_metrics::record_put_object_stage_duration_from(
-        rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_POLICY_EVALUATE,
-        policy_evaluate_started,
-    );
-    let capability_proof_started = rustfs_io_metrics::put_stage_timer();
     let capability_proof = if durable_quota.is_some() {
         Some(
             crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof()
@@ -1003,12 +934,7 @@ pub(crate) async fn begin(
     } else {
         None
     };
-    rustfs_io_metrics::record_put_object_stage_duration_from(
-        rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CAPABILITY_PROOF,
-        capability_proof_started,
-    );
     let durable_quota_limit = durable_quota.and_then(|quota| quota.quota);
-    let snapshot_admission_started = rustfs_io_metrics::put_stage_timer();
     let reservation_protocol = durable_quota.and_then(|quota| quota.reservation_protocol);
     let snapshot_admission = match quota.as_ref().filter(|quota| !quota.uses_durable_reservations()) {
         Some(quota) => match (quota.quota, snapshot_admission) {
@@ -1028,10 +954,6 @@ pub(crate) async fn begin(
         },
         None => None,
     };
-    rustfs_io_metrics::record_put_object_stage_duration_from(
-        rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SNAPSHOT_ADMISSION,
-        snapshot_admission_started,
-    );
     let legacy_data_movement = durable_quota_limit.is_none()
         && quota.as_ref().and_then(|quota| quota.quota).is_some()
         && snapshot_admission.is_none()
@@ -1043,21 +965,11 @@ pub(crate) async fn begin(
                 .then(|| quota.as_ref().and_then(|quota| quota.quota))
                 .flatten()
         });
-    let object_store_lookup_started = rustfs_io_metrics::put_stage_timer();
     let store = if durable_quota_limit.is_some() {
         Some(metadata_sys::object_store_in(ctx).await?)
     } else {
         None
     };
-    rustfs_io_metrics::record_put_object_stage_duration_from(
-        rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_OBJECT_STORE_LOOKUP,
-        object_store_lookup_started,
-    );
-    let context_build_started = rustfs_io_metrics::put_stage_timer();
-    rustfs_io_metrics::record_put_object_stage_duration_from(
-        rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONTEXT_BUILD,
-        context_build_started,
-    );
     Ok(QuotaContext {
         store,
         bucket: bucket.to_string(),
@@ -1076,15 +988,6 @@ pub(crate) async fn begin(
     })
 }
 
-fn quota_cache_reason(quota: Option<&crate::bucket::quota::BucketQuota>) -> &'static str {
-    match quota {
-        None => "no_quota",
-        Some(quota) if quota.uses_durable_reservations() => "durable_quota",
-        Some(quota) if quota.quota.is_some() => "snapshot_quota",
-        Some(_) => "quota_without_limit",
-    }
-}
-
 fn quota_capability_error(bucket: &str, object: &str) -> StorageError {
     StorageError::NamespaceLockQuorumUnavailable {
         mode: "quota_capability",

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -65,9 +65,9 @@ use super::replication_target_boundary::{
     SsecPassthroughGate, TargetClient, VersionIdentityCapability, is_replication_target_offline_error,
     replication_action_for_target_head, replication_complete_multipart_options, replication_delete_marker_purge_remove_options,
     replication_delete_remove_options, replication_force_delete_remove_options, replication_object_is_ssec_encrypted,
-    replication_put_object_header_size, replication_put_object_options, replication_source_etag_is_comparable,
-    replication_target_head_is_newer_null_version, resolve_read_api_version_id, ssec_passthrough_evidence_present,
-    ssec_passthrough_gate, version_identity_capability_from_put, version_identity_drifted,
+    replication_put_object_header_size, replication_put_object_options, replication_target_head_is_newer_null_version,
+    resolve_read_api_version_id, ssec_passthrough_evidence_present, ssec_passthrough_gate, version_identity_capability_from_put,
+    version_identity_drifted,
 };
 use super::replication_versioning_boundary::ReplicationVersioningStore;
 use super::runtime_boundary as runtime_sources;
@@ -349,22 +349,15 @@ fn replica_etag_verification_enabled() -> bool {
 /// target that stores a transformed payload (e.g. undecoded `aws-chunked`
 /// frames, #6853) returns the ETag of what it actually wrote. Reporting
 /// COMPLETED over such a replica is silent corruption, so a decidable
-/// mismatch fails the replication instead. Encrypted sources are exempt: an
-/// SSE-C ciphertext passthrough sends ciphertext while the source ETag is the
-/// plaintext MD5 (that path has its own HEAD-back audit), and a managed-SSE
-/// (SSE-S3 / SSE-KMS) source stores a keyed fingerprint, or a legacy plaintext
-/// MD5, that the ETag of the target's re-encrypted replica can neither confirm
-/// nor refute. The checksum header on the replication PUT stays the integrity
-/// check for those objects.
+/// mismatch fails the replication instead. An SSE-C ciphertext passthrough
+/// transfer is exempt: the wire bytes are ciphertext while the source ETag is
+/// the plaintext MD5, and that path has its own HEAD-back audit.
 fn verify_single_part_replica(
     object_info: &ObjectInfo,
     response: &RemotePutObjectResponse,
     ciphertext_passthrough: bool,
 ) -> std::result::Result<(), std::io::Error> {
-    if ciphertext_passthrough
-        || !replication_source_etag_is_comparable(&object_info.user_defined)
-        || !replica_etag_verification_enabled()
-    {
+    if ciphertext_passthrough || !replica_etag_verification_enabled() {
         return Ok(());
     }
     if single_part_replica_etag_mismatch(object_info.etag.as_deref(), response.etag.as_deref()) {
@@ -455,26 +448,6 @@ fn is_version_id_format_mismatch(err: &SdkError<HeadObjectError>) -> bool {
     is_version_id_mismatch(code, raw_status)
 }
 
-/// A version-addressed HEAD rejected because the target cannot address the
-/// source version id (AWS S3: bodiless 400). Narrower than
-/// [`is_version_id_format_mismatch`]: a bodiless 403 may be a genuine
-/// `s3:GetObjectVersion` denial, so it is not accepted where a mutation or a
-/// completion verdict follows instead of another non-destructive HEAD.
-fn rejects_source_version_id(err: &SdkError<HeadObjectError>) -> bool {
-    is_version_id_format_mismatch(err) && !has_raw_status(err, 403)
-}
-
-/// Whether the target's latest version of `object` is a delete marker. For
-/// targets that mint their own version ids, which cannot be asked about the
-/// source marker id: S3 answers a versionless HEAD on such a key with 404
-/// (405 on some targets) plus `x-amz-delete-marker: true`.
-async fn latest_target_version_is_delete_marker(tgt_client: &TargetClient, object: &str) -> bool {
-    match head_object_for_worker(tgt_client, &tgt_client.bucket, object, None).await {
-        Ok(head) => head.delete_marker == Some(true),
-        Err(e) => raw_header_value(e.as_ref(), "x-amz-delete-marker").as_deref() == Some("true"),
-    }
-}
-
 async fn mark_replication_target_offline_if_needed(target_client: &Arc<TargetClient>, err: &(impl Display + ?Sized)) {
     if is_replication_target_offline_error(err) {
         ReplicationTargetStore::mark_target_offline(target_client).await;
@@ -1873,17 +1846,7 @@ async fn verify_resync_head_result(
             // only absence proves completion. All other outcomes remain
             // failed (backlog#862 / #799 B13).
             let retryable = if roi.version_purge_status.is_empty() {
-                match classify_delete_marker_head_error(&err) {
-                    // A target that mints its own version ids never holds the
-                    // source marker id; verify the marker on the latest version.
-                    DeleteMarkerHeadOutcome::Ambiguous => {
-                        let mints_own = rejects_source_version_id(&err)
-                            || ReplicationTargetStore::version_identity_capability(&target_client.arn)
-                                .version_addressing_unreliable();
-                        !(mints_own && latest_target_version_is_delete_marker(target_client, &roi.name).await)
-                    }
-                    DeleteMarkerHeadOutcome::Exists | DeleteMarkerHeadOutcome::Missing => false,
-                }
+                matches!(classify_delete_marker_head_error(&err), DeleteMarkerHeadOutcome::Ambiguous)
             } else {
                 !matches!(classify_delete_marker_head_error(&err), DeleteMarkerHeadOutcome::Missing)
             };
@@ -3604,16 +3567,7 @@ async fn replicate_delete_to_target(
         }
     }
 
-    // A target that mints its own version ids never holds the source marker
-    // id, so a version-addressed pre-check cannot succeed there; the marker
-    // is created by the versionless DELETE below.
-    let version_addressable =
-        !ReplicationTargetStore::version_identity_capability(&tgt_client.arn).version_addressing_unreliable();
-    if version_addressable
-        && !is_version_purge
-        && dobj.delete_object.delete_marker
-        && dobj.delete_object.delete_marker_version_id.is_some()
-    {
+    if !is_version_purge && dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
         match head_object_for_worker(
             tgt_client.as_ref(),
             &tgt_client.bucket,
@@ -3622,11 +3576,6 @@ async fn replicate_delete_to_target(
         )
         .await
         {
-            // The verdict is not known yet (e.g. no PUT since restart) and the
-            // target rejected the source id: learn it as the PUT path does.
-            Err(e) if rejects_source_version_id(e.as_ref()) => {
-                ReplicationTargetStore::record_version_identity_capability(&tgt_client.arn, VersionIdentityCapability::MintsOwn);
-            }
             Ok(head) if head.delete_marker == Some(true) => {
                 // Some targets expose an existing marker as a successful HEAD
                 // with the marker bit instead of the usual 405. Treat that as
@@ -5877,57 +5826,6 @@ mod tests {
     use super::super::replication_filemeta_boundary::ReplicateTargetDecision;
     use super::super::replication_object_decision_boundary::ReplicationMultipartPlanError;
 
-    #[test]
-    fn verify_single_part_replica_exempts_managed_sse_sources() {
-        // A new encrypted write stores a keyed fingerprint; a target that
-        // computes its own ETag answers with the MD5 of the plaintext it
-        // received. Both are 32 hex characters, so without the exemption the
-        // audit would fail every SSE-S3 / SSE-KMS replica (rustfs/backlog).
-        let fingerprint = "00112233445566778899aabbccddeeff";
-        let replica_md5 = "ffeeddccbbaa99887766554433221100";
-        let response = RemotePutObjectResponse {
-            version_id: None,
-            etag: Some(replica_md5.to_string()),
-        };
-        for sse in ["AES256", "aws:kms"] {
-            let object_info = ObjectInfo {
-                bucket: "source".to_string(),
-                name: "encrypted".to_string(),
-                etag: Some(fingerprint.to_string()),
-                user_defined: HashMap::from([("x-amz-server-side-encryption".to_string(), sse.to_string())]).into(),
-                ..Default::default()
-            };
-            verify_single_part_replica(&object_info, &response, false)
-                .unwrap_or_else(|err| panic!("{sse}: a managed-SSE source must not be audited against the replica MD5: {err}"));
-        }
-    }
-
-    #[test]
-    fn verify_single_part_replica_still_fails_plaintext_mismatch() {
-        if !replica_etag_verification_enabled() {
-            return;
-        }
-        let object_info = ObjectInfo {
-            bucket: "source".to_string(),
-            name: "plain".to_string(),
-            etag: Some("00112233445566778899aabbccddeeff".to_string()),
-            ..Default::default()
-        };
-        let mismatch = RemotePutObjectResponse {
-            version_id: None,
-            etag: Some("ffeeddccbbaa99887766554433221100".to_string()),
-        };
-        let err = verify_single_part_replica(&object_info, &mismatch, false)
-            .expect_err("a plaintext source whose replica MD5 differs must fail the replication");
-        assert_eq!(err.to_string(), REPLICA_ETAG_MISMATCH_ERROR);
-
-        let matching = RemotePutObjectResponse {
-            version_id: None,
-            etag: object_info.etag.clone(),
-        };
-        verify_single_part_replica(&object_info, &matching, false).expect("matching plaintext MD5s pass the audit");
-    }
-
     #[test]
     fn multipart_read_plan_preserves_legacy_plain_part_ranges() {
         const MIB: usize = 1024 * 1024;
@@ -6777,93 +6675,6 @@ mod tests {
         assert!(seen[0].starts_with("HEAD "), "{seen:?}");
     }
 
-    /// AWS S3 answers a HEAD for a version id it did not mint with a bodiless
-    /// 400. On a target with no verdict yet, that teaches `MintsOwn` and the
-    /// marker is still created with one versionless DELETE.
-    #[tokio::test]
-    async fn live_delete_marker_worker_learns_mints_own_from_bodiless_400() {
-        let (endpoint, server) = spawn_scripted_target_server(2, |line| {
-            if line.starts_with("HEAD ") {
-                empty_response("400 Bad Request")
-            } else if line.starts_with("DELETE ") {
-                empty_response("204 No Content")
-            } else {
-                empty_response("500 Unexpected")
-            }
-        });
-        let target = test_target_client(endpoint);
-        register_test_target(&target).await;
-        let dobj = marker_creation_dobj(&target.arn);
-
-        let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
-
-        assert_eq!(rinfo.replication_status, ReplicationStatusType::Completed, "{rinfo:?}");
-        assert_eq!(
-            ReplicationTargetStore::version_identity_capability(&target.arn),
-            VersionIdentityCapability::MintsOwn
-        );
-        let seen = server.join().expect("test HTTP server should finish");
-        assert_eq!(seen.len(), 2, "{seen:?}");
-        assert!(seen[0].starts_with("HEAD "), "{seen:?}");
-        assert!(seen[1].starts_with("DELETE "), "{seen:?}");
-        assert!(!seen[1].contains("versionId="), "marker creation must remain versionless: {seen:?}");
-    }
-
-    /// A bodiless 403 on the version-addressed HEAD may be a genuine
-    /// GetObjectVersion denial on an id-adopting target. It must fail closed:
-    /// no DELETE, and no `MintsOwn` verdict cached for the ARN.
-    #[tokio::test]
-    async fn live_delete_marker_worker_keeps_bodiless_403_failure_closed() {
-        let (endpoint, server) = spawn_scripted_target_server(1, |line| {
-            if line.starts_with("HEAD ") {
-                empty_response("403 Forbidden")
-            } else if line.starts_with("DELETE ") {
-                empty_response("204 No Content")
-            } else {
-                empty_response("500 Unexpected")
-            }
-        });
-        let target = test_target_client(endpoint);
-        register_test_target(&target).await;
-        let dobj = marker_creation_dobj(&target.arn);
-
-        let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
-
-        assert_eq!(rinfo.replication_status, ReplicationStatusType::Failed, "{rinfo:?}");
-        assert_ne!(
-            ReplicationTargetStore::version_identity_capability(&target.arn),
-            VersionIdentityCapability::MintsOwn
-        );
-        let seen = server.join().expect("test HTTP server should finish");
-        assert_eq!(seen.len(), 1, "a denied HEAD must not trigger DELETE: {seen:?}");
-        assert!(seen[0].starts_with("HEAD "), "{seen:?}");
-    }
-
-    /// A target already known to mint its own version ids is never asked for
-    /// the source marker id: the marker is created by the versionless DELETE.
-    #[tokio::test]
-    async fn live_delete_marker_worker_skips_version_head_on_mints_own_target() {
-        let (endpoint, server) = spawn_scripted_target_server(1, |line| {
-            if line.starts_with("DELETE ") {
-                empty_response("204 No Content")
-            } else {
-                empty_response("500 Unexpected")
-            }
-        });
-        let target = test_target_client(endpoint);
-        register_test_target(&target).await;
-        ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
-        let dobj = marker_creation_dobj(&target.arn);
-
-        let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
-
-        assert_eq!(rinfo.replication_status, ReplicationStatusType::Completed, "{rinfo:?}");
-        let seen = server.join().expect("test HTTP server should finish");
-        assert_eq!(seen.len(), 1, "no version-addressed HEAD on a MintsOwn target: {seen:?}");
-        assert!(seen[0].starts_with("DELETE "), "{seen:?}");
-        assert!(!seen[0].contains("versionId="), "marker creation must remain versionless: {seen:?}");
-    }
-
     /// A target answers `HEAD ?versionId=<delete marker>` with a bodiless
     /// 405, so the SDK reports no error code. That is the marker having
     /// propagated, not a failure (rustfs/backlog#2479).
@@ -6954,76 +6765,6 @@ mod tests {
         server.join().expect("test HTTP server should finish");
     }
 
-    /// AWS answers `HEAD ?versionId=<source marker id>` with a bodiless 400.
-    /// The marker is verified through the latest version instead: a
-    /// versionless 404 with `x-amz-delete-marker: true` proves it propagated.
-    #[tokio::test]
-    async fn resync_verification_accepts_latest_delete_marker_when_target_rejects_source_id() {
-        let (endpoint, server) = spawn_scripted_target_server(2, |line| {
-            if line.contains("versionId=") {
-                empty_response("400 Bad Request")
-            } else {
-                "HTTP/1.1 404 Not Found\r\nX-Amz-Delete-Marker: true\r\nX-Amz-Version-Id: target-marker\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
-                    .to_string()
-            }
-        });
-        let target = test_target_client(endpoint);
-        let roi = delete_marker_roi();
-        let mut st = TargetReplicationResyncStatus::default();
-
-        let head_result =
-            head_object_for_worker(target.as_ref(), &target.bucket, &roi.name, roi.version_id.map(|v| v.to_string())).await;
-        let (size, err) = verify_resync_head_result(head_result, &roi, &mut st, &target).await;
-
-        assert!(err.is_none(), "{err:?}");
-        assert_eq!((size, st.replicated_count, st.failed_count), (0, 1, 0));
-        let seen = server.join().expect("test HTTP server should finish");
-        assert!(!seen[1].contains("versionId="), "the re-check must address the latest version: {seen:?}");
-    }
-
-    /// A live latest version on a target that rejects the source id means the
-    /// marker has not propagated: the verification stays failed.
-    #[tokio::test]
-    async fn resync_verification_fails_delete_marker_when_latest_target_version_is_live() {
-        let (endpoint, server) = spawn_scripted_target_server(2, |line| {
-            if line.contains("versionId=") {
-                empty_response("400 Bad Request")
-            } else {
-                empty_response("200 OK")
-            }
-        });
-        let target = test_target_client(endpoint);
-        let roi = delete_marker_roi();
-        let mut st = TargetReplicationResyncStatus::default();
-
-        let head_result =
-            head_object_for_worker(target.as_ref(), &target.bucket, &roi.name, roi.version_id.map(|v| v.to_string())).await;
-        let (size, err) = verify_resync_head_result(head_result, &roi, &mut st, &target).await;
-
-        assert!(err.is_some(), "a live latest version must not verify the marker");
-        assert_eq!((size, st.replicated_count, st.failed_count), (0, 0, 1));
-        server.join().expect("test HTTP server should finish");
-    }
-
-    /// A bodiless 403 may be a genuine permission denial: it is not evidence
-    /// that the target mints its own ids, so no latest-version re-check runs.
-    #[tokio::test]
-    async fn resync_verification_keeps_delete_marker_failed_on_bodiless_403() {
-        let (endpoint, server) = spawn_scripted_target_server(1, |_| empty_response("403 Forbidden"));
-        let target = test_target_client(endpoint);
-        let roi = delete_marker_roi();
-        let mut st = TargetReplicationResyncStatus::default();
-
-        let head_result =
-            head_object_for_worker(target.as_ref(), &target.bucket, &roi.name, roi.version_id.map(|v| v.to_string())).await;
-        let (size, err) = verify_resync_head_result(head_result, &roi, &mut st, &target).await;
-
-        assert!(err.is_some(), "a 403 must not count as a propagated delete marker");
-        assert_eq!((size, st.replicated_count, st.failed_count), (0, 0, 1));
-        let seen = server.join().expect("test HTTP server should finish");
-        assert_eq!(seen.len(), 1, "{seen:?}");
-    }
-
     /// Serves exactly `requests` connections, answering each from
     /// `respond(request_line)`, and returns the request lines it saw. Reads
     /// the whole request (headers plus `Content-Length` body, honoring

crates/ecstore/src/bucket/replication/replication_target_boundary.rs

@@ -157,19 +157,6 @@ pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap<String
     rustfs_replication::is_ssec_encrypted(user_defined)
 }
 
-/// Whether the stored ETag of a replication source can be compared with the
-/// ETag a target returns for the bytes it received. Only an unencrypted source
-/// stores the MD5 of the bytes that go on the wire. A managed-SSE (SSE-S3 /
-/// SSE-KMS) or SSE-C source stores a keyed fingerprint (new writes) or a legacy
-/// plaintext MD5, and the target re-encrypts what it receives, so neither
-/// equality nor inequality of the two ETags proves anything about the replica.
-pub(crate) fn replication_source_etag_is_comparable(user_defined: &HashMap<String, String>) -> bool {
-    matches!(
-        classify_replication_source_encryption(user_defined),
-        ReplicationSourceEncryption::Plaintext
-    )
-}
-
 /// HeadObjectOutput adapter over the pure SSE-C passthrough evidence
 /// judgment owned by `rustfs-replication`: extract the echoed
 /// customer-algorithm header and let the crate-owned policy decide.
@@ -591,22 +578,6 @@ mod tests {
     use time::Duration;
     use uuid::Uuid;
 
-    #[test]
-    fn replication_source_etag_is_comparable_only_for_plaintext_sources() {
-        assert!(replication_source_etag_is_comparable(&HashMap::new()));
-        for (key, value) in [
-            (metadata_keys::SERVER_SIDE_ENCRYPTION, "AES256"),
-            (metadata_keys::SERVER_SIDE_ENCRYPTION, "aws:kms"),
-            (SSEC_ALGORITHM_HEADER, "AES256"),
-        ] {
-            let metadata = HashMap::from([(key.to_string(), value.to_string())]);
-            assert!(
-                !replication_source_etag_is_comparable(&metadata),
-                "{key}={value} must not expose its stored ETag to the replica audit"
-            );
-        }
-    }
-
     /// backlog#1735 A3b SSE key-case finding. Every RustFS writer persists the
     /// SSE intent as lowercase `x-amz-server-side-encryption`
     /// (`encryption_material_to_metadata`; header-derived keys come from a

crates/ecstore/src/cluster/rpc/client.rs

@@ -353,13 +353,6 @@ fn apply_peer_replay_response(
     }
 }
 
-fn rename_data_grpc_stage(path: &str) -> bool {
-    matches!(
-        path,
-        "/node_service.NodeService/RenameData" | "/node_service.NodeService/RenameDataAtIncarnation"
-    )
-}
-
 impl<S, ReqBody, ResBody> Service<HttpRequest<ReqBody>> for ReplayScopeChannel<S>
 where
     S: Service<HttpRequest<ReqBody>, Response = HttpResponse<ResBody>>,
@@ -377,12 +370,6 @@ where
     }
 
     fn call(&mut self, mut request: HttpRequest<ReqBody>) -> Self::Future {
-        let observe_rename_data = rustfs_io_metrics::put_stage_metrics_enabled() && rename_data_grpc_stage(request.uri().path());
-        let request_scope_started = if observe_rename_data {
-            rustfs_io_metrics::put_stage_timer()
-        } else {
-            None
-        };
         let authenticated = self.audience.as_ref().is_some_and(|_| {
             request
                 .headers()
@@ -417,30 +404,11 @@ where
                 }
             }
         }
-        rustfs_io_metrics::record_put_object_stage_duration_from(
-            rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REQUEST_SCOPE,
-            request_scope_started,
-        );
 
         let audience = self.audience.clone();
         let future = self.inner.call(request);
         Box::pin(async move {
-            let response = if observe_rename_data {
-                rustfs_io_metrics::observe_put_stage_future(
-                    future,
-                    rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL,
-                    rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_POLL_PENDING_COUNT,
-                    rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_FIRST_PENDING_TO_READY,
-                )
-                .await?
-            } else {
-                future.await?
-            };
-            let replay_response_started = if observe_rename_data {
-                rustfs_io_metrics::put_stage_timer()
-            } else {
-                None
-            };
+            let response = future.await?;
             if let (Some(audience), Some(challenge)) = (audience, challenge) {
                 let response_state = verify_tonic_peer_replay_capabilities_response(&audience, challenge, response.headers());
                 if let Err(error) = &response_state
@@ -458,10 +426,6 @@ where
                 }
                 apply_peer_replay_response(audience, sent_state, response_state);
             }
-            rustfs_io_metrics::record_put_object_stage_duration_from(
-                rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REPLAY_RESPONSE,
-                replay_response_started,
-            );
             Ok(response)
         })
     }

crates/ecstore/src/cluster/rpc/network_probe.rs

@@ -325,24 +325,6 @@ fn map_status(status: tonic::Status) -> NetworkPeerProbeError {
     match status.code() {
         Code::Unavailable | Code::Unknown => NetworkPeerProbeError::Unreachable,
         Code::DeadlineExceeded => NetworkPeerProbeError::TimedOut,
-        Code::Cancelled | Code::Internal => {
-            let mut source = std::error::Error::source(&status);
-            while let Some(error) = source {
-                if error
-                    .downcast_ref::<h2::Error>()
-                    .is_some_and(|error| matches!(error.reason(), Some(h2::Reason::CANCEL | h2::Reason::NO_ERROR)))
-                {
-                    return NetworkPeerProbeError::Unreachable;
-                }
-                source = if let Some(error) = error.downcast_ref::<std::io::Error>() {
-                    // io::Error::source skips its immediate custom error.
-                    error.get_ref().map(|inner| inner as &(dyn std::error::Error + 'static))
-                } else {
-                    error.source()
-                };
-            }
-            NetworkPeerProbeError::ProtocolFailure
-        }
         _ => NetworkPeerProbeError::ProtocolFailure,
     }
 }
@@ -353,35 +335,6 @@ mod tests {
     use crate::layout::endpoint::Endpoint;
     use crate::layout::endpoints::{Endpoints, PoolEndpoints};
 
-    #[test]
-    fn peer_transport_resets_are_unreachable_without_reclassifying_application_errors() {
-        for reason in [h2::Reason::CANCEL, h2::Reason::NO_ERROR] {
-            let status = tonic::Status::from_error(Box::new(h2::Error::from(reason)));
-            assert!(matches!(status.code(), Code::Cancelled | Code::Internal));
-            let code = status.code();
-            assert_eq!(map_status(status), NetworkPeerProbeError::Unreachable);
-            let mut wrapped = tonic::Status::new(code, "wrapped transport reset");
-            wrapped.set_source(Arc::new(std::io::Error::other(h2::Error::from(reason))));
-            assert_eq!(map_status(wrapped), NetworkPeerProbeError::Unreachable);
-        }
-        for reason in [h2::Reason::PROTOCOL_ERROR, h2::Reason::COMPRESSION_ERROR] {
-            let status = tonic::Status::from_error(Box::new(h2::Error::from(reason)));
-            assert_eq!(map_status(status), NetworkPeerProbeError::ProtocolFailure);
-        }
-        for status in [
-            tonic::Status::cancelled("peer cancelled the application request"),
-            tonic::Status::internal("peer rejected the probe"),
-            tonic::Status::unauthenticated("authentication rejected"),
-            tonic::Status::permission_denied("authorization rejected"),
-        ] {
-            assert_eq!(map_status(status), NetworkPeerProbeError::ProtocolFailure);
-        }
-        assert_eq!(
-            map_status(tonic::Status::deadline_exceeded("probe deadline elapsed")),
-            NetworkPeerProbeError::TimedOut
-        );
-    }
-
     fn pacing(duration: Duration) -> Arc<DiagnosticPacing> {
         NetworkPeerProbeClient::from_endpoint_pools(&topology())
             .with_diagnostic_pacing(Instant::now(), duration)

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -2279,7 +2279,6 @@ impl RemoteDisk {
         self.execute_with_timeout_for_op(
             "rename_data",
             || async {
-                let prepare_started = rustfs_io_metrics::put_stage_timer();
                 let file_info = compat_json(fi)?;
                 let file_info_bin = encode_file_info_msgpack(fi)?;
                 let mut client = self.get_client().await?;
@@ -2307,67 +2306,24 @@ impl RemoteDisk {
                 } else {
                     attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
                 }
-                rustfs_io_metrics::record_put_object_stage_duration_from(
-                    rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE,
-                    prepare_started,
-                );
 
-                let rpc_started = rustfs_io_metrics::put_stage_timer();
-                let response = rustfs_io_metrics::observe_put_stage_future(
-                    async {
-                        if incarnation_bound {
-                            // Older peers return Unimplemented before mutation; never downgrade.
-                            client.rename_data_at_incarnation(request).await
-                        } else {
-                            client.rename_data(request).await
-                        }
-                    },
-                    rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT,
-                    rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT,
-                    rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY,
-                )
-                .await?;
-                let response_process_started = rustfs_io_metrics::put_stage_timer();
-                let response_headers_started = rustfs_io_metrics::put_stage_timer();
-                let _ = response.metadata().len();
-                rustfs_io_metrics::record_put_object_stage_duration_from(
-                    rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_HEADERS,
-                    response_headers_started,
-                );
-                let into_inner_started = rustfs_io_metrics::put_stage_timer();
-                let response = response.into_inner();
-                rustfs_io_metrics::record_put_object_stage_duration_from(
-                    rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_INTO_INNER,
-                    into_inner_started,
-                );
-                rustfs_io_metrics::record_put_object_stage_duration_from(
-                    rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC,
-                    rpc_started,
-                );
+                let response = if incarnation_bound {
+                    // Older peers return Unimplemented before mutation; never downgrade.
+                    client.rename_data_at_incarnation(request).await?
+                } else {
+                    client.rename_data(request).await?
+                }
+                .into_inner();
 
-                let success_check_started = rustfs_io_metrics::put_stage_timer();
                 if !response.success {
                     return Err(response.error.unwrap_or_default().into());
                 }
-                rustfs_io_metrics::record_put_object_stage_duration_from(
-                    rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_SUCCESS_CHECK,
-                    success_check_started,
-                );
 
-                let decode_started = rustfs_io_metrics::put_stage_timer();
                 let rename_data_resp = decode_msgpack_or_json::<RenameDataResp>(
                     &response.rename_data_resp_bin,
                     &response.rename_data_resp,
                     "RenameDataResp",
                 )?;
-                rustfs_io_metrics::record_put_object_stage_duration_from(
-                    rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_DECODE,
-                    decode_started,
-                );
-                rustfs_io_metrics::record_put_object_stage_duration_from(
-                    rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_PROCESS,
-                    response_process_started,
-                );
 
                 Ok(rename_data_resp)
             },
@@ -5160,8 +5116,6 @@ mod tests {
             cleanup_data_dir: Some(Uuid::new_v4()),
             sign: Some(vec![0x14, 0x35]),
             old_current_size: Some(crate::disk::OldCurrentSize::Present(64 * 1024)),
-            old_current_source_checked: false,
-            old_current_source: None,
         };
         let json = serde_json::to_string(&response).expect("legacy rename_data JSON response should encode");
         let decode_before = rustfs_io_metrics::internode_metrics::global_internode_metrics().msgpack_json_decode_total_for_test();
@@ -5564,8 +5518,6 @@ mod tests {
             cleanup_data_dir: Some(Uuid::new_v4()),
             sign: Some(vec![1_u8; 32]),
             old_current_size: Some(crate::disk::OldCurrentSize::Present(4096)),
-            old_current_source_checked: false,
-            old_current_source: None,
         };
         let json = serde_json::to_vec(&response).expect("rename data response json should encode");
         let named_msgpack = rmp_serde::encode::to_vec_named(&response).expect("rename data response named msgpack should encode");

crates/ecstore/src/core/pools.rs

@@ -3691,55 +3691,6 @@ fn activation_pool_meta_lock_error(err: rustfs_lock::LockError) -> Error {
     }
 }
 
-fn rebalance_worker_read_lock_error(err: rustfs_lock::LockError, object: &str) -> Error {
-    match err {
-        rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable {
-            mode: "read",
-            bucket: RUSTFS_META_BUCKET.to_string(),
-            object: object.to_string(),
-            required,
-            achieved,
-        },
-        other => Error::Lock(other),
-    }
-}
-
-/// Read-only admission to an already durable rebalance run. Keeping this
-/// separate from the activation writer prevents admission from authorizing a
-/// pool or rebalance metadata write under shared namespace locks.
-pub(crate) struct PoolRebalanceWorkerFence {
-    pool_meta_guard: rustfs_lock::NamespaceLockGuard,
-    rebalance_meta_guard: rustfs_lock::NamespaceLockGuard,
-    fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
-}
-
-impl PoolRebalanceWorkerFence {
-    pub(crate) fn set_fleet_proof(
-        &mut self,
-        fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
-    ) {
-        self.fleet_proof = fleet_proof;
-    }
-
-    pub(crate) fn ensure_held(&self) -> Result<()> {
-        if self.pool_meta_guard.is_lock_lost()
-            || self.pool_meta_guard.is_released()
-            || self.rebalance_meta_guard.is_lock_lost()
-            || self.rebalance_meta_guard.is_released()
-        {
-            return Err(Error::other("worker admission lock lost before rebalance activation"));
-        }
-        if self
-            .fleet_proof
-            .as_ref()
-            .is_some_and(|proof| !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof))
-        {
-            return Err(Error::other(POOL_ACTIVATION_FLEET_PROOF_EXPIRED));
-        }
-        Ok(())
-    }
-}
-
 pub(crate) struct PoolRebalanceActivationFence {
     pool_meta_guard: Arc<rustfs_lock::NamespaceLockGuard>,
     rebalance_meta_guard: Arc<rustfs_lock::NamespaceLockGuard>,
@@ -3811,35 +3762,6 @@ tokio::task_local! {
     pub(crate) static REBALANCE_ACTIVATION_LOCK_ATTEMPT: Arc<tokio::sync::Notify>;
 }
 
-pub(crate) async fn acquire_pool_rebalance_worker_locks<S>(pool: Arc<S>) -> Result<PoolRebalanceWorkerFence>
-where
-    S: crate::storage_api_contracts::namespace::NamespaceLocking<
-            Error = Error,
-            NamespaceLock = rustfs_lock::NamespaceLockWrapper,
-        >,
-{
-    // Match activation and entry admission: rebalance.bin -> pool.bin. Joining
-    // the same durable run must coexist with entries retaining their run fence.
-    let rebalance_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
-    #[cfg(test)]
-    let _ = REBALANCE_ACTIVATION_LOCK_ATTEMPT.try_with(|attempted| attempted.notify_one());
-    let rebalance_meta_guard = rebalance_meta_lock
-        .get_read_lock(get_lock_acquire_timeout())
-        .await
-        .map_err(|err| rebalance_worker_read_lock_error(err, REBAL_META_NAME))?;
-    let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
-    let pool_meta_guard = pool_meta_lock
-        .get_read_lock(get_lock_acquire_timeout())
-        .await
-        .map_err(|err| rebalance_worker_read_lock_error(err, POOL_META_NAME))?;
-
-    Ok(PoolRebalanceWorkerFence {
-        pool_meta_guard,
-        rebalance_meta_guard,
-        fleet_proof: None,
-    })
-}
-
 pub(crate) async fn acquire_pool_rebalance_activation_locks<S>(
     pool: Arc<S>,
     fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
@@ -12508,18 +12430,6 @@ impl ECStore {
         Ok(pool_meta)
     }
 
-    pub(crate) async fn load_runtime_pool_meta_under_worker_fence(
-        &self,
-        write_state: &mut PoolMetaWriteState,
-        worker_fence: &PoolRebalanceWorkerFence,
-        operation: &str,
-    ) -> Result<PoolMeta> {
-        worker_fence.ensure_held()?;
-        let pool_meta = self.load_runtime_pool_meta_observing(write_state, operation).await?;
-        worker_fence.ensure_held()?;
-        Ok(pool_meta)
-    }
-
     pub(crate) async fn load_runtime_pool_meta_under_activation_fence(
         &self,
         write_state: &mut PoolMetaWriteState,

crates/ecstore/src/core/sets.rs

@@ -72,6 +72,38 @@ type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
 
 const LIST_MULTIPART_SETS_CONCURRENCY: usize = 4;
 
+#[cfg(test)]
+type TestSetLockers = HashMap<usize, Vec<Arc<dyn rustfs_lock::LockClient>>>;
+
+#[cfg(test)]
+static TEST_SET_LOCKERS: std::sync::LazyLock<std::sync::Mutex<TestSetLockers>> =
+    std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
+
+/// Keep a fixture's real locker domain scoped to its owning instance.
+#[cfg(test)]
+pub(crate) struct ScopedSetLockers(Arc<InstanceContext>);
+
+#[cfg(test)]
+impl ScopedSetLockers {
+    pub(crate) fn install(ctx: Arc<InstanceContext>, lockers: Vec<Arc<dyn rustfs_lock::LockClient>>) -> Self {
+        let key = Arc::as_ptr(&ctx) as usize;
+        let mut domains = TEST_SET_LOCKERS.lock().expect("test set lockers mutex");
+        assert!(!domains.contains_key(&key), "the instance already owns a test locker domain");
+        domains.insert(key, lockers);
+        Self(ctx)
+    }
+}
+
+#[cfg(test)]
+impl Drop for ScopedSetLockers {
+    fn drop(&mut self) {
+        TEST_SET_LOCKERS
+            .lock()
+            .expect("test set lockers mutex")
+            .remove(&(Arc::as_ptr(&self.0) as usize));
+    }
+}
+
 fn is_idempotent_delete_prefix_error(err: &Error) -> bool {
     is_err_object_not_found(err) || is_err_strict_volume_not_found(err)
 }
@@ -273,6 +305,14 @@ impl Sets {
                 }
             }
 
+            #[cfg(test)]
+            let pool_lockers = TEST_SET_LOCKERS
+                .lock()
+                .expect("test set lockers mutex")
+                .get(&(Arc::as_ptr(&instance_ctx) as usize))
+                .cloned()
+                .unwrap_or_else(|| pool_lockers.clone());
+
             let set_disks = SetDisks::new_with_instance_ctx(
                 runtime_sources::local_node_name().await,
                 Arc::new(RwLock::new(set_drive)),

crates/ecstore/src/diagnostics/admin_server_info.rs

@@ -44,11 +44,7 @@ use std::{
 use time::OffsetDateTime;
 use tokio::time::timeout;
 use tonic::Request;
-use tracing::{error, trace, warn};
-
-const EVENT_SERVER_PING: &str = "server_ping";
-const LOG_COMPONENT_ECSTORE: &str = "ecstore";
-const LOG_SUBSYSTEM_DIAGNOSTICS: &str = "diagnostics";
+use tracing::warn;
 
 use shadow_rs::shadow;
 
@@ -163,22 +159,11 @@ async fn is_server_resolvable(endpoint: &Endpoint) -> Result<()> {
 
         let response: PingResponse = client.ping(request).await?.into_inner();
 
-        if flatbuffers::root::<PingBody>(&response.body).is_err() {
-            error!(
-                event = EVENT_SERVER_PING,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_DIAGNOSTICS,
-                result = "invalid_response",
-                "Server ping response decoding failed"
-            );
+        let ping_response_body = flatbuffers::root::<PingBody>(&response.body);
+        if let Err(e) = ping_response_body {
+            eprintln!("{e}");
         } else {
-            trace!(
-                event = EVENT_SERVER_PING,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_DIAGNOSTICS,
-                result = "success",
-                "Server ping completed"
-            );
+            println!("ping_resp:body(flatbuffer): {ping_response_body:?}");
         }
 
         Ok(())

crates/ecstore/src/disk/disk_store.rs

@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::disk::cleanup_runtime;
 use crate::disk::io_admission::{
     AdmissionReader, BoxedChunkReader, DiskIoAdmission, DiskIoClass, DiskIoContext, current_disk_io_context,
     with_disk_io_context, with_disk_io_permit,
@@ -485,10 +484,10 @@ impl LocalDiskWrapper {
         opts: DeleteOptions,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
     ) -> Result<()> {
-        self.track_cleanup_mutation(
+        self.track_disk_health_mutation(
             "delete",
             DiskMetricMutation::Delete,
-            || async { Box::pin(self.cleanup_delete(volume, path, opts, namespace_owner)).await },
+            || async { Box::pin(self.disk.delete_with_namespace_owner(volume, path, opts, namespace_owner)).await },
             get_max_timeout_duration(),
         )
         .await
@@ -1476,28 +1475,6 @@ impl LocalDiskWrapper {
         )
     }
 
-    async fn cleanup_delete(
-        &self,
-        volume: &str,
-        path: &str,
-        options: DeleteOptions,
-        owner: Option<Arc<dyn Send + Sync>>,
-    ) -> Result<()> {
-        if !cleanup_runtime::enabled()? {
-            return self.disk.delete_with_namespace_owner(volume, path, options, owner).await;
-        }
-        let disk = self.disk.clone();
-        let volume = volume.to_owned();
-        let path = path.to_owned();
-        let guarded = owner.is_some();
-        let operation = async move { disk.delete_with_namespace_owner(&volume, &path, options, owner).await };
-        if guarded {
-            cleanup_runtime::guarded_disk_operation(operation).await
-        } else {
-            cleanup_runtime::disk_operation(operation).await
-        }
-    }
-
     /// Run a delete under an owned coordinator task when a publication guard
     /// is present. This keeps the guard alive if the RPC waiter is cancelled
     /// while the local namespace mutation is still in progress.
@@ -1516,13 +1493,12 @@ impl LocalDiskWrapper {
         } else {
             get_max_timeout_duration()
         };
-        let cleanup_owner = external_guard.clone();
         run_owned_mutation(external_guard, move || async move {
             operation
-                .track_cleanup_mutation(
+                .track_disk_health_mutation(
                     "delete",
                     DiskMetricMutation::Delete,
-                    || async { operation.cleanup_delete(&volume, &path, options, cleanup_owner).await },
+                    || async { operation.disk.delete(&volume, &path, options).await },
                     timeout_duration,
                 )
                 .await
@@ -2116,31 +2092,6 @@ impl LocalDiskWrapper {
         self.track_disk_health_with_op("unknown", operation, timeout_duration).await
     }
 
-    /// Cleanup latency includes waiting behind a node-wide execution budget,
-    /// so a waiter deadline alone is not evidence that this disk is faulty.
-    /// The active disk health probe retains its original timeout policy.
-    async fn track_cleanup_mutation<T, F, Fut>(
-        &self,
-        op: &'static str,
-        mutation: DiskMetricMutation,
-        operation: F,
-        timeout_duration: Duration,
-    ) -> Result<T>
-    where
-        F: FnOnce() -> Fut,
-        Fut: std::future::Future<Output = Result<T>>,
-    {
-        // Runtime initialization must not prevent control-plane release of
-        // reader tokens. Disk I/O entries still propagate configuration errors.
-        let action = if cleanup_runtime::enabled().unwrap_or(false) {
-            TimeoutHealthAction::IgnoreFailure
-        } else {
-            TimeoutHealthAction::MarkFailure
-        };
-        self.track_disk_health_with_op_timeout_action_and_mutation(op, operation, timeout_duration, action, mutation)
-            .await
-    }
-
     async fn track_disk_health_mutation<T, F, Fut>(
         &self,
         op: &'static str,
@@ -2527,6 +2478,15 @@ impl DiskAPI for LocalDiskWrapper {
         })
     }
 
+    async fn bucket_creation_witness(&self, volume: &str) -> Result<Option<super::os::NativeBucketCreationWitness>> {
+        self.track_disk_health_with_op(
+            "bucket_creation_witness",
+            || async { self.disk.bucket_creation_witness(volume).await },
+            Duration::ZERO,
+        )
+        .await
+    }
+
     async fn make_volume(&self, volume: &str) -> Result<()> {
         // Scoped heal must drain directory creation before releasing its lifecycle owner.
         let timeout = if crate::store::bucket_heal_scope(volume).is_some() {
@@ -2638,37 +2598,9 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<()> {
-        // Quota fence release is control-plane work and may wait for in-flight
-        // mutations. Keep its original waiter lifecycle instead of creating
-        // detached cleanup tasks for repeated fence-release RPCs.
-        if volume == super::RUSTFS_META_BUCKET && super::is_quota_mutation_fence_path(path) {
-            return self
-                .track_disk_health_with_op(
-                    "release_snapshot_lease",
-                    || async { self.disk.release_snapshot_lease(volume, path, token).await },
-                    get_max_timeout_duration(),
-                )
-                .await;
-        }
-        self.track_cleanup_mutation(
+        self.track_disk_health_with_op(
             "release_snapshot_lease",
-            DiskMetricMutation::None,
-            || async {
-                if self.disk.release_snapshot_lease_without_cleanup(volume, path, token).await {
-                    return Ok(());
-                }
-                if !cleanup_runtime::enabled().unwrap_or(false) {
-                    // The raw release removes the token before trying the
-                    // reclaim budget. Initialization failure therefore keeps
-                    // the cleanup intent and data, without leaking a reader
-                    // or falling back to unbudgeted filesystem deletion.
-                    return self.disk.release_snapshot_lease(volume, path, token).await;
-                }
-                let disk = self.disk.clone();
-                let volume = volume.to_owned();
-                let path = path.to_owned();
-                cleanup_runtime::release_lease(async move { disk.release_snapshot_lease(&volume, &path, token).await }).await
-            },
+            || async { self.disk.release_snapshot_lease(volume, path, token).await },
             get_max_timeout_duration(),
         )
         .await
@@ -2693,22 +2625,10 @@ impl DiskAPI for LocalDiskWrapper {
         } else {
             get_max_timeout_duration()
         };
-        self.track_cleanup_mutation(
+        self.track_disk_health_mutation(
             "delete_data_dir",
             DiskMetricMutation::Delete,
-            || async {
-                if !cleanup_runtime::enabled()? {
-                    return self.disk.delete_data_dir(volume, path, opts).await;
-                }
-                let disk = self.disk.clone();
-                let volume = volume.to_owned();
-                let path = path.to_owned();
-                cleanup_runtime::disk_operation(async move {
-                    let _scope = scope;
-                    disk.delete_data_dir(&volume, &path, opts).await
-                })
-                .await
-            },
+            || async { self.disk.delete_data_dir(volume, path, opts).await },
             timeout,
         )
         .await
@@ -2955,10 +2875,10 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
-        self.track_cleanup_mutation(
+        self.track_disk_health_mutation(
             "delete",
             DiskMetricMutation::Delete,
-            || async { self.cleanup_delete(volume, path, opt, None).await },
+            || async { self.disk.delete(volume, path, opt).await },
             get_max_timeout_duration(),
         )
         .await
@@ -2991,25 +2911,11 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn write_all(&self, volume: &str, path: &str, data: Bytes) -> Result<()> {
-        let cleanup = path.rsplit('/').next() == Some(cleanup_runtime::OLD_DATA_CLEANUP_RECEIPT_FILE);
-        self.track_disk_health_with_op_timeout_action_and_mutation(
+        self.track_disk_health_mutation(
             "write_all",
-            || async {
-                if !cleanup || !cleanup_runtime::enabled()? {
-                    return self.disk.write_all(volume, path, data).await;
-                }
-                let disk = self.disk.clone();
-                let volume = volume.to_owned();
-                let path = path.to_owned();
-                cleanup_runtime::disk_operation(async move { disk.write_all(&volume, &path, data).await }).await
-            },
-            get_max_timeout_duration(),
-            if cleanup && cleanup_runtime::enabled()? {
-                TimeoutHealthAction::IgnoreFailure
-            } else {
-                TimeoutHealthAction::MarkFailure
-            },
             DiskMetricMutation::Write,
+            || async { self.disk.write_all(volume, path, data).await },
+            get_max_timeout_duration(),
         )
         .await
     }
@@ -3402,256 +3308,6 @@ mod tests {
         let _ = walk.await;
     }
 
-    #[tokio::test]
-    async fn invalid_cleanup_config_does_not_leak_snapshot_readers() {
-        const CHILD: &str = "RUSTFS_CLEANUP_INVALID_LEASE_TEST_CHILD";
-        if std::env::var_os(CHILD).is_none() {
-            let output = tokio::task::spawn_blocking(|| {
-                std::process::Command::new(std::env::current_exe().expect("test executable"))
-                    .args([
-                        "--exact",
-                        "disk::disk_store::tests::invalid_cleanup_config_does_not_leak_snapshot_readers",
-                        "--nocapture",
-                    ])
-                    .env(CHILD, "1")
-                    .env("RUSTFS_CLEANUP_ISOLATE_ENABLE", "true")
-                    .env("RUSTFS_CLEANUP_ASYNC_THREADS", "0")
-                    .output()
-                    .expect("run with invalid cleanup configuration")
-            })
-            .await
-            .expect("child process task");
-            assert!(
-                output.status.success(),
-                "{}\n{}",
-                String::from_utf8_lossy(&output.stdout),
-                String::from_utf8_lossy(&output.stderr)
-            );
-            assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed"));
-            return;
-        }
-        let dir = tempfile::tempdir().expect("temporary disk");
-        let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint");
-        let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
-        let wrapper = LocalDiskWrapper::new(disk.clone(), false);
-        let volume = "invalid-cleanup-config";
-        let path = "object/data";
-        wrapper.make_volume(volume).await.expect("bucket");
-        wrapper
-            .write_all(volume, "object/data/part.1", Bytes::from_static(b"live shard"))
-            .await
-            .expect("write shard");
-        let token = wrapper.acquire_snapshot_lease(volume, path).await.expect("GET lease");
-        wrapper
-            .release_snapshot_lease(volume, path, token)
-            .await
-            .expect("control-only release must not depend on cleanup initialization");
-        assert!(matches!(
-            wrapper.renew_snapshot_lease(volume, path, token).await,
-            Err(DiskError::FileNotFound)
-        ));
-
-        let token = wrapper
-            .acquire_snapshot_lease(volume, path)
-            .await
-            .expect("reader of old data");
-        // Register reclaim intent directly; this only updates the lease
-        // registry while a reader is present and performs no filesystem I/O.
-        assert_eq!(
-            disk.delete_data_dir(
-                volume,
-                path,
-                DeleteOptions {
-                    recursive: true,
-                    ..Default::default()
-                }
-            )
-            .await
-            .expect("defer reclaim"),
-            DataDirDeleteStatus::Deferred
-        );
-        assert!(
-            wrapper.release_snapshot_lease(volume, path, token).await.is_err(),
-            "invalid configuration must still reject reclamation"
-        );
-        assert!(
-            matches!(wrapper.renew_snapshot_lease(volume, path, token).await, Err(DiskError::FileNotFound)),
-            "failed reclamation must not strand a reader token"
-        );
-        assert_eq!(
-            wrapper.read_all(volume, "object/data/part.1").await.expect("retained data"),
-            Bytes::from_static(b"live shard"),
-            "configuration failure must not fall back to unbudgeted deletion"
-        );
-    }
-
-    #[tokio::test]
-    async fn rpc_cleanup_entries_share_disk_budget() {
-        const CHILD: &str = "RUSTFS_CLEANUP_RPC_TEST_CHILD";
-        if std::env::var_os(CHILD).is_none() {
-            let output = tokio::task::spawn_blocking(|| {
-                std::process::Command::new(std::env::current_exe().unwrap())
-                    .args([
-                        "--exact",
-                        "disk::disk_store::tests::rpc_cleanup_entries_share_disk_budget",
-                        "--nocapture",
-                    ])
-                    .env(CHILD, "1")
-                    .env("RUSTFS_CLEANUP_ISOLATE_ENABLE", "true")
-                    .env("RUSTFS_CLEANUP_ASYNC_THREADS", "1")
-                    .env("RUSTFS_CLEANUP_DISK_MAX_PENDING", "1")
-                    .env("RUSTFS_CLEANUP_DISK_WORKERS", "1")
-                    .env("RUSTFS_CLEANUP_GC_WORKERS", "1")
-                    .env("RUSTFS_CLEANUP_CPUS", "none")
-                    .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP")
-                    .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER")
-                    .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP")
-                    .output()
-                    .unwrap()
-            })
-            .await
-            .unwrap();
-            assert!(
-                output.status.success(),
-                "{}\n{}",
-                String::from_utf8_lossy(&output.stdout),
-                String::from_utf8_lossy(&output.stderr)
-            );
-            assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed"));
-            return;
-        }
-        let dir = tempfile::tempdir().unwrap();
-        let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
-        let disk = Arc::new(LocalDisk::new(&endpoint, false).await.unwrap());
-        let wrapper = LocalDiskWrapper::new(disk, false);
-        let volume = "cleanup-receiver";
-        let path = "object/old-data";
-        let body_path = format!("{path}/part.1");
-        let receipt_path = format!("{path}/{}", cleanup_runtime::OLD_DATA_CLEANUP_RECEIPT_FILE);
-        wrapper.make_volume(volume).await.unwrap();
-        wrapper
-            .write_all(volume, &body_path, Bytes::from_static(b"old body"))
-            .await
-            .unwrap();
-        let read_lease = wrapper
-            .acquire_snapshot_lease(volume, path)
-            .await
-            .expect("ordinary GET lease");
-        let (entered, entry) = tokio::sync::oneshot::channel();
-        let (release, released) = std::sync::mpsc::channel();
-        let occupied = tokio::spawn(cleanup_runtime::disk_operation(async move {
-            entered.send(()).unwrap();
-            // Deliberately stall the only cleanup scheduler to prove that
-            // ordinary GET completion does not dispatch to it.
-            released
-                .recv_timeout(Duration::from_secs(30))
-                .expect("release stalled scheduler");
-            Ok(())
-        }));
-        tokio::time::timeout(Duration::from_secs(20), entry).await.unwrap().unwrap();
-        tokio::time::timeout(Duration::from_secs(5), wrapper.release_snapshot_lease(volume, path, read_lease))
-            .await
-            .expect("ordinary GET release must progress while the cleanup scheduler is stalled")
-            .expect("release without reclamation");
-        let fence_path = super::super::quota_mutation_fence_path(volume, "object");
-        let fence = wrapper
-            .acquire_snapshot_lease(super::super::RUSTFS_META_BUCKET, &fence_path)
-            .await
-            .unwrap();
-        wrapper
-            .release_snapshot_lease(super::super::RUSTFS_META_BUCKET, &fence_path, fence)
-            .await
-            .unwrap();
-        let timeout = wrapper
-            .track_cleanup_mutation(
-                "delete",
-                DiskMetricMutation::Delete,
-                std::future::pending::<Result<()>>,
-                Duration::from_millis(10),
-            )
-            .await;
-        assert!(matches!(timeout, Err(DiskError::Timeout)));
-        assert!(!wrapper.health.is_faulty(), "cleanup queue delay must not mark a healthy disk faulty");
-        let options = DeleteOptions {
-            recursive: true,
-            ..Default::default()
-        };
-        let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        // This is the same entry point used after RPC authentication and
-        // scanner publication-lease validation by handle_delete.
-        let result = wrapper
-            .delete_with_publication_guard(volume, path, options.clone(), Some(Arc::new(DropProbe(drops.clone()))))
-            .await;
-        assert!(matches!(result, Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock));
-        assert_eq!(drops.load(Ordering::SeqCst), 1);
-        assert!(matches!(wrapper.delete_data_dir(volume, path, options.clone()).await,
-            Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock));
-        assert!(matches!(wrapper.write_all(volume, &receipt_path, Bytes::from_static(b"receipt")).await,
-            Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock));
-        assert_eq!(wrapper.read_all(volume, &body_path).await.unwrap(), Bytes::from_static(b"old body"));
-        release.send(()).unwrap();
-        occupied.await.unwrap().unwrap();
-        wrapper
-            .write_all(volume, &receipt_path, Bytes::from_static(b"receipt"))
-            .await
-            .unwrap();
-        wrapper
-            .delete_with_publication_guard(volume, path, options, None)
-            .await
-            .unwrap();
-        assert!(matches!(wrapper.read_all(volume, &body_path).await, Err(DiskError::FileNotFound)));
-        // Snapshot readers defer old-data cleanup. Saturation must release
-        // their token while retaining reclaim intent, rather than leaking a
-        // live token or performing an unbudgeted delete on the RPC runtime.
-        wrapper
-            .write_all(volume, &body_path, Bytes::from_static(b"leased body"))
-            .await
-            .unwrap();
-        let lease = wrapper.acquire_snapshot_lease(volume, path).await.unwrap();
-        assert_eq!(
-            wrapper
-                .delete_data_dir(
-                    volume,
-                    path,
-                    DeleteOptions {
-                        recursive: true,
-                        ..Default::default()
-                    }
-                )
-                .await
-                .unwrap(),
-            DataDirDeleteStatus::Deferred
-        );
-        let (entered, entry) = tokio::sync::oneshot::channel();
-        let (release, released) = tokio::sync::oneshot::channel();
-        let occupied = tokio::spawn(cleanup_runtime::disk_operation(async move {
-            entered.send(()).unwrap();
-            released.await.unwrap();
-            Ok(())
-        }));
-        tokio::time::timeout(Duration::from_secs(20), entry).await.unwrap().unwrap();
-        assert!(matches!(wrapper.release_snapshot_lease(volume, path, lease).await,
-            Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock));
-        assert_eq!(wrapper.read_all(volume, &body_path).await.unwrap(), Bytes::from_static(b"leased body"));
-        release.send(()).unwrap();
-        occupied.await.unwrap().unwrap();
-        assert_eq!(
-            wrapper
-                .delete_data_dir(
-                    volume,
-                    path,
-                    DeleteOptions {
-                        recursive: true,
-                        ..Default::default()
-                    }
-                )
-                .await
-                .unwrap(),
-            DataDirDeleteStatus::Deleted
-        );
-        assert!(matches!(wrapper.read_all(volume, &body_path).await, Err(DiskError::FileNotFound)));
-    }
-
     #[tokio::test]
     async fn owned_mutation_keeps_publication_guard_after_waiter_cancellation() {
         let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));

crates/ecstore/src/disk/fs.rs

@@ -117,6 +117,9 @@ pub const O_APPEND: FileMode = 0x00400;
 //     create_new: bool,
 
 pub async fn open_file(path: impl AsRef<Path>, mode: FileMode) -> io::Result<File> {
+    if mode & O_CREATE != 0 {
+        super::os::record_native_publication(path.as_ref());
+    }
     let base_opts = match mode & (O_RDONLY | O_WRONLY | O_RDWR) {
         O_RDONLY => get_readonly_options(),
         O_WRONLY => get_writeonly_options(),
@@ -160,6 +163,7 @@ pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
 }
 
 pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
+    super::os::record_native_publication(path.as_ref());
     fs::create_dir_all(path.as_ref()).await
 }
 
@@ -211,18 +215,27 @@ pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
 }
 
 pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
+    super::os::record_native_publication(path.as_ref());
     fs::create_dir(path.as_ref()).await
 }
 
 pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
+    super::os::record_native_publication(to.as_ref());
     fs::rename(from, to).await
 }
 
 #[cfg(any(not(windows), test))]
 pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
+    super::os::record_native_publication(to.as_ref());
     std::fs::rename(from, to)
 }
 
+#[cfg(any(not(windows), test))]
+#[tracing::instrument(level = "debug", skip_all)]
+pub async fn read_file(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
+    fs::read(path.as_ref()).await
+}
+
 // Bucket existence cache - reduces statx syscalls for repeated bucket checks
 
 /// Cache for bucket directory existence checks.
@@ -608,6 +621,28 @@ mod tests {
         assert_eq!(content, "test content");
     }
 
+    #[tokio::test]
+    async fn test_read_file() {
+        let temp_dir = TempDir::new().unwrap();
+        let file_path = temp_dir.path().join("test_read.txt");
+
+        let test_content = b"This is test content for reading";
+        tokio::fs::write(&file_path, test_content).await.unwrap();
+
+        // Read file
+        let read_content = read_file(&file_path).await.unwrap();
+        assert_eq!(read_content, test_content);
+    }
+
+    #[tokio::test]
+    async fn test_read_file_nonexistent() {
+        let temp_dir = TempDir::new().unwrap();
+        let file_path = temp_dir.path().join("nonexistent.txt");
+
+        // Should fail for non-existent file
+        assert!(read_file(&file_path).await.is_err());
+    }
+
     #[tokio::test]
     async fn test_same_file() {
         let temp_dir = TempDir::new().unwrap();

crates/ecstore/src/disk/local/commit.rs

@@ -47,33 +47,6 @@ use tokio::fs;
 use tracing::{info, warn};
 use uuid::Uuid;
 
-fn transitioned_old_current_source(
-    dst_meta_existed: bool,
-    dst_meta_unparsable: bool,
-    xlmeta: &FileMeta,
-    volume: &str,
-    path: &str,
-    version_id: Uuid,
-) -> (bool, Option<FileInfo>) {
-    if dst_meta_unparsable {
-        return (false, None);
-    }
-    if !dst_meta_existed {
-        return (true, None);
-    }
-    let Ok(old_current) = xlmeta.into_fileinfo(volume, path, "", false, false, true) else {
-        return (false, None);
-    };
-    if old_current.version_id.unwrap_or_default() != version_id {
-        return (true, None);
-    }
-    if old_current.transition_status == rustfs_filemeta::TRANSITION_COMPLETE {
-        (true, Some(old_current))
-    } else {
-        (true, None)
-    }
-}
-
 /// Hold later repair publications after admitting one baseline object. The
 /// fixture arms this on one replacement disk before rejoining the cluster.
 #[cfg(feature = "e2e-test-hooks")]
@@ -241,7 +214,7 @@ pub(super) async fn lock_rename_commit_directories(
     publication_root: &os::PublicationRoot,
     mutation_lease: Arc<os::NamespaceMutationLease>,
 ) -> Result<os::RenameCommitGuard> {
-    #[cfg(any(windows, target_os = "linux"))]
+    #[cfg(windows)]
     let result = {
         let source_parent = source_parent.to_path_buf();
         let destination_parent = destination_parent.to_path_buf();
@@ -249,7 +222,7 @@ pub(super) async fn lock_rename_commit_directories(
         let publication_root = publication_root.clone();
         os::run_blocking_namespace_operation(mutation_lease, move || {
             let result = os::prepare_rename_commit_guard(&source_parent, &destination_parent, &base_dir, &publication_root);
-            #[cfg(all(test, windows))]
+            #[cfg(test)]
             if result.is_ok() {
                 run_destination_commit_directory_preparation(&destination_parent);
             }
@@ -257,7 +230,7 @@ pub(super) async fn lock_rename_commit_directories(
         })
         .await
     };
-    #[cfg(not(any(windows, target_os = "linux")))]
+    #[cfg(not(windows))]
     let result = {
         let _ = mutation_lease;
         os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root)
@@ -271,12 +244,12 @@ pub(super) async fn lock_rename_commit_directories(
     result.map_err(to_file_error).map_err(DiskError::from)
 }
 
-async fn prepare_rename_destination_metadata(
+async fn read_rename_destination_metadata(
     file_path: &Path,
     rename_commit_guard: &os::RenameCommitGuard,
     mutation_lease: Arc<os::NamespaceMutationLease>,
 ) -> Result<Option<Bytes>> {
-    #[cfg(any(windows, target_os = "linux"))]
+    #[cfg(windows)]
     let result = {
         let file_path = file_path.to_path_buf();
         let rename_commit_guard = rename_commit_guard.clone();
@@ -285,10 +258,10 @@ async fn prepare_rename_destination_metadata(
         })
         .await
     };
-    #[cfg(not(any(windows, target_os = "linux")))]
+    #[cfg(not(windows))]
     let _ = (rename_commit_guard, mutation_lease);
-    #[cfg(not(any(windows, target_os = "linux")))]
-    let result = match fs::read(file_path).await {
+    #[cfg(not(windows))]
+    let result = match super::super::fs::read_file(file_path).await {
         Ok(data) => Ok(Some(data)),
         Err(err) if err.kind() == ErrorKind::NotFound => Ok(None),
         Err(err) => Err(err),
@@ -425,7 +398,7 @@ impl LocalDisk {
 
         // xl.meta path
         let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?;
-        let dst_file_path = self.io_get_object_open_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?;
+        let dst_file_path = self.io_get_object_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?;
 
         // data_dir path
         let has_data_dir_path = {
@@ -480,7 +453,8 @@ impl LocalDisk {
             fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?;
         }
         // Acquire the common trees before reading destination metadata. On
-        // Windows this pins the object directory identity across publication.
+        // Windows this pins the object directory identity across metadata
+        // preparation, data publication, rollback backup, and final commit.
         let rename_commit_guard = lock_rename_commit_directories(
             src_file_parent,
             dst_file_parent,
@@ -498,12 +472,9 @@ impl LocalDisk {
             source_volume_dir: &src_volume_dir,
             data_paths: has_data_dir_path.as_ref(),
         };
-        let has_dst_buf = prepare_rename_destination_metadata(
-            commit.destination_metadata,
-            &commit.directory_guard,
-            commit.mutation_lease.clone(),
-        )
-        .await?;
+        let has_dst_buf =
+            read_rename_destination_metadata(commit.destination_metadata, &commit.directory_guard, commit.mutation_lease.clone())
+                .await?;
 
         if no_inline {
             // Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta
@@ -537,14 +508,6 @@ impl LocalDisk {
             }
 
             let version_id = fi.version_id.unwrap_or_default();
-            let (old_current_source_checked, old_current_source) = transitioned_old_current_source(
-                has_dst_buf.is_some(),
-                dst_meta_unparsable,
-                &xlmeta,
-                dst_volume,
-                dst_path,
-                version_id,
-            );
             let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
             let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok();
             let rollback_data_dir = has_old_data_dir.or_else(|| {
@@ -835,9 +798,7 @@ impl LocalDisk {
                 && let Some(parent) = dst_file_path.parent()
             {
                 let fsync_started = rustfs_io_metrics::put_stage_timer();
-                if let Err(err) =
-                    os::fsync_commit_directory(parent, &commit.directory_guard, commit.mutation_lease.clone(), None).await
-                {
+                if let Err(err) = os::fsync_dst_dir_group_commit(parent, Some(commit.mutation_lease.clone())).await {
                     rustfs_io_metrics::record_put_object_stage_duration_from(
                         rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
                         fsync_started,
@@ -936,8 +897,6 @@ impl LocalDisk {
                 cleanup_data_dir: has_old_data_dir,
                 sign: version_signature,
                 old_current_size,
-                old_current_source_checked,
-                old_current_source,
             })
         } else {
             // Inline metadata preparation is blocking. The transaction lease is
@@ -951,8 +910,6 @@ impl LocalDisk {
                 None
             };
             let dst_path_for_failpoint = dst_path.to_string();
-            let dst_volume_for_source = dst_volume.to_string();
-            let dst_path_for_source = dst_path.to_string();
             #[cfg(windows)]
             let source_parent = src_file_parent.to_path_buf();
             let rename_commit_guard_for_preparation = commit.directory_guard.clone();
@@ -997,14 +954,6 @@ impl LocalDisk {
                 };
 
                 let version_id = fi.version_id.unwrap_or_default();
-                let (old_current_source_checked, old_current_source) = transitioned_old_current_source(
-                    has_dst_buf.is_some(),
-                    dst_meta_unparsable,
-                    &xlmeta,
-                    &dst_volume_for_source,
-                    &dst_path_for_source,
-                    version_id,
-                );
                 let old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
                 let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok();
                 let rollback_data_dir = old_data_dir.or_else(|| {
@@ -1054,8 +1003,6 @@ impl LocalDisk {
                     old_data_dir,
                     version_signature,
                     old_current_size,
-                    old_current_source_checked,
-                    old_current_source,
                     staged_rollback_path,
                     has_dst_buf.is_none(),
                     prepared_metadata_source,
@@ -1075,8 +1022,6 @@ impl LocalDisk {
                 cleanup_data_dir,
                 version_signature,
                 old_current_size,
-                old_current_source_checked,
-                old_current_source,
                 mut local_rollback_path,
                 destination_was_absent,
                 prepared_metadata_source,
@@ -1202,11 +1147,10 @@ impl LocalDisk {
                     && let Some(dst_parent) = dst_file_path.parent()
                 {
                     let fsync_started = rustfs_io_metrics::put_stage_timer();
-                    if let Err(err) = os::fsync_commit_directory(
+                    if let Err(err) = os::fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
                         dst_parent,
-                        &commit.directory_guard,
                         commit.mutation_lease.clone(),
-                        Some(admission),
+                        admission,
                     )
                     .await
                     {
@@ -1318,8 +1262,6 @@ impl LocalDisk {
                 cleanup_data_dir,
                 sign: version_signature,
                 old_current_size,
-                old_current_source_checked,
-                old_current_source,
             })
         }
     }

crates/ecstore/src/disk/local/delete_recovery.rs

@@ -1953,7 +1953,10 @@ async fn barrier(volume: &str, object: &str, transaction: Uuid, stage: &str, dri
         .map_err(to_file_error)?
         == b"io_error"
     {
-        return Err(DiskError::Io(std::io::Error::other("injected delete recovery barrier failure")));
+        return Err(DiskError::Io(std::io::Error::new(
+            ErrorKind::Other,
+            "injected delete recovery barrier failure",
+        )));
     }
     Ok(())
 }

crates/ecstore/src/disk/mod.rs

@@ -14,7 +14,6 @@
 
 // #730: disk abstractions still carry staged health and direct-I/O migration paths.
 
-pub(crate) mod cleanup_runtime;
 pub mod disk_store;
 pub mod endpoint;
 pub mod error;
@@ -491,6 +490,13 @@ impl DiskAPI for Disk {
         }
     }
 
+    async fn bucket_creation_witness(&self, volume: &str) -> Result<Option<os::NativeBucketCreationWitness>> {
+        match self {
+            Disk::Local(local_disk) => local_disk.bucket_creation_witness(volume).await,
+            Disk::Remote(_) => Ok(None),
+        }
+    }
+
     #[tracing::instrument(level = "trace", skip_all)]
     async fn make_volume(&self, volume: &str) -> Result<()> {
         match self {
@@ -1285,6 +1291,11 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     // NSScanner
 
     // Volume operations.
+    /// Unknown or remote backends cannot attest process-local publication history.
+    async fn bucket_creation_witness(&self, _volume: &str) -> Result<Option<os::NativeBucketCreationWitness>> {
+        Ok(None)
+    }
+
     async fn make_volume(&self, volume: &str) -> Result<()>;
     async fn make_volumes(&self, volume: Vec<&str>) -> Result<()>;
     async fn list_volumes(&self) -> Result<Vec<VolumeInfo>>;
@@ -1777,14 +1788,6 @@ pub struct RenameDataResp {
     /// "cannot vote", never as `Absent`.
     #[serde(default)]
     pub old_current_size: Option<OldCurrentSize>,
-    /// Whether this disk decoded enough of the overwritten current version to
-    /// decide if a transitioned source has to be returned.
-    #[serde(default)]
-    pub old_current_source_checked: bool,
-    /// Transitioned old-current source used by PUT free-version cleanup receipt.
-    /// Empty when the old current is not transitioned or no source was known.
-    #[serde(default)]
-    pub old_current_source: Option<FileInfo>,
 }
 
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
@@ -1901,7 +1904,7 @@ impl DeleteRecoveryJournal {
             }
             let id = request.version_id.unwrap_or_default();
             let Some(original) = originals.get(&id) else {
-                return false;
+                return !id.is_nil() && !locals.contains_key(&id);
             };
             match locals.get(&id) {
                 Some(local) => local == original,
@@ -1951,7 +1954,10 @@ impl DeleteRecoveryJournal {
             };
             match (local, remote) {
                 (Some(local), Some(remote)) => local == remote,
-                (None, None) => self.original_digest.is_none() && authority.original_digest.is_none(),
+                (None, None) => {
+                    (!(request.deleted && request.mark_deleted) && request.version_id.is_some_and(|id| !id.is_nil()))
+                        || (self.original_digest.is_none() && authority.original_digest.is_none())
+                }
                 _ => false,
             }
         })
@@ -2526,8 +2532,6 @@ mod tests {
             cleanup_data_dir: Some(uuid),
             sign: Some(signature.clone()),
             old_current_size: Some(OldCurrentSize::Present(42)),
-            old_current_source_checked: false,
-            old_current_source: None,
         };
 
         assert_eq!(resp.old_data_dir, Some(uuid));
@@ -2548,8 +2552,6 @@ mod tests {
                 cleanup_data_dir: Some(Uuid::new_v4()),
                 sign: Some(vec![0x01, 0x02, 0x03]),
                 old_current_size,
-                old_current_source_checked: false,
-                old_current_source: None,
             };
 
             let encoded = rmp_serde::encode::to_vec_named(&resp).expect("named msgpack should encode");
@@ -2587,8 +2589,6 @@ mod tests {
         assert_eq!(decoded.cleanup_data_dir, None);
         assert_eq!(decoded.sign, legacy.sign);
         assert_eq!(decoded.old_current_size, None);
-        assert!(!decoded.old_current_source_checked);
-        assert!(decoded.old_current_source.is_none());
     }
 
     /// Test constants
@@ -2947,6 +2947,46 @@ mod delete_recovery_adoption_tests {
         };
         assert!(journal.matches_adoption_source(&[]), "certified non-null removal can already be absent");
         assert!(journal.matches_adoption_source(std::slice::from_ref(&source)));
+        let missing = Uuid::new_v4();
+        journal.requests.push(FileInfo {
+            version_id: Some(missing),
+            ..Default::default()
+        });
+        journal.original_digest = Some([3; 32]);
+        assert!(
+            journal.matches_prepared_authority(&journal),
+            "an explicit UUID absent from both prepared sources remains a no-op with original metadata"
+        );
+        assert!(journal.matches_adoption_source(&[]), "both non-null versions can remain absent");
+        assert!(
+            journal.matches_adoption_source(std::slice::from_ref(&source)),
+            "a missing non-null version in the batch remains a no-op"
+        );
+        let mut unexpected = source.clone();
+        unexpected.version_id = Some(missing);
+        let mut foreign_preparation = journal.clone();
+        foreign_preparation.original_versions.push(unexpected.clone());
+        assert!(
+            !journal.matches_prepared_authority(&foreign_preparation)
+                && !foreign_preparation.matches_prepared_authority(&journal),
+            "absence from only one prepared source cannot establish a generation"
+        );
+        assert!(
+            !journal.matches_adoption_source(&[source.clone(), unexpected]),
+            "a version absent from the authority cannot be deleted locally"
+        );
+        for version_id in [None, Some(Uuid::nil())] {
+            journal.requests.last_mut().expect("missing-version request").version_id = version_id;
+            assert!(
+                !journal.matches_prepared_authority(&journal),
+                "a missing null version with original metadata cannot establish its generation"
+            );
+            assert!(
+                !journal.matches_adoption_source(std::slice::from_ref(&source)),
+                "a missing null version cannot establish its generation"
+            );
+        }
+        journal.requests.pop();
         let mut foreign = source.clone();
         foreign.metadata_digest = [2; 32];
         assert!(!journal.matches_adoption_source(std::slice::from_ref(&foreign)));

crates/ecstore/src/error/mod.rs

@@ -17,8 +17,8 @@
 use crate::bucket::error::BucketMetadataError;
 use crate::disk::error::DiskError;
 use crate::storage_api_contracts::{error::StorageErrorCode, range::HTTPRangeError};
+use rustfs_s3_types::S3ErrorCode;
 use rustfs_utils::path::decode_dir_object;
-use s3s::S3ErrorCode;
 
 pub type Error = StorageError;
 pub type Result<T> = core::result::Result<T, Error>;

crates/ecstore/src/object_api/object_api_utils.rs

@@ -29,7 +29,7 @@ use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
 use crate::storage_api_contracts::range::HTTPRangeSpec;
 use rustfs_filemeta::ObjectPartInfo;
 use rustfs_rio::HashReader;
-use s3s::S3ErrorCode;
+use rustfs_s3_types::S3ErrorCode;
 
 //#[derive(Clone)]
 pub struct PutObjReader {
@@ -231,6 +231,45 @@ mod tests {
         assert_eq!(result.2, 1024 * 1024);
     }
 
+    #[test]
+    fn test_new_getobjectreader_rejects_a_range_past_the_object_as_invalid_range() {
+        let oi = multipart_object_info();
+        let past_the_end = Some(HTTPRangeSpec {
+            is_suffix_length: false,
+            start: oi.size,
+            end: oi.size + 10,
+        });
+
+        let err = new_getobjectreader(&past_the_end, &oi, &ObjectOptions::default(), &HeaderMap::new())
+            .err()
+            .expect("a range starting at the object size must be rejected");
+
+        assert_eq!(err.code, S3ErrorCode::InvalidRange);
+        assert_eq!(err.code.as_str(), "InvalidRange");
+        assert_eq!(err.key, None);
+        assert_eq!(err.bucket_name, None);
+    }
+
+    #[test]
+    fn test_new_getobjectreader_rejects_an_unknown_size_without_a_range_as_invalid_range() {
+        let oi = ObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "object".to_string(),
+            size: -1,
+            ..Default::default()
+        };
+
+        let err = new_getobjectreader(&None, &oi, &ObjectOptions::default(), &HeaderMap::new())
+            .err()
+            .expect("an object of unknown size cannot be read without a range");
+
+        assert_eq!(err.code, S3ErrorCode::InvalidRange);
+        assert_eq!(err.code.as_str(), "InvalidRange");
+        assert_eq!(err.message, "Invalid range");
+        assert_eq!(err.key.as_deref(), Some("object"));
+        assert_eq!(err.bucket_name.as_deref(), Some("bucket"));
+    }
+
     #[test]
     fn test_to_s3s_etag() {
         // Test unquoted ETag - should become strong etag

crates/ecstore/src/object_api/types.rs

@@ -1006,9 +1006,6 @@ pub struct ObjectOptions {
     /// Proves an upper layer holds the bucket lifecycle sentinel. A separate
     /// fence avoids recursively acquiring the read lock behind a queued writer.
     pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>,
-    /// Retains the actual lifecycle guard for an owned multipart commit. A
-    /// lock-loss fence alone does not keep a cancelled caller's lock alive.
-    pub bucket_lifecycle_lock_owner: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
     pub replication_request: bool,
     /// True when the inbound request carried the
     /// `{x-rustfs-,x-minio-}source-proxy-request` header family with the
@@ -1168,7 +1165,6 @@ impl std::fmt::Debug for ObjectOptions {
             .field("delete_mutation_owner", &self.delete_mutation_owner.is_some())
             .field("namespace_lock_fence", &self.namespace_lock_fence.is_some())
             .field("bucket_lifecycle_lock_fence", &self.bucket_lifecycle_lock_fence.is_some())
-            .field("bucket_lifecycle_lock_owner", &self.bucket_lifecycle_lock_owner.is_some())
             .field("replication_request", &self.replication_request)
             .field("proxy_request", &self.proxy_request)
             .field("proxy_header_set", &self.proxy_header_set)

crates/ecstore/src/services/rebalance/control.rs

@@ -18,8 +18,8 @@ use super::{
     encode_rebalance_stop_propagation_record,
 };
 use crate::core::pools::{
-    PoolMeta, PoolRebalanceActivationFence, PoolRebalanceWorkerFence, acquire_pool_activation_fleet_proof,
-    acquire_pool_rebalance_activation_locks, acquire_pool_rebalance_worker_locks, pool_meta_has_active_decommission,
+    PoolMeta, PoolRebalanceActivationFence, acquire_pool_activation_fleet_proof, acquire_pool_rebalance_activation_locks,
+    pool_meta_has_active_decommission,
 };
 use crate::error::{Error, Result};
 use crate::object_api::ObjectOptions;
@@ -77,33 +77,10 @@ fn ensure_rebalance_activation_pool_meta_allowed(meta: &PoolMeta) -> Result<()>
 }
 
 pub(super) enum RebalanceWorkerActivationFence {
-    Ready(Box<PoolRebalanceWorkerFence>),
+    Ready(Box<PoolRebalanceActivationFence>),
     NotStartedTerminal,
 }
 
-async fn persisted_rebalance_requires_worker_activation<S>(pool: Arc<S>, expected_id: &str) -> Result<bool>
-where
-    S: EcstoreObjectIO,
-{
-    let mut persisted = RebalanceMeta::new();
-    persisted
-        .load_with_opts(
-            pool,
-            ObjectOptions {
-                no_lock: true,
-                ..Default::default()
-            },
-        )
-        .await?;
-    if persisted.id != expected_id {
-        return Err(Error::other(format!(
-            "rebalance metadata changed before worker activation: expected {expected_id}, found {}",
-            persisted.id
-        )));
-    }
-    Ok(crate::services::rebalance::rebalance_requires_worker_activation(&persisted))
-}
-
 pub(super) struct RebalanceRunGuard {
     _local_guard: tokio::sync::OwnedRwLockReadGuard<()>,
     persisted_guard: rustfs_lock::NamespaceLockGuard,
@@ -600,58 +577,48 @@ impl ECStore {
     where
         S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
     {
-        // Lock order: pool_meta_save_gate -> rebalance.bin -> pool.bin. These
-        // shared fences admit a peer without waiting for the run's own entries.
+        // Lock order: pool_meta_save_gate -> rebalance.bin -> pool.bin.
         let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
         pool_meta_guard.ensure_write_safe("rebalance worker activation")?;
-        let mut worker_fence = acquire_pool_rebalance_worker_locks(pool.clone()).await?;
+        // Classify the durable rebalance record while holding both namespace
+        // fences. A terminal record is a no-op and must not depend on the
+        // notification subsystem having published a fleet proof yet.
+        let mut activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), None).await?;
         let pool_meta = self
-            .load_runtime_pool_meta_under_worker_fence(&mut pool_meta_guard, &worker_fence, "rebalance worker activation")
+            .load_runtime_pool_meta_under_activation_fence(&mut pool_meta_guard, &activation_fence, "rebalance worker activation")
             .await?;
         ensure_rebalance_activation_pool_meta_allowed(&pool_meta)?;
-        let needs_activation = persisted_rebalance_requires_worker_activation(pool, expected_id).await?;
-        worker_fence.ensure_held()?;
-        if !needs_activation {
+
+        let mut persisted = RebalanceMeta::new();
+        persisted
+            .load_with_opts(
+                pool,
+                ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await?;
+        if persisted.id != expected_id {
+            return Err(Error::other(format!(
+                "rebalance metadata changed before worker activation: expected {expected_id}, found {}",
+                persisted.id
+            )));
+        }
+
+        activation_fence.ensure_held()?;
+        if !crate::services::rebalance::rebalance_requires_worker_activation(&persisted) {
             return Ok(RebalanceWorkerActivationFence::NotStartedTerminal);
         }
 
-        let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
-        worker_fence.set_fleet_proof(fleet_proof);
-        worker_fence.ensure_held()?;
-        Ok(RebalanceWorkerActivationFence::Ready(Box::new(worker_fence)))
-    }
-
-    pub(super) async fn fence_rebalance_worker_metadata_update<S>(
-        &self,
-        pool: Arc<S>,
-        expected_id: &str,
-    ) -> Result<Option<Box<PoolRebalanceActivationFence>>>
-    where
-        S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
-    {
-        // The admission read fences must already be dropped. Reacquire in the
-        // original order and revalidate every durable boundary across that gap.
-        let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
-        pool_meta_guard.ensure_write_safe("rebalance worker metadata update")?;
-        let mut activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), None).await?;
-        let pool_meta = self
-            .load_runtime_pool_meta_under_activation_fence(
-                &mut pool_meta_guard,
-                &activation_fence,
-                "rebalance worker metadata update",
-            )
-            .await?;
-        ensure_rebalance_activation_pool_meta_allowed(&pool_meta)?;
-        let needs_activation = persisted_rebalance_requires_worker_activation(pool, expected_id).await?;
-        activation_fence.ensure_held()?;
-        if !needs_activation {
-            return Ok(None);
-        }
-
+        // Active worker admission still requires the fail-closed fleet proof.
+        // Attach it immediately before the final fence validation so expiry or
+        // topology changes are checked again at every later commit boundary.
         let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
         activation_fence.set_fleet_proof(fleet_proof);
         activation_fence.ensure_held()?;
-        Ok(Some(Box::new(activation_fence)))
+
+        Ok(RebalanceWorkerActivationFence::Ready(Box::new(activation_fence)))
     }
 
     #[tracing::instrument(skip_all)]
@@ -1849,107 +1816,6 @@ mod tests {
         assert_activation_locks_released(&store).await;
     }
 
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn rebalance_init_missing_fleet_proof_preserves_metadata_and_rechecks_competing_run() {
-        let previous_id = "completed-before-capability-refresh";
-        let terminal = RebalanceMeta {
-            id: previous_id.to_string(),
-            stopped_at: Some(OffsetDateTime::now_utc()),
-            pool_stats: vec![RebalanceStats::default(); 2],
-            ..Default::default()
-        };
-        let (_temp_dirs, store, competing_store) = crate::services::rebalance::test_two_pool_stores(Some(terminal.clone())).await;
-        let disk_stats = vec![
-            DiskStat {
-                total_space: 100,
-                available_space: 0,
-            },
-            DiskStat {
-                total_space: 100,
-                available_space: 100,
-            },
-        ];
-        let initial_epoch = store.ctx.data_movement_operation_epoch();
-        let proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
-        set_rebalance_disk_stats_override_for_test(store.id, disk_stats.clone());
-        let err = store
-            .init_rebalance_start(vec!["waiting-request".to_string()])
-            .await
-            .expect_err("initialization must fail closed while the live proof is missing");
-        assert!(
-            err.to_string()
-                .contains("pool activation requires a live fleet capability proof")
-        );
-        assert!(
-            crate::core::pools::is_pool_activation_fleet_proof_error(&err),
-            "the admin handler retries init only on this error shape: {err}"
-        );
-        assert_eq!(store.ctx.data_movement_operation_epoch(), initial_epoch);
-        {
-            let local = store.rebalance_meta.read().await;
-            let local = local.as_ref().expect("failed initialization must preserve the previous run");
-            assert_eq!(local.id, previous_id);
-            assert_eq!(local.stopped_at, terminal.stopped_at);
-            assert!(local.cancel.is_none(), "failed initialization must not start a worker");
-        }
-        let mut persisted = RebalanceMeta::new();
-        persisted
-            .load(store.pools[0].clone())
-            .await
-            .expect("failed initialization must leave durable metadata readable");
-        assert_eq!(persisted.id, previous_id);
-        assert_eq!(persisted.stopped_at, terminal.stopped_at);
-
-        drop(proof_guard);
-        set_rebalance_disk_stats_override_for_test(competing_store.id, disk_stats.clone());
-        let winning_id = competing_store
-            .init_rebalance_start(vec!["winning-request".to_string()])
-            .await
-            .expect("a ready request must be able to initialize one run after the failed attempt");
-        assert_ne!(winning_id, previous_id);
-
-        // The waiting request still holds its old local snapshot. Its retry must
-        // recheck durable state under the activation locks before publishing.
-        set_rebalance_disk_stats_override_for_test(store.id, disk_stats);
-        let remote_conflict = store
-            .init_rebalance_start(vec!["waiting-request".to_string()])
-            .await
-            .expect_err("the waiting request must not replace a durable competing run");
-        assert!(
-            matches!(
-                crate::data_movement::data_movement_stage_source(&remote_conflict),
-                Some(Error::RebalanceAlreadyRunning)
-            ),
-            "the save-stage error must retain the typed competing-run conflict: {remote_conflict:?}"
-        );
-        let local_conflict = competing_store
-            .init_rebalance_start(vec!["another-request".to_string()])
-            .await
-            .expect_err("a local retry must not replace its active run");
-        assert!(
-            matches!(local_conflict, Error::RebalanceAlreadyRunning),
-            "the local admission check must report the competing-run conflict: {local_conflict:?}"
-        );
-        persisted
-            .load(store.pools[0].clone())
-            .await
-            .expect("the competing run must remain durable after both rejected retries");
-        assert_eq!(persisted.id, winning_id);
-        assert_eq!(persisted.pool_stats[0].buckets, vec!["winning-request".to_string()]);
-        assert_eq!(
-            store
-                .rebalance_meta
-                .read()
-                .await
-                .as_ref()
-                .expect("old local run must survive")
-                .id,
-            previous_id
-        );
-        assert_eq!(store.ctx.data_movement_operation_epoch(), initial_epoch + 1);
-    }
-
     #[tokio::test]
     #[serial_test::serial]
     async fn rebalance_worker_skips_terminal_metadata_without_fleet_proof() {
@@ -2008,128 +1874,6 @@ mod tests {
         );
     }
 
-    async fn assert_worker_write_upgrade_revalidates_durable_state(change: &str) {
-        let rebalance_id = "rebalance-worker-write-upgrade";
-        let active = RebalanceMeta {
-            id: rebalance_id.to_string(),
-            percent_free_goal: 0.5,
-            pool_stats: vec![
-                RebalanceStats {
-                    participating: true,
-                    init_capacity: 100,
-                    info: RebalanceInfo {
-                        status: RebalStatus::Started,
-                        ..Default::default()
-                    },
-                    ..Default::default()
-                },
-                RebalanceStats::default(),
-            ],
-            ..Default::default()
-        };
-        let (_temp_dirs, store, peer) =
-            crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(Some(active.clone())).await;
-        let arrived = Arc::new(tokio::sync::Notify::new());
-        let release = Arc::new(tokio::sync::Notify::new());
-        let worker = Arc::clone(&store);
-        let mut tasks = tokio::task::JoinSet::new();
-        tasks.spawn(
-            super::super::runtime::REBALANCE_WORKER_WRITE_UPGRADE_BARRIER
-                .scope((Arc::clone(&arrived), Arc::clone(&release)), async move {
-                    worker.start_rebalance_for_id(rebalance_id).await
-                }),
-        );
-        tokio::time::timeout(std::time::Duration::from_secs(30), arrived.notified())
-            .await
-            .expect("empty-queue activation must release its read fences before requesting write authority");
-
-        let mut changed = active.clone();
-        match change {
-            "stopped" => {
-                changed.stopped_at = Some(OffsetDateTime::now_utc());
-                changed.pool_stats[0].info.status = RebalStatus::Stopped;
-            }
-            "replaced" => changed.id = "replacement-rebalance-run".to_string(),
-            "decommission" => {
-                let mut pool_meta = peer.pool_meta.read().await.clone();
-                pool_meta.pools[0].decommission = Some(crate::core::pools::PoolDecommissionInfo {
-                    start_time: Some(OffsetDateTime::now_utc()),
-                    ..Default::default()
-                });
-                pool_meta
-                    .save(peer.pools.clone())
-                    .await
-                    .expect("a competing pool metadata writer must acquire the released pool fence");
-            }
-            _ => panic!("unknown durable state transition"),
-        }
-        changed
-            .save_with_opts(
-                peer.pools[0].clone(),
-                ObjectOptions {
-                    write_completion: crate::object_api::WriteCompletion::TailDrained,
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("a competing run writer must acquire the released rebalance fence");
-        release.notify_one();
-        let result = tokio::time::timeout(std::time::Duration::from_secs(30), tasks.join_next())
-            .await
-            .expect("activation must finish after the durable transition")
-            .expect("activation task must be tracked")
-            .expect("activation must not panic");
-        match change {
-            "stopped" => result.expect("a durably stopped run must become an activation no-op"),
-            "replaced" => assert!(
-                result
-                    .expect_err("a replaced run must reject stale activation")
-                    .to_string()
-                    .contains("expected"),
-            ),
-            "decommission" => assert!(matches!(result, Err(Error::DecommissionAlreadyRunning))),
-            _ => unreachable!(),
-        }
-        let mut persisted = RebalanceMeta::new();
-        persisted
-            .load(peer.pools[0].clone())
-            .await
-            .expect("the competing durable state must remain readable");
-        assert_eq!(persisted.id, changed.id, "activation must not overwrite the run identity");
-        assert_eq!(persisted.stopped_at, changed.stopped_at, "activation must not overwrite the stop marker");
-        assert_eq!(persisted.pool_stats[0].info.status, changed.pool_stats[0].info.status);
-        assert!(
-            store
-                .rebalance_meta
-                .read()
-                .await
-                .as_ref()
-                .expect("local metadata must remain present")
-                .cancel
-                .is_none(),
-            "rejected activation must not install a worker token",
-        );
-        assert_activation_locks_released(&store).await;
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn rebalance_worker_write_upgrade_rechecks_stop() {
-        assert_worker_write_upgrade_revalidates_durable_state("stopped").await;
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn rebalance_worker_write_upgrade_rechecks_run_identity() {
-        assert_worker_write_upgrade_revalidates_durable_state("replaced").await;
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn rebalance_worker_write_upgrade_rechecks_decommission() {
-        assert_worker_write_upgrade_revalidates_durable_state("decommission").await;
-    }
-
     #[tokio::test]
     #[serial_test::serial]
     async fn rebalance_activation_adopts_commit_after_post_save_fence_loss() {

crates/ecstore/src/services/rebalance/entry.rs

@@ -1440,7 +1440,7 @@ mod tests {
 
     #[tokio::test]
     #[serial_test::serial]
-    async fn real_rebalance_peer_activates_while_entry_retains_run_fence() {
+    async fn real_rebalance_entry_progresses_while_peer_activation_waits_for_run_fence() {
         const REBALANCE_ID: &str = "rebalance-peer-activation-lock-order";
         let (_temp_dirs, store, peer) = crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(Some(
             active_rebalance_meta(REBALANCE_ID),
@@ -1456,7 +1456,7 @@ mod tests {
         let bucket = crate::disk::RUSTFS_META_BUCKET;
         let object = "rebalance-peer-activation-object";
         let version_id = uuid::Uuid::new_v4();
-        let payload = b"peer activation must not wait for this entry to drain".repeat(1024);
+        let payload = b"entry must drain before peer activation takes the pool fence".repeat(1024);
         let source_set = store.pools[0].get_disks_by_key(object);
         let target_set = store.pools[1].get_disks_by_key(object);
         let opts = ObjectOptions {
@@ -1516,12 +1516,7 @@ mod tests {
         );
         tokio::time::timeout(StdDuration::from_secs(30), attempted.notified())
             .await
-            .expect("peer activation must attempt the persisted rebalance fence");
-        tokio::time::timeout(StdDuration::from_secs(30), activation_result)
-            .await
-            .expect("peer activation must finish while the entry remains paused")
-            .expect("peer activation result should be sent")
-            .expect("joining the same run must not require its active entry to release the run fence");
+            .expect("peer activation must attempt the persisted rebalance write fence");
         release.notify_one();
 
         tokio::time::timeout(StdDuration::from_secs(30), async {
@@ -1536,6 +1531,10 @@ mod tests {
         })
         .await
         .expect("entry and peer activation must both make progress");
+        activation_result
+            .await
+            .expect("peer activation result should be sent")
+            .expect("peer activation must not time out behind the entry it blocks");
 
         let mut reader = target_set
             .get_object_reader(bucket, object, None, HeaderMap::new(), &opts)

crates/ecstore/src/services/rebalance/runtime.rs

@@ -27,11 +27,6 @@ use tokio::time::{Duration, Instant};
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, warn};
 
-#[cfg(test)]
-tokio::task_local! {
-    pub(super) static REBALANCE_WORKER_WRITE_UPGRADE_BARRIER: (Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>);
-}
-
 pub(super) fn should_fail_repeated_rebalance_bucket_defer(
     deferred_buckets: &mut HashSet<String>,
     bucket: &str,
@@ -218,22 +213,6 @@ impl ECStore {
         }
 
         if must_persist {
-            // Shared admission grants no write authority. Release it before
-            // taking the exclusive metadata fences, then recheck stop/run/pool
-            // state before publishing the locally staged terminal candidate.
-            drop(activation_fence);
-            #[cfg(test)]
-            if let Ok((arrived, release)) = REBALANCE_WORKER_WRITE_UPGRADE_BARRIER.try_with(Clone::clone) {
-                arrived.notify_one();
-                release.notified().await;
-            }
-            let Some(activation_fence) = self
-                .fence_rebalance_worker_metadata_update(pool.clone(), expected_id.as_ref())
-                .await?
-            else {
-                cancel_tx.cancel();
-                return Ok(false);
-            };
             let save_result = resolve_rebalance_meta_save_result(
                 self.save_rebalance_meta_under_activation_fence(
                     pool,
@@ -263,14 +242,13 @@ impl ECStore {
                 cancel_tx.cancel();
                 return Err(err);
             }
-        } else {
-            if let Err(err) = activation_fence.ensure_held() {
-                let mut rebalance_meta = self.rebalance_meta.write().await;
-                rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
-                return Err(err);
-            }
-            drop(activation_fence);
         }
+        if !must_persist && let Err(err) = activation_fence.ensure_held() {
+            let mut rebalance_meta = self.rebalance_meta.write().await;
+            rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
+            return Err(err);
+        }
+        drop(activation_fence);
 
         if activation_outcome != RebalanceLocalActivationOutcome::Started {
             return Ok(must_persist);

crates/ecstore/src/services/tier/tier.rs

@@ -82,8 +82,9 @@ use crate::{
 use rustfs_filemeta::FileInfo;
 use rustfs_rio::HashReader;
 use rustfs_s3_client::{admin_handler_utils::AdminError, provider_versions::ProviderVersionCapabilities};
+use rustfs_s3_types::S3ErrorCode;
 use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
-use s3s::{S3ErrorCode, dto::BucketLifecycleConfiguration};
+use s3s::dto::BucketLifecycleConfiguration;
 
 use super::{
     tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_PERM_ERR},

crates/ecstore/src/set_disk/core/metadata_read.rs

@@ -576,7 +576,6 @@ impl MetadataFanoutDiagnostics {
 
     pub(in crate::set_disk) fn record_quorum_candidate_latency(&self, path: &'static str, read_quorum: usize) {
         if let Some(latency) = self.quorum_candidate_latency(read_quorum) {
-            rustfs_io_metrics::record_get_object_metadata_quorum_result(path, "reached");
             rustfs_io_metrics::record_get_object_quorum_reached_latency(path, latency.as_secs_f64());
         }
     }

crates/ecstore/src/set_disk/mod.rs

@@ -166,7 +166,7 @@ use sha2::Sha256;
 use std::hash::{BuildHasher, Hash, Hasher};
 use std::mem::{self};
 use std::pin::Pin;
-use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::{Arc, OnceLock};
 use std::task::{Context, Poll};
 use std::time::{Instant, SystemTime, UNIX_EPOCH};
@@ -410,12 +410,6 @@ impl Drop for ObjectLockDiagGuard {
         }
 
         let hold = self.acquired_at.elapsed();
-        if self.op == "put_object_commit" && self.mode == "write" && rustfs_io_metrics::put_stage_metrics_enabled() {
-            rustfs_io_metrics::record_put_object_stage_duration(
-                rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD,
-                hold.as_secs_f64() * 1000.0,
-            );
-        }
         record_object_lock_diag_hold_duration(self.op, self.mode, hold);
         let threshold = get_object_lock_diag_slow_hold_threshold();
         if hold >= threshold {
@@ -931,21 +925,16 @@ pub(in crate::set_disk) struct GetObjectFileInfo {
     owned: Option<OwnedGetObjectFileInfo>,
     shared: Option<Arc<GetObjectMetadataCacheEntry>>,
     condition_stopped: bool,
-    late_metadata_fanout: Option<LateMetadataFanout>,
 }
 
 struct OwnedGetObjectFileInfo {
     fi: FileInfo,
     parts_metadata: Vec<FileInfo>,
     online_disks: Vec<Option<DiskStore>>,
+    late_metadata_fanout_disks: Option<Vec<Option<DiskStore>>>,
 }
 
-enum LateMetadataFanout {
-    CapturedDisks(Vec<Option<DiskStore>>),
-    RefreshCurrentSet,
-}
-
-type OwnedGetObjectFileInfoParts = (FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>, Option<LateMetadataFanout>);
+type OwnedGetObjectFileInfoParts = (FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>, Option<Vec<Option<DiskStore>>>);
 
 impl GetObjectFileInfo {
     fn owned(fi: FileInfo, parts_metadata: Vec<FileInfo>, online_disks: Vec<Option<DiskStore>>) -> Self {
@@ -954,10 +943,10 @@ impl GetObjectFileInfo {
                 fi,
                 parts_metadata,
                 online_disks,
+                late_metadata_fanout_disks: None,
             }),
             shared: None,
             condition_stopped: false,
-            late_metadata_fanout: None,
         }
     }
 
@@ -967,9 +956,16 @@ impl GetObjectFileInfo {
         online_disks: Vec<Option<DiskStore>>,
         late_metadata_fanout_disks: Vec<Option<DiskStore>>,
     ) -> Self {
-        let mut snapshot = Self::owned(fi, parts_metadata, online_disks);
-        snapshot.late_metadata_fanout = Some(LateMetadataFanout::CapturedDisks(late_metadata_fanout_disks));
-        snapshot
+        Self {
+            owned: Some(OwnedGetObjectFileInfo {
+                fi,
+                parts_metadata,
+                online_disks,
+                late_metadata_fanout_disks: Some(late_metadata_fanout_disks),
+            }),
+            shared: None,
+            condition_stopped: false,
+        }
     }
 
     fn shared(entry: Arc<GetObjectMetadataCacheEntry>) -> Self {
@@ -978,16 +974,9 @@ impl GetObjectFileInfo {
             owned: None,
             shared: Some(entry),
             condition_stopped,
-            late_metadata_fanout: None,
         }
     }
 
-    fn shared_with_late_metadata_refresh(entry: Arc<GetObjectMetadataCacheEntry>) -> Self {
-        let mut snapshot = Self::shared(entry);
-        snapshot.late_metadata_fanout = Some(LateMetadataFanout::RefreshCurrentSet);
-        snapshot
-    }
-
     fn fi(&self) -> &FileInfo {
         match (&self.owned, &self.shared) {
             (Some(snapshot), None) => &snapshot.fi,
@@ -1013,7 +1002,9 @@ impl GetObjectFileInfo {
     }
 
     fn has_late_metadata_fanout(&self) -> bool {
-        self.late_metadata_fanout.is_some()
+        self.owned
+            .as_ref()
+            .is_some_and(|snapshot| snapshot.late_metadata_fanout_disks.is_some())
     }
 
     fn into_owned(self) -> (FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>) {
@@ -1022,22 +1013,16 @@ impl GetObjectFileInfo {
     }
 
     fn into_owned_with_late_metadata_fanout(self) -> OwnedGetObjectFileInfoParts {
-        let Self {
-            owned,
-            shared,
-            late_metadata_fanout,
-            ..
-        } = self;
-        match (owned, shared) {
-            (Some(snapshot), None) => (snapshot.fi, snapshot.parts_metadata, snapshot.online_disks, late_metadata_fanout),
+        match (self.owned, self.shared) {
+            (Some(snapshot), None) => (
+                snapshot.fi,
+                snapshot.parts_metadata,
+                snapshot.online_disks,
+                snapshot.late_metadata_fanout_disks,
+            ),
             (None, Some(entry)) => match Arc::try_unwrap(entry) {
-                Ok(entry) => (entry.fi, entry.parts_metadata, entry.online_disks, late_metadata_fanout),
-                Err(entry) => (
-                    entry.fi.clone(),
-                    entry.parts_metadata.clone(),
-                    entry.online_disks.clone(),
-                    late_metadata_fanout,
-                ),
+                Ok(entry) => (entry.fi, entry.parts_metadata, entry.online_disks, None),
+                Err(entry) => (entry.fi.clone(), entry.parts_metadata.clone(), entry.online_disks.clone(), None),
             },
             _ => unreachable!("GET metadata snapshot representation must be exclusive"),
         }
@@ -1428,11 +1413,11 @@ mod prepared_get_object_metadata_tests {
         let snapshot = GetObjectFileInfo::shared(Arc::clone(&cached));
 
         assert!(snapshot.has_valid_representation());
-        // Keep the original payload budget after moving the captured disks out
-        // of OwnedGetObjectFileInfo.
-        let owned_payload_size = std::mem::size_of::<(OwnedGetObjectFileInfo, Option<Vec<Option<DiskStore>>>)>();
-        assert!(std::mem::size_of::<GetObjectFileInfo>() >= owned_payload_size);
-        assert!(std::mem::size_of::<GetObjectFileInfo>() <= owned_payload_size + 2 * std::mem::size_of::<usize>());
+        assert!(std::mem::size_of::<GetObjectFileInfo>() >= std::mem::size_of::<OwnedGetObjectFileInfo>());
+        assert!(
+            std::mem::size_of::<GetObjectFileInfo>()
+                <= std::mem::size_of::<OwnedGetObjectFileInfo>() + 2 * std::mem::size_of::<usize>()
+        );
         assert_eq!(Arc::strong_count(&cached), 2, "a cache hit must add one snapshot reference");
         assert_eq!(snapshot.fi().name, "object");
         assert_eq!(snapshot.parts_metadata().len(), 1);
@@ -4352,9 +4337,6 @@ pub struct SetDisks {
     /// writes skip the global registry mutex (backlog#1315). `Arc` so clones of
     /// a set share one generation marker.
     capacity_dirty_generation: Arc<AtomicU64>,
-    /// Per-set proof that successful rename_data peers return the old-current
-    /// source capability needed before PUT may skip its pre-rename lookup.
-    tier_free_version_rename_data_source_capable: Arc<AtomicBool>,
     /// Orphan prefixes whose last purge scan met data that can never be
     /// purged by listing (residue without a committed marker, an in-flight
     /// write), keyed by `bucket/prefix` with the time of that scan. Empty
@@ -4610,14 +4592,6 @@ struct GetObjectMetadataCacheKey {
     hash: u64,
 }
 
-#[derive(Clone, Copy, Debug)]
-struct GetObjectMetadataCacheLookupKey<'a> {
-    bucket: &'a str,
-    object: &'a str,
-    generation: u64,
-    hash: u64,
-}
-
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 struct GetObjectMetadataCacheGeneration {
     index: usize,
@@ -4706,33 +4680,6 @@ impl GetObjectMetadataCacheKey {
     }
 }
 
-impl<'a> GetObjectMetadataCacheLookupKey<'a> {
-    fn new(bucket: &'a str, object: &'a str, generation: GetObjectMetadataCacheGeneration) -> Self {
-        Self {
-            bucket,
-            object,
-            generation: generation.value,
-            hash: generation.hash,
-        }
-    }
-}
-
-impl Hash for GetObjectMetadataCacheLookupKey<'_> {
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        self.hash.hash(state);
-        self.generation.hash(state);
-    }
-}
-
-impl moka::Equivalent<GetObjectMetadataCacheKey> for GetObjectMetadataCacheLookupKey<'_> {
-    fn equivalent(&self, key: &GetObjectMetadataCacheKey) -> bool {
-        self.bucket == key.bucket.as_ref()
-            && self.object == key.object.as_ref()
-            && self.generation == key.generation
-            && self.hash == key.hash
-    }
-}
-
 impl Hash for GetObjectMetadataCacheKey {
     fn hash<H: Hasher>(&self, state: &mut H) {
         self.hash.hash(state);
@@ -4743,9 +4690,7 @@ impl Hash for GetObjectMetadataCacheKey {
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 enum GetObjectMetadataCacheCapability {
     Full,
-    // A quorum-selected layout can serve body reads, but is not a mutation layout.
-    QuorumSelected,
-    // Quorum-validated metadata used only to evaluate a terminal read condition.
+    // Quorum-validated metadata without a complete body or write-target layout.
     ConditionOnly,
 }
 
@@ -5288,7 +5233,6 @@ impl SetDisks {
             ctx,
             capacity_scope_cache: Arc::new(std::sync::RwLock::new(CapacityScopeCache::default())),
             capacity_dirty_generation: Arc::new(AtomicU64::new(u64::MAX)),
-            tier_free_version_rename_data_source_capable: Arc::new(AtomicBool::new(false)),
             orphan_purge_backoff: Arc::new(std::sync::Mutex::new(HashMap::new())),
             #[cfg(test)]
             storage_class_config_override: Arc::new(std::sync::RwLock::new(None)),
@@ -5303,15 +5247,6 @@ impl SetDisks {
         &self.ctx
     }
 
-    pub(in crate::set_disk) fn tier_free_version_rename_data_source_capable(&self) -> bool {
-        self.tier_free_version_rename_data_source_capable.load(Ordering::Acquire)
-    }
-
-    pub(in crate::set_disk) fn observe_tier_free_version_rename_data_source_capability(&self, capable: bool) {
-        self.tier_free_version_rename_data_source_capable
-            .store(capable, Ordering::Release);
-    }
-
     #[cfg(test)]
     pub(crate) fn set_instance_ctx_for_test(&mut self, ctx: Arc<InstanceContext>) {
         self.local_lock_manager = ctx.lock_manager();

crates/ecstore/src/set_disk/ops/bucket.rs

@@ -31,7 +31,6 @@ use crate::disk::DiskAPI;
 pub(crate) enum BucketInfoQuorum {
     Read,
     Write,
-    HealCandidates,
 }
 
 impl SetDisks {
@@ -105,9 +104,7 @@ impl SetDisks {
             // class. A namespace read must intersect that majority; object
             // readers still enforce the persisted layout's data-shard quorum.
             BucketInfoQuorum::Read => reduce_read_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, disk_count.div_ceil(2).max(1)),
-            BucketInfoQuorum::Write | BucketInfoQuorum::HealCandidates => {
-                reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, disk_count / 2 + 1)
-            }
+            BucketInfoQuorum::Write => reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, disk_count / 2 + 1),
         };
         if let Some(err) = error {
             return Err(err.into());
@@ -129,29 +126,6 @@ impl SetDisks {
         self.list_bucket_with_quorum(BucketInfoQuorum::Write).await
     }
 
-    pub(crate) async fn bucket_needs_reconstruction(&self, bucket: &str) -> Result<bool> {
-        let results = join_all(self.disk_inventory().await.into_iter().map(|disk| async move {
-            match disk {
-                Some(disk) => disk.stat_volume(bucket).await,
-                None => Err(DiskError::DiskNotFound),
-            }
-        }))
-        .await;
-        let mut present = false;
-        let mut missing = false;
-        for result in results {
-            match result {
-                Ok(_) => present = true,
-                Err(DiskError::VolumeNotFound) => missing = true,
-                Err(error) => return Err(error.into()),
-            }
-        }
-        if !present {
-            return Err(Error::VolumeNotFound);
-        }
-        Ok(missing)
-    }
-
     pub(crate) async fn list_bucket_with_quorum(&self, quorum: BucketInfoQuorum) -> Result<(Vec<BucketInfo>, bool)> {
         let disks = self.disk_inventory().await;
         let write_quorum = (disks.len() / 2) + 1;
@@ -160,7 +134,6 @@ impl SetDisks {
         let required = match quorum {
             BucketInfoQuorum::Read => disks.len().div_ceil(2).max(1),
             BucketInfoQuorum::Write => write_quorum,
-            BucketInfoQuorum::HealCandidates => disks.len().max(1),
         };
 
         let mut futures = Vec::with_capacity(disks.len());
@@ -192,9 +165,7 @@ impl SetDisks {
 
         let error = match quorum {
             BucketInfoQuorum::Read => reduce_read_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, required),
-            BucketInfoQuorum::Write | BucketInfoQuorum::HealCandidates => {
-                reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, required)
-            }
+            BucketInfoQuorum::Write => reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, required),
         };
         if let Some(err) = error {
             return Err(err.into());
@@ -219,17 +190,10 @@ impl SetDisks {
             }
         }
 
-        let minimum_presence = if matches!(quorum, BucketInfoQuorum::HealCandidates) {
-            // Candidate presence is not mutation authority. Every disk must answer,
-            // and the heal owner validates the committed generation before recreation.
-            1
-        } else {
-            topology_complete &= counts.values().all(|(count, _)| *count >= write_quorum);
-            required
-        };
+        topology_complete &= counts.values().all(|(count, _)| *count >= write_quorum);
         let mut buckets = counts
             .into_values()
-            .filter_map(|(count, bucket)| (count >= minimum_presence).then_some(bucket))
+            .filter_map(|(count, bucket)| (count >= required).then_some(bucket))
             .collect::<Vec<_>>();
         buckets.sort_by(|left, right| left.name.cmp(&right.name));
         Ok((buckets, topology_complete))

crates/ecstore/src/set_disk/ops/delete_recovery.rs

@@ -1569,6 +1569,86 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn delete_recovery_batch_with_missing_version_recovers_after_preparation_failure() {
+        use crate::disk::RUSTFS_META_TMP_BUCKET;
+        use crate::storage_api_contracts::object::ObjectToDelete;
+
+        let bucket = "recover-batch-missing-version";
+        let (dirs, store, disks) = fixture(bucket).await;
+        let deleted = put(&store, bucket, &vec![48; 4096], true).await.expect("version to delete");
+        let remaining = vec![49; 4096];
+        let retained = put(&store, bucket, &remaining, true).await.expect("version to retain");
+        let missing = Uuid::new_v4();
+        let temporary = dirs[3].path().join(RUSTFS_META_TMP_BUCKET);
+        let saved = dirs[3].path().join("saved-delete-tmp");
+        tokio::fs::rename(&temporary, &saved)
+            .await
+            .expect("save returning disk's staging directory");
+        tokio::fs::write(&temporary, b"block preparation")
+            .await
+            .expect("fail staging after the recovery preflight");
+
+        let (objects, errors) = store
+            .delete_objects(
+                bucket,
+                [deleted, missing]
+                    .into_iter()
+                    .map(|version| ObjectToDelete {
+                        object_name: "object".into(),
+                        version_id: Some(version),
+                        ..Default::default()
+                    })
+                    .collect(),
+                ObjectOptions {
+                    versioned: true,
+                    ..Default::default()
+                },
+            )
+            .await;
+        assert_eq!(objects.len(), 2);
+        assert!(errors.iter().all(Option::is_none), "batch deletion succeeds at quorum: {errors:?}");
+        assert!(
+            disks[3]
+                .discover_delete_transactions(bucket, Some("object"), None, 100)
+                .await
+                .expect("inspect failed preparation")
+                .transactions
+                .is_empty(),
+            "returning disk must adopt the quorum decision without a local preparation"
+        );
+        tokio::fs::remove_file(&temporary).await.expect("remove preparation failure");
+        tokio::fs::rename(&saved, &temporary)
+            .await
+            .expect("restore returning disk staging");
+
+        let result = store
+            .recover_delete_transactions(bucket, &HealOpts::default())
+            .await
+            .expect("recover batch containing a missing non-null version");
+        assert!(result.detail.contains("blocked=0"), "{}", result.detail);
+        assert_eq!(body(&store, bucket, Some(retained)).await, remaining);
+        for version in [deleted, missing] {
+            assert_eq!(
+                disks[3]
+                    .read_version(bucket, bucket, "object", &version.to_string(), &ReadOptions::default())
+                    .await
+                    .expect_err("returning disk must not retain or create a deleted version"),
+                DiskError::FileVersionNotFound
+            );
+        }
+        for disk in &disks {
+            assert!(
+                disk.discover_delete_transactions(bucket, Some("object"), None, 100)
+                    .await
+                    .expect("inspect converged batch")
+                    .transactions
+                    .is_empty(),
+                "every participant must converge and release the decision"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn delete_recovery_dry_run_matches_adoption_of_already_absent_non_null_version() {
         let bucket = "recover-preview-non-null";
@@ -1935,6 +2015,140 @@ mod tests {
             .expect("no retained version decision may keep the empty bucket nonempty");
     }
 
+    #[tokio::test]
+    async fn delete_recovery_partial_batch_receipt_handles_missing_versions() {
+        use crate::disk::RUSTFS_META_TMP_BUCKET;
+        use crate::set_disk::ops::object::DeleteCleanupBarrier;
+        use crate::storage_api_contracts::object::ObjectToDelete;
+
+        for include_missing in [false, true] {
+            let bucket = "recover-prepared-batch-missing-versions";
+            let (dirs, store, disks) = fixture(bucket).await;
+            metadata_sys::update_in(
+                &store.ctx,
+                bucket,
+                crate::bucket::metadata::BUCKET_VERSIONING_CONFIG,
+                b"<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
+            )
+            .await
+            .expect("enable supported versioned bucket");
+            let first = put(&store, bucket, &vec![81; 1024 * 1024], true)
+                .await
+                .expect("first version");
+            let second = put(&store, bucket, &vec![82; 4096], true).await.expect("retained version");
+            let missing = Uuid::new_v4();
+            let barrier = DeleteCleanupBarrier::install(bucket);
+            let deleting = store.clone();
+            let task = tokio::spawn(async move {
+                let mut versions = vec![first];
+                if include_missing {
+                    versions.push(missing);
+                }
+                deleting
+                    .delete_objects(
+                        bucket,
+                        versions
+                            .into_iter()
+                            .map(|version| ObjectToDelete {
+                                object_name: "object".into(),
+                                version_id: Some(version),
+                                ..Default::default()
+                            })
+                            .collect(),
+                        ObjectOptions {
+                            versioned: true,
+                            ..Default::default()
+                        },
+                    )
+                    .await
+            });
+            barrier.wait_until_paused().await;
+            let transactions = disks[0]
+                .discover_delete_transactions(bucket, Some("object"), None, 100)
+                .await
+                .expect("discover batch preparation")
+                .transactions;
+            assert_eq!(transactions.len(), 1);
+            let transaction = transactions[0].transaction;
+            for disk in &disks {
+                let state = disk
+                    .inspect_delete_transaction(bucket, "object", transaction)
+                    .await
+                    .expect("inspect prepared batch");
+                assert_eq!(state.state, DeleteRecoveryState::NewPrepared);
+                let journal = state.journal.expect("prepared journal");
+                assert!(journal.original_digest.is_some());
+                assert!(journal.original_versions.iter().all(|v| v.version_id != Some(missing)));
+            }
+            // Fail durable receipt staging on two participants after metadata quorum.
+            for dir in dirs.iter().skip(2) {
+                tokio::fs::rename(dir.path().join(RUSTFS_META_TMP_BUCKET), dir.path().join("saved-receipt-staging"))
+                    .await
+                    .expect("save receipt staging directory");
+                tokio::fs::write(dir.path().join(RUSTFS_META_TMP_BUCKET), b"receipt staging I/O failure")
+                    .await
+                    .expect("inject receipt staging failure");
+            }
+            barrier.release();
+            let (_, errors) = task.await.expect("batch deletion must join");
+            drop(barrier);
+            assert_eq!(errors.len(), if include_missing { 2 } else { 1 });
+            assert!(errors.iter().all(Option::is_some), "partial receipts are indeterminate: {errors:?}");
+            let mut receipts = 0;
+            let mut prepared = 0;
+            for disk in &disks {
+                match disk
+                    .inspect_delete_transaction(bucket, "object", transaction)
+                    .await
+                    .expect("partial decision")
+                    .state
+                {
+                    DeleteRecoveryState::QuorumCommitted => receipts += 1,
+                    DeleteRecoveryState::NewPrepared => prepared += 1,
+                    state => panic!("unexpected post-publication state: {state:?}"),
+                }
+            }
+            assert_eq!((receipts, prepared), (2, 2));
+            for dir in dirs.iter().skip(2) {
+                tokio::fs::remove_file(dir.path().join(RUSTFS_META_TMP_BUCKET))
+                    .await
+                    .expect("remove injected staging failure");
+                tokio::fs::rename(dir.path().join("saved-receipt-staging"), dir.path().join(RUSTFS_META_TMP_BUCKET))
+                    .await
+                    .expect("restore receipt staging");
+            }
+            store
+                .delete_object(
+                    bucket,
+                    "object",
+                    ObjectOptions {
+                        versioned: true,
+                        version_id: Some(second.to_string()),
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("later version deletion succeeds");
+            store
+                .recover_delete_transactions(bucket, &HealOpts::default())
+                .await
+                .expect("certified batch must recover after later version deletion");
+            for disk in &disks {
+                assert!(
+                    disk.discover_delete_transactions(bucket, Some("object"), None, 100)
+                        .await
+                        .expect("settled transaction inventory")
+                        .transactions
+                        .is_empty()
+                );
+            }
+            store
+                .delete_bucket(bucket, &DeleteBucketOptions::default())
+                .await
+                .expect("settled receipts must not block removal of the empty bucket");
+        }
+    }
+
     #[tokio::test]
     async fn delete_recovery_partial_receipt_converges_after_later_version_absence() {
         let bucket = "recover-partial-chained-versions";

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -63,6 +63,7 @@ use crate::crash_inject::{self, CrashPoint};
 use crate::disk::DiskAPI;
 #[cfg(test)]
 use crate::disk::DiskOption;
+#[cfg(test)]
 use crate::disk::STORAGE_FORMAT_FILE;
 #[cfg(test)]
 use crate::disk::new_disk;
@@ -230,60 +231,6 @@ pub(crate) struct StaleMultipartCleanupGuard {
     lock_guard: ObjectLockDiagGuard,
 }
 
-/// Exact-key recovery is admitted only after every disk proves that the upload
-/// metadata is absent. Keep the checked disk generation and upload lock through
-/// deletion; a quorum miss is not proof that an upload is an orphan.
-/// Final object data, completion replay identity, and old-data cleanup receipts
-/// live under the destination bucket/object, never this staging path. This also
-/// permits removing an exact orphan from a previous bucket incarnation without
-/// claiming that its UploadID belongs to the current incarnation.
-pub(crate) struct OrphanMultipartCleanupGuard {
-    set: Arc<SetDisks>,
-    disks: Vec<Option<DiskStore>>,
-    upload_path: String,
-    lock_guard: Arc<ObjectLockDiagGuard>,
-    exists: bool,
-}
-
-impl OrphanMultipartCleanupGuard {
-    pub(crate) async fn delete(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<bool> {
-        fence_commit_on_lock_loss(Some(&self.lock_guard), "orphan_multipart_cleanup", &self.upload_path)?;
-        ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
-        if !self.exists {
-            return Ok(false);
-        }
-
-        let results = join_all(self.disks.iter().map(|disk| async {
-            let Some(disk) = disk else {
-                return Err(DiskError::DiskNotFound);
-            };
-            match disk
-                .delete(
-                    RUSTFS_META_MULTIPART_BUCKET,
-                    &self.upload_path,
-                    crate::disk::DeleteOptions {
-                        recursive: true,
-                        ..Default::default()
-                    },
-                )
-                .await
-            {
-                Err(DiskError::FileNotFound | DiskError::PathNotFound | DiskError::VolumeNotFound) => Ok(()),
-                result => result,
-            }
-        }))
-        .await;
-        self.set.record_capacity_scope_if_needed(None, &self.disks);
-        let errors = results.into_iter().map(|result| result.err()).collect::<Vec<_>>();
-        if let Some(err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, self.disks.len()) {
-            return Err(to_object_err(err.into(), vec![bucket, object]));
-        }
-        fence_commit_on_lock_loss(Some(&self.lock_guard), "orphan_multipart_cleanup", &self.upload_path)?;
-        ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
-        Ok(true)
-    }
-}
-
 pub(crate) struct DataMovementMultipartAbortGuard {
     upload_path: String,
     write_quorum: Option<usize>,
@@ -610,34 +557,6 @@ fn multipart_bucket_incarnation_matches(metadata: &HashMap<String, String>, expe
     matches!(multipart_bucket_incarnation_id(metadata), Ok(Some(actual)) if actual == expected)
 }
 
-fn orphan_multipart_upload_id_is_canonical(upload_id: &str) -> bool {
-    let Ok(decoded) = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(upload_id.as_bytes()) else {
-        return false;
-    };
-    let Ok(decoded) = std::str::from_utf8(&decoded) else {
-        return false;
-    };
-    let Some((_, raw)) = decoded.split_once('.') else {
-        return false;
-    };
-    let Some((uuid, timestamp)) = raw.split_once('x') else {
-        return false;
-    };
-    let Ok(id) = Uuid::parse_str(uuid) else {
-        return false;
-    };
-    if id.is_nil() || id.to_string() != uuid {
-        return false;
-    }
-    let Ok(timestamp_ns) = timestamp.parse::<i128>() else {
-        return false;
-    };
-    if timestamp_ns.to_string() != timestamp {
-        return false;
-    }
-    OffsetDateTime::from_unix_timestamp_nanos(timestamp_ns).is_ok()
-}
-
 fn validate_multipart_bucket_incarnation(
     metadata: &HashMap<String, String>,
     bucket: &str,
@@ -1327,114 +1246,6 @@ impl SetDisks {
         })
     }
 
-    pub(crate) async fn lock_orphan_multipart_cleanup(
-        self: &Arc<Self>,
-        bucket: &str,
-        object: &str,
-        upload_id: &str,
-        opts: &ObjectOptions,
-        held_guards: &[OrphanMultipartCleanupGuard],
-    ) -> Result<OrphanMultipartCleanupGuard> {
-        let invalid_upload = || StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned());
-        if opts.data_movement
-            || opts.no_lock
-            || self.local_lock_manager.is_disabled()
-            || !orphan_multipart_upload_id_is_canonical(upload_id)
-        {
-            return Err(invalid_upload());
-        }
-        // This path is derived from the caller's exact bucket/key and a canonical
-        // UUID/timestamp upload identity, never from the shared MPU directory list.
-        let upload_path = Self::get_upload_id_dir(bucket, object, upload_id);
-        // Pool labels do not partition the raw ObjectKey sent to lock clients.
-        // Reuse the actual owner only for this exact path and proven backend;
-        // independent distributed domains still need their own guards.
-        let distributed = self.ctx.is_dist_erasure().await;
-        let mut lock_guard = None;
-        for held in held_guards {
-            if held.upload_path != upload_path {
-                continue;
-            }
-            let shared_domain = match (distributed, held.set.ctx.is_dist_erasure().await) {
-                (false, false) => self.shares_namespace_lock_domain(&held.set).await,
-                (true, true) => {
-                    let same = super::super::same_distributed_lock_domain(&self.lockers, &held.set.lockers);
-                    if !same
-                        && self
-                            .lockers
-                            .iter()
-                            .any(|client| held.set.lockers.iter().any(|held_client| Arc::ptr_eq(client, held_client)))
-                    {
-                        // A guard for only part of another quorum is not proof
-                        // of ownership there, and acquiring it again may wait
-                        // on ourselves. Preserve all staging in this topology.
-                        return Err(Error::other("orphan multipart cleanup cannot cover partially overlapping lock domains"));
-                    }
-                    same
-                }
-                _ => false,
-            };
-            if shared_domain {
-                lock_guard = Some(Arc::clone(&held.lock_guard));
-                break;
-            }
-        }
-        let lock_guard = match lock_guard {
-            Some(guard) => guard,
-            None => Arc::new(
-                self.acquire_write_lock_diag("orphan_multipart_cleanup", RUSTFS_META_MULTIPART_BUCKET, &upload_path)
-                    .await?,
-            ),
-        };
-        fence_commit_on_lock_loss(Some(&lock_guard), "orphan_multipart_cleanup", &upload_path)?;
-        ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
-        let disks = self.disks.read().await.clone();
-        if disks.len() != self.set_drive_count || disks.is_empty() {
-            return Err(Error::ErasureReadQuorum);
-        }
-        // Raw full fan-out cannot quorum-short-circuit or interpret an empty
-        // version list as absence of the physical metadata file.
-        let (_, errors) = Self::read_all_raw_file_info(&disks, RUSTFS_META_MULTIPART_BUCKET, &upload_path, false).await;
-        if errors.len() != disks.len() {
-            return Err(Error::ErasureReadQuorum);
-        }
-        for error in errors {
-            match error {
-                Some(DiskError::FileNotFound | DiskError::VolumeNotFound) => {}
-                // Even one valid metadata copy may still be recoverable. Never
-                // convert a partial, corrupt, or unreachable upload into debris.
-                None => return Err(invalid_upload()),
-                Some(error) => return Err(to_object_err(error.into(), vec![bucket, object, upload_id])),
-            }
-        }
-        let mut exists = false;
-        for disk in &disks {
-            let Some(disk) = disk else {
-                return Err(Error::ErasureReadQuorum);
-            };
-            match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, -1).await {
-                Ok(entries) => {
-                    // Raw metadata reads report a directory named xl.meta as
-                    // FileNotFound. Its physical presence is unknown/corrupt
-                    // metadata, not proof that this staging can be deleted.
-                    if entries.iter().any(|entry| entry.trim_end_matches('/') == STORAGE_FORMAT_FILE) {
-                        return Err(to_object_err(DiskError::FileCorrupt.into(), vec![bucket, object, upload_id]));
-                    }
-                    exists = true;
-                }
-                Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => {}
-                Err(error) => return Err(to_object_err(error.into(), vec![bucket, object, upload_id])),
-            }
-        }
-        Ok(OrphanMultipartCleanupGuard {
-            set: self.clone(),
-            disks,
-            upload_path,
-            lock_guard,
-            exists,
-        })
-    }
-
     #[allow(clippy::too_many_arguments)]
     pub(crate) async fn list_multipart_uploads_for_incarnation(
         &self,
@@ -1867,7 +1678,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             drop(admission_guard);
         }
 
-        let mut commit_owns_cleanup = false;
         let result: Result<PartInfo> = async {
             let erasure =
                 Arc::new(coding::Erasure::try_new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size)
@@ -2071,171 +1881,120 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                     .await?;
                 (Some(upload_guard), Some(part_guard))
             };
-            // Only guards owned by this operation may outlive its caller. Internal
-            // data-movement and borrowed-lock callers retain their synchronous
-            // lifetime; a fence by itself cannot transfer the outer lock lease.
-            let detach_commit_owner = _upload_commit_guard.is_some()
-                && opts.namespace_lock_fence.is_none()
-                && (opts.bucket_lifecycle_lock_fence.is_none() || opts.bucket_lifecycle_lock_owner.is_some());
-            let commit_set = self.clone();
-            let commit_bucket = bucket.to_owned();
-            let commit_object = object.to_owned();
-            let commit_upload_id = upload_id.to_owned();
-            let commit_upload_id_path = upload_id_path.clone();
-            let commit_tmp_part = tmp_part.clone();
-            let commit_opts = opts.clone();
-            commit_owns_cleanup = true;
-            // The body is already staged. Keep every commit guard through the
-            // prepare, publish, settlement/rollback and temporary cleanup I/O,
-            // even when the HTTP request stops awaiting this task.
-            let commit = async move {
-                let _upload_commit_guard = _upload_commit_guard;
-                let _part_commit_guard = _part_commit_guard;
-                let _capped_staging_guard = _capped_staging_guard;
-                let bucket = commit_bucket.as_str();
-                let object = commit_object.as_str();
-                let upload_id = commit_upload_id.as_str();
-                let upload_id_path = commit_upload_id_path;
-                let opts = &commit_opts;
-                let result: Result<PartInfo> = async {
-                    let (commit_fi, _) = commit_set
-                        .check_upload_id_exists_with_opts(bucket, object, upload_id, multipart_size_limit.is_some(), opts)
-                        .await?;
-                    let commit_size_limit = multipart_size_limit_from_metadata(&commit_fi.metadata)?;
-                    if commit_size_limit != multipart_size_limit {
-                        return Err(Error::InvalidArgument(
-                            "multipart upload".to_string(),
-                            SUFFIX_MAX_TOTAL_OBJECT_SIZE.to_string(),
-                            "size limit metadata changed or is missing".to_string(),
-                        ));
-                    }
-                    ensure_data_movement_upload_access(&commit_fi, bucket, object, upload_id, opts)?;
-                    ensure_multipart_bucket_incarnation(
-                        &commit_set.ctx,
-                        &commit_fi,
+            let (commit_fi, _) = self
+                .check_upload_id_exists_with_opts(bucket, object, upload_id, multipart_size_limit.is_some(), opts)
+                .await?;
+            let commit_size_limit = multipart_size_limit_from_metadata(&commit_fi.metadata)?;
+            if commit_size_limit != multipart_size_limit {
+                return Err(Error::InvalidArgument(
+                    "multipart upload".to_string(),
+                    SUFFIX_MAX_TOTAL_OBJECT_SIZE.to_string(),
+                    "size limit metadata changed or is missing".to_string(),
+                ));
+            }
+            ensure_data_movement_upload_access(&commit_fi, bucket, object, upload_id, opts)?;
+            ensure_multipart_bucket_incarnation(
+                &self.ctx,
+                &commit_fi,
+                bucket,
+                object,
+                upload_id,
+                opts.expected_bucket_incarnation_id,
+            )
+            .await?;
+            #[cfg(test)]
+            pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost).await;
+            fence_commit_on_lock_loss(_upload_commit_guard.as_ref(), "put_object_part_commit", &upload_id_path)?;
+            fence_commit_on_lock_loss(_part_commit_guard.as_ref(), "put_object_part_commit", &part_lock_path)?;
+            if opts
+                .namespace_lock_fence
+                .as_ref()
+                .is_some_and(NamespaceLockFence::is_lock_lost)
+            {
+                return Err(StorageError::NamespaceLockQuorumUnavailable {
+                    mode: "put_object_part_outer_lock",
+                    bucket: bucket.to_string(),
+                    object: object.to_string(),
+                    required: 1,
+                    achieved: 0,
+                });
+            }
+            ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
+
+            if let Some(limit) = commit_size_limit {
+                let current_size = self
+                    .current_multipart_logical_size(bucket, object, upload_id, &upload_id_path, &commit_fi, part_id)
+                    .await?;
+                let candidate_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
+                admitted_multipart_size(current_size, candidate_size, limit)?;
+            }
+
+            let decommission_capacity_guard = if let Some(store) = opts.decommission_capacity_admission.as_ref() {
+                Some(
+                    store
+                        .acquire_external_decommission_capacity_fence(
+                            &[self.pool_index],
+                            DecommissionCapacityAdmission::ExistingMultipart,
+                        )
+                        .await?,
+                )
+            } else {
+                None
+            };
+            #[cfg(test)]
+            pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterCapacityAdmission).await;
+            if decommission_capacity_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
+                return Err(StorageError::NamespaceLockQuorumUnavailable {
+                    mode: "put_object_part_decommission_capacity",
+                    bucket: bucket.to_string(),
+                    object: object.to_string(),
+                    required: 1,
+                    achieved: 0,
+                });
+            }
+            let rename_result = self
+                .rename_part(
+                    &shuffle_disks,
+                    RUSTFS_META_TMP_BUCKET,
+                    &tmp_part_path,
+                    RUSTFS_META_MULTIPART_BUCKET,
+                    &part_path,
+                    part_info_buff.into(),
+                    write_quorum,
+                    Some(MultipartWriteQuorumContext {
+                        stage: MULTIPART_WRITE_QUORUM_RENAME_PART,
                         bucket,
                         object,
                         upload_id,
-                        opts.expected_bucket_incarnation_id,
-                    )
-                    .await?;
-                    #[cfg(test)]
-                    pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost).await;
-                    fence_commit_on_lock_loss(_upload_commit_guard.as_ref(), "put_object_part_commit", &upload_id_path)?;
-                    fence_commit_on_lock_loss(_part_commit_guard.as_ref(), "put_object_part_commit", &part_lock_path)?;
-                    if opts
-                        .namespace_lock_fence
-                        .as_ref()
-                        .is_some_and(NamespaceLockFence::is_lock_lost)
-                    {
-                        return Err(StorageError::NamespaceLockQuorumUnavailable {
-                            mode: "put_object_part_outer_lock",
-                            bucket: bucket.to_string(),
-                            object: object.to_string(),
-                            required: 1,
-                            achieved: 0,
-                        });
-                    }
-                    ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
-
-                    if let Some(limit) = commit_size_limit {
-                        let current_size = commit_set
-                            .current_multipart_logical_size(bucket, object, upload_id, &upload_id_path, &commit_fi, part_id)
-                            .await?;
-                        let candidate_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
-                        admitted_multipart_size(current_size, candidate_size, limit)?;
-                    }
-
-                    let decommission_capacity_guard = if let Some(store) = opts.decommission_capacity_admission.as_ref() {
-                        Some(
-                            store
-                                .acquire_external_decommission_capacity_fence(
-                                    &[commit_set.pool_index],
-                                    DecommissionCapacityAdmission::ExistingMultipart,
-                                )
-                                .await?,
-                        )
-                    } else {
-                        None
-                    };
-                    #[cfg(test)]
-                    pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterCapacityAdmission).await;
-                    if decommission_capacity_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
-                        return Err(StorageError::NamespaceLockQuorumUnavailable {
-                            mode: "put_object_part_decommission_capacity",
-                            bucket: bucket.to_string(),
-                            object: object.to_string(),
-                            required: 1,
-                            achieved: 0,
-                        });
-                    }
-                    let rename_result = commit_set
-                        .rename_part(
-                            &shuffle_disks,
-                            RUSTFS_META_TMP_BUCKET,
-                            &tmp_part_path,
-                            RUSTFS_META_MULTIPART_BUCKET,
-                            &part_path,
-                            part_info_buff.into(),
-                            write_quorum,
-                            Some(MultipartWriteQuorumContext {
-                                stage: MULTIPART_WRITE_QUORUM_RENAME_PART,
-                                bucket,
-                                object,
-                                upload_id,
-                                part_number: Some(part_id),
-                            }),
-                        )
-                        .await;
-                    drop(decommission_capacity_guard);
-                    let _ = rename_result?;
-                    #[cfg(test)]
-                    observe_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
-
-                    #[cfg(test)]
-                    pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterRename).await;
-
-                    let ret: PartInfo = PartInfo {
-                        etag: Some(etag.clone()),
-                        part_num: part_id,
-                        last_mod: Some(OffsetDateTime::now_utc()),
-                        size: w_size,
-                        actual_size,
-                    };
-
-                    // error!("put_object_part ret {:?}", &ret);
-
-                    Ok(ret)
-                }
+                        part_number: Some(part_id),
+                    }),
+                )
                 .await;
-                if result.is_err()
-                    && let Err(err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_part).await
-                {
-                    warn!(
-                        event = EVENT_SET_DISK_MULTIPART,
-                        component = LOG_COMPONENT_ECSTORE,
-                        subsystem = LOG_SUBSYSTEM_SET_DISK,
-                        result = "cleanup_failed",
-                        bucket,
-                        object,
-                        error = ?err,
-                        "failed to clean multipart commit staging"
-                    );
-                }
-                result
+            drop(decommission_capacity_guard);
+            let _ = rename_result?;
+            #[cfg(test)]
+            observe_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
+
+            #[cfg(test)]
+            pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterRename).await;
+            drop(_part_commit_guard);
+            drop(_upload_commit_guard);
+
+            let ret: PartInfo = PartInfo {
+                etag: Some(etag.clone()),
+                part_num: part_id,
+                last_mod: Some(OffsetDateTime::now_utc()),
+                size: w_size,
+                actual_size,
             };
-            if detach_commit_owner {
-                tokio::spawn(commit)
-                    .await
-                .map_err(|err| Error::other_with_context("put_object_part commit task failed", err))?
-            } else {
-                commit.await
-            }
+
+            // error!("put_object_part ret {:?}", &ret);
+
+            Ok(ret)
         }
         .await;
 
-        if !commit_owns_cleanup
-            && result.is_err()
+        if result.is_err()
             && let Err(err) = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_part).await
         {
             warn!(tmp_part = %tmp_part, error = ?err, "failed to cleanup multipart temporary data");
@@ -4474,300 +4233,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn orphan_multipart_identity_requires_canonical_upload_id() {
-        let created = OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(10);
-        let id = Uuid::new_v4();
-        let timestamp = (created + time::Duration::seconds(1)).unix_timestamp_nanos();
-        let encode = |raw: &str| runtime_sources::deployment_upload_id(raw);
-        let upload_id = encode(&format!("{id}x{timestamp}"));
-        assert!(orphan_multipart_upload_id_is_canonical(&upload_id));
-        for raw in [
-            format!("{}x{timestamp}", Uuid::nil()),
-            format!("{}x{timestamp}", id.simple()),
-            format!("{id}x0{timestamp}"),
-            format!("{id}x+{timestamp}"),
-            format!("{id}x{timestamp}/../another-upload"),
-            format!("../{id}x{timestamp}"),
-            format!("{id}x{}", i128::MAX),
-        ] {
-            assert!(
-                !orphan_multipart_upload_id_is_canonical(&encode(&raw)),
-                "noncanonical upload identity must be rejected: {raw}"
-            );
-        }
-        assert!(!orphan_multipart_upload_id_is_canonical("not-base64"));
-    }
-
-    #[tokio::test]
-    async fn orphan_multipart_cleanup_requires_every_disk_to_prove_metadata_absence() {
-        let (_temp_dirs, disks, set) = hermetic_set_disks(4).await;
-        let bucket = "orphan-multipart-proof";
-        let object = "object";
-        make_bucket_on_all(&disks, bucket).await;
-        let upload = set
-            .new_multipart_upload(bucket, object, &ObjectOptions::default())
-            .await
-            .expect("create upload");
-        let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload.upload_id);
-        let metadata_path = format!("{upload_path}/{STORAGE_FORMAT_FILE}");
-        let leftover_path = format!("{upload_path}/part.late");
-        for disk in &disks {
-            disk.write_all(RUSTFS_META_MULTIPART_BUCKET, &leftover_path, Bytes::from_static(b"late part"))
-                .await
-                .expect("write late part fixture");
-        }
-        for disk in &disks[..3] {
-            disk.delete(RUSTFS_META_MULTIPART_BUCKET, &metadata_path, Default::default())
-                .await
-                .expect("remove majority metadata");
-        }
-        let opts = ObjectOptions::default();
-        assert!(
-            set.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
-                .await
-                .is_err(),
-            "one surviving metadata copy must block orphan recovery"
-        );
-        disks[3]
-            .write_all(RUSTFS_META_MULTIPART_BUCKET, &metadata_path, Bytes::from_static(b"corrupt metadata"))
-            .await
-            .expect("corrupt final metadata copy");
-        assert!(
-            set.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
-                .await
-                .is_err(),
-            "corrupt metadata must not be treated as absence"
-        );
-        disks[3]
-            .delete(RUSTFS_META_MULTIPART_BUCKET, &metadata_path, Default::default())
-            .await
-            .expect("remove final metadata copy");
-        set.disks.write().await[3] = None;
-        assert!(
-            set.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
-                .await
-                .is_err(),
-            "an unreachable disk must not be hidden by absence quorum"
-        );
-        set.disks.write().await[3] = Some(disks[3].clone());
-        for disk in &disks {
-            assert_eq!(
-                disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &leftover_path)
-                    .await
-                    .expect("failed preflight must preserve every part"),
-                Bytes::from_static(b"late part")
-            );
-        }
-
-        let guard = set
-            .lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
-            .await
-            .expect("all disks now prove metadata absence");
-        assert!(guard.delete(bucket, object, &opts).await.expect("recover exact staging"));
-        drop(guard);
-        for disk in &disks {
-            assert!(matches!(
-                disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, -1).await,
-                Err(DiskError::FileNotFound | DiskError::VolumeNotFound)
-            ));
-        }
-        let guard = set
-            .lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
-            .await
-            .expect("already absent staging remains provably absent");
-        assert!(
-            !guard
-                .delete(bucket, object, &opts)
-                .await
-                .expect("absence does not report recovery")
-        );
-    }
-
-    #[tokio::test]
-    async fn orphan_multipart_cleanup_rejects_metadata_directory_on_one_disk() {
-        let (temp_dirs, disks, set) = hermetic_set_disks(4).await;
-        let bucket = "orphan-multipart-directory";
-        let object = "object";
-        make_bucket_on_all(&disks, bucket).await;
-        let upload = set
-            .new_multipart_upload(bucket, object, &ObjectOptions::default())
-            .await
-            .expect("create upload");
-        let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload.upload_id);
-        let metadata_path = format!("{upload_path}/{STORAGE_FORMAT_FILE}");
-        let leftover_path = format!("{upload_path}/part.late");
-        for disk in &disks {
-            disk.delete(RUSTFS_META_MULTIPART_BUCKET, &metadata_path, Default::default())
-                .await
-                .expect("remove upload metadata");
-            disk.write_all(RUSTFS_META_MULTIPART_BUCKET, &leftover_path, Bytes::from_static(b"late part"))
-                .await
-                .expect("write leftover part");
-        }
-        let metadata_directory = temp_dirs[3].path().join(RUSTFS_META_MULTIPART_BUCKET).join(&metadata_path);
-        tokio::fs::create_dir_all(&metadata_directory)
-            .await
-            .expect("replace one missing metadata file with a directory");
-        let (_, errors) =
-            SetDisks::read_all_raw_file_info(&set.disks.read().await, RUSTFS_META_MULTIPART_BUCKET, &upload_path, false).await;
-        assert!(
-            errors.iter().all(|error| matches!(error, Some(DiskError::FileNotFound))),
-            "raw metadata reads alone must reproduce the directory-as-absence case: {errors:?}"
-        );
-        assert!(
-            set.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &ObjectOptions::default(), &[])
-                .await
-                .is_err(),
-            "one physical metadata directory must block cleanup on every disk"
-        );
-        assert!(metadata_directory.is_dir(), "failed preflight must preserve the metadata directory");
-        for disk in &disks {
-            assert_eq!(
-                disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &leftover_path)
-                    .await
-                    .expect("failed preflight must preserve every leftover part"),
-                Bytes::from_static(b"late part")
-            );
-        }
-    }
-
-    #[derive(Clone, Copy, PartialEq, Eq)]
-    enum OrphanMultipartLockDomainCase {
-        Shared,
-        Independent,
-        PartiallyOverlapping,
-    }
-
-    async fn check_orphan_multipart_distributed_lock_owners(case: OrphanMultipartLockDomainCase) {
-        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
-        ctx.update_erasure_type(SetupType::DistErasure).await;
-        let first_client: Arc<dyn LockClient> =
-            Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
-        let second_client: Arc<dyn LockClient> =
-            Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
-        let second_clients = match case {
-            OrphanMultipartLockDomainCase::Shared => vec![first_client.clone()],
-            OrphanMultipartLockDomainCase::Independent => vec![second_client],
-            OrphanMultipartLockDomainCase::PartiallyOverlapping => vec![first_client.clone(), second_client],
-        };
-        let (first_dirs, first_disks, first_set) =
-            hermetic_set_disks_with_lockers_and_ctx(4, 0, 2, vec![first_client], ctx.clone()).await;
-        let (second_dirs, second_disks, second_set) = hermetic_set_disks_with_lockers_and_ctx(4, 1, 2, second_clients, ctx).await;
-        let bucket = "orphan-multipart-dist-locks";
-        let object = "object";
-        make_bucket_on_all(&first_disks, bucket).await;
-        make_bucket_on_all(&second_disks, bucket).await;
-        let upload_id = runtime_sources::deployment_upload_id(&format!(
-            "{}x{}",
-            Uuid::new_v4(),
-            OffsetDateTime::now_utc().unix_timestamp_nanos()
-        ));
-        let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
-        for directory in first_dirs.iter().chain(second_dirs.iter()) {
-            let staging = directory.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_path);
-            tokio::fs::create_dir_all(&staging)
-                .await
-                .expect("create exact orphan staging");
-            tokio::fs::write(staging.join("part.late"), b"late part")
-                .await
-                .expect("write orphan part");
-        }
-        let opts = ObjectOptions::default();
-        let first_guard = first_set
-            .lock_orphan_multipart_cleanup(bucket, object, &upload_id, &opts, &[])
-            .await
-            .expect("first domain has an orphan");
-        let second_result = tokio::time::timeout(
-            Duration::from_secs(10),
-            second_set.lock_orphan_multipart_cleanup(bucket, object, &upload_id, &opts, std::slice::from_ref(&first_guard)),
-        )
-        .await
-        .expect("lock-domain admission must not wait on a guard it already owns");
-        if case == OrphanMultipartLockDomainCase::PartiallyOverlapping {
-            let error = match second_result {
-                Ok(_) => panic!("one existing quorum cannot authorize a partially overlapping domain"),
-                Err(error) => error,
-            };
-            assert!(error.to_string().contains("partially overlapping lock domains"));
-            for directory in first_dirs.iter().chain(second_dirs.iter()) {
-                assert!(
-                    directory
-                        .path()
-                        .join(RUSTFS_META_MULTIPART_BUCKET)
-                        .join(&upload_path)
-                        .join("part.late")
-                        .exists(),
-                    "uncertain lock coverage must preserve every physical part"
-                );
-            }
-            return;
-        }
-        let second_guard = second_result.expect("the second pool has a proven lock domain");
-        assert_eq!(
-            Arc::ptr_eq(&first_guard.lock_guard, &second_guard.lock_guard),
-            case == OrphanMultipartLockDomainCase::Shared,
-            "only the same exact backend may share a real upload guard across pool labels"
-        );
-        let second_namespace = second_set
-            .new_ns_lock(RUSTFS_META_MULTIPART_BUCKET, &upload_path)
-            .await
-            .unwrap();
-        let mut second_writer = Box::pin(second_namespace.get_write_lock(Duration::from_secs(10)));
-        assert!(futures::poll!(&mut second_writer).is_pending());
-        if case == OrphanMultipartLockDomainCase::Shared {
-            drop(first_guard);
-            assert!(
-                futures::poll!(&mut second_writer).is_pending(),
-                "the second pool must retain the shared real owner after the first guard drops"
-            );
-            drop(second_guard);
-            drop(
-                tokio::time::timeout(Duration::from_secs(10), second_writer)
-                    .await
-                    .expect("the shared backend should release after the last owner")
-                    .expect("shared-domain writer resumes"),
-            );
-            return;
-        }
-        let first_namespace = first_set
-            .new_ns_lock(RUSTFS_META_MULTIPART_BUCKET, &upload_path)
-            .await
-            .unwrap();
-        let mut first_writer = Box::pin(first_namespace.get_write_lock(Duration::from_secs(10)));
-        assert!(futures::poll!(&mut first_writer).is_pending());
-        drop(first_guard);
-        drop(
-            tokio::time::timeout(Duration::from_secs(10), first_writer)
-                .await
-                .expect("first domain should release independently")
-                .expect("first writer resumes"),
-        );
-        assert!(futures::poll!(&mut second_writer).is_pending(), "the second real owner must remain held");
-        drop(second_guard);
-        drop(
-            tokio::time::timeout(Duration::from_secs(10), second_writer)
-                .await
-                .expect("second domain should release with its owner")
-                .expect("second writer resumes"),
-        );
-    }
-
-    #[tokio::test]
-    async fn orphan_multipart_cleanup_keeps_separate_distributed_lock_owners() {
-        check_orphan_multipart_distributed_lock_owners(OrphanMultipartLockDomainCase::Independent).await;
-    }
-
-    #[tokio::test]
-    async fn orphan_multipart_cleanup_reuses_shared_distributed_owner_across_pools() {
-        check_orphan_multipart_distributed_lock_owners(OrphanMultipartLockDomainCase::Shared).await;
-    }
-
-    #[tokio::test]
-    async fn orphan_multipart_cleanup_rejects_partially_overlapping_distributed_domains() {
-        check_orphan_multipart_distributed_lock_owners(OrphanMultipartLockDomainCase::PartiallyOverlapping).await;
-    }
-
     async fn stage_upload_with_create_opts(
         set_disks: &Arc<SetDisks>,
         bucket: &str,
@@ -6905,76 +6370,6 @@ mod tests {
             .expect("abort should delete the upload after UploadPart releases the lock");
     }
 
-    #[tokio::test(flavor = "multi_thread")]
-    #[serial]
-    async fn cancelled_put_object_part_keeps_upload_lock_through_commit() {
-        for pause in [
-            MultipartCommitPause::PutPartAfterCapacityAdmission,
-            MultipartCommitPause::PutPartAfterRename,
-        ] {
-            let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
-            let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
-            let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
-            let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
-            let bucket = "multipart-cancelled-part-commit-bucket";
-            let object = "object";
-            make_bucket_on_all(&disk_stores, bucket).await;
-            let upload = set_disks
-                .new_multipart_upload(bucket, object, &ObjectOptions::default())
-                .await
-                .expect("multipart upload should be created");
-            let upload_id = upload.upload_id;
-            let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
-            signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_path.clone()));
-            let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
-            let barrier = MultipartCommitBarrier::install(bucket, object, pause);
-
-            let put_store = set_disks.clone();
-            let put_upload_id = upload_id.clone();
-            let put = tokio::spawn(async move {
-                let mut reader = PutObjReader::from_vec(vec![0x45; 4096]);
-                put_store
-                    .put_object_part(bucket, object, &put_upload_id, 1, &mut reader, &ObjectOptions::default())
-                    .await
-            });
-            barrier.wait_until_paused().await;
-            put.abort();
-            assert!(
-                put.await
-                    .expect_err("the request waiter should remain cancellable")
-                    .is_cancelled()
-            );
-
-            let abort_store = set_disks.clone();
-            let abort_upload_id = upload_id.clone();
-            let mut abort = tokio::spawn(async move {
-                abort_store
-                    .abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
-                    .await
-            });
-            signaling.wait_for_attempts(2).await;
-            assert!(
-                tokio::time::timeout(Duration::from_millis(100), &mut abort).await.is_err(),
-                "Abort must not acquire the upload lock while a cancelled request's commit is paused at {pause:?}"
-            );
-            barrier.release();
-            tokio::time::timeout(Duration::from_secs(10), abort)
-                .await
-                .expect("Abort should finish after the commit drains")
-                .expect("Abort task should not panic")
-                .expect("Abort should delete the committed part staging");
-            for disk in &disk_stores {
-                assert!(
-                    matches!(
-                        disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, -1).await,
-                        Err(DiskError::FileNotFound)
-                    ),
-                    "the upload directory must be absent after the commit and Abort finish"
-                );
-            }
-        }
-    }
-
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
     async fn put_object_part_different_part_numbers_commit_concurrently() {
@@ -10404,18 +9799,6 @@ mod tests {
                 "the post-commit crash must leave the upload listable for reclamation"
             );
 
-            let cleanup = set_disks
-                .lock_orphan_multipart_cleanup(bucket, object, &u_new, &ObjectOptions::default(), &[])
-                .await
-                .expect("the committed upload has no staging metadata on any disk");
-            assert!(
-                cleanup
-                    .delete(bucket, object, &ObjectOptions::default())
-                    .await
-                    .expect("recover completed upload staging")
-            );
-            drop(cleanup);
-
             // The commit rename already consumed the upload's xl.meta, so the
             // retry cannot read staging. It must still return the committed
             // object: AWS CompleteMultipartUpload is idempotent for the same
@@ -10434,6 +9817,21 @@ mod tests {
                 !upload_is_listed(&set_disks, bucket, object, &u_new).await,
                 "the idempotent retry must reclaim staging left behind by the crash"
             );
+
+            // Reclaim the leftover exactly as the production tail does: delete_all
+            // on the upload path (abort_multipart_upload cannot — the upload's
+            // metadata is gone, so it too answers InvalidUploadID).
+            let upload_dir = SetDisks::get_upload_id_dir(bucket, object, &u_new);
+            set_disks
+                .delete_all(RUSTFS_META_MULTIPART_BUCKET, &upload_dir)
+                .await
+                .expect("sweeping the leftover upload staging should succeed");
+            assert!(
+                !upload_is_listed(&set_disks, bucket, object, &u_new).await,
+                "reclaiming the upload must drop it from the listing"
+            );
+            let (body_after, _) = read_object(&set_disks, bucket, object).await;
+            assert_eq!(body_after, new, "reclaiming the leftover upload must not disturb the committed object");
         }
 
         #[tokio::test]
@@ -10480,18 +9878,6 @@ mod tests {
                             .expect("the failed post-commit completion should release its object guard"),
                     );
 
-                    let cleanup = set_disks
-                        .lock_orphan_multipart_cleanup(bucket, object, &u_new, &ObjectOptions::default(), &[])
-                        .await
-                        .expect("committed staging has no upload metadata");
-                    assert!(
-                        cleanup
-                            .delete(bucket, object, &ObjectOptions::default())
-                            .await
-                            .expect("orphan recovery must only remove upload staging")
-                    );
-                    drop(cleanup);
-
                     let (body, _) = read_object(&set_disks, bucket, object).await;
                     assert_eq!(body, new, "the committed replacement must remain readable after the crash");
                     let mut receipts = 0;

crates/ecstore/src/set_disk/read.rs

@@ -15,9 +15,9 @@
 use super::{
     Arc, Bytes, DiskError, DiskOption, DiskStore, Endpoint, ErasureCache, Error, FileInfo, GetCodecStreamingFallbackReason,
     GetObjectFileInfo, GetObjectMetadataCacheCapability, GetObjectMetadataCacheEntry, GetObjectMetadataCacheGeneration,
-    GetObjectMetadataCacheKey, GetObjectMetadataCacheLookupKey, GetObjectReadPolicy, HashAlgorithm, LOG_COMPONENT_ECSTORE,
-    LOG_SUBSYSTEM_SET_DISK, OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectOptions, RUSTFS_META_BUCKET, ReadOptions, Result, SetDisks,
-    StorageError, adaptive_duplex_buffer_size, build_get_codec_streaming_decode_engine, build_inline_bitrot_readers_from_refs,
+    GetObjectMetadataCacheKey, GetObjectReadPolicy, HashAlgorithm, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK,
+    OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectOptions, RUSTFS_META_BUCKET, ReadOptions, Result, SetDisks, StorageError,
+    adaptive_duplex_buffer_size, build_get_codec_streaming_decode_engine, build_inline_bitrot_readers_from_refs,
     collect_inline_data_shard_fileinfos_by_index, debug, error, get_codec_streaming_metrics_path,
     get_codec_streaming_multipart_max_parts, get_object_read_policy, is_codec_streaming_multipart_enabled,
     is_multipart_reader_setup_prefetch_enabled, object_fits_single_block, reduce_read_quorum_errs, to_object_err,
@@ -208,16 +208,6 @@ use tokio::time::timeout;
 #[cfg(test)]
 use uuid::Uuid;
 
-fn metadata_quorum_outcome(error: &StorageError) -> &'static str {
-    if error.is_not_found() {
-        "not_found"
-    } else if error.is_quorum_error() {
-        "not_reached"
-    } else {
-        "unavailable"
-    }
-}
-
 pub(super) struct GetObjectDownstreamWriter<W> {
     inner: W,
 }
@@ -410,7 +400,7 @@ impl SetDisks {
         let Some(generation) = self.get_object_metadata_cache_generation(bucket, object) else {
             return MetadataCacheLookup::Miss;
         };
-        let key = GetObjectMetadataCacheLookupKey::new(bucket, object, generation);
+        let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
         // moka handles TTL expiry automatically; no is_fresh() check needed
         let Some(entry) = self.get_object_metadata_cache.get(&key).await else {
             return MetadataCacheLookup::Miss;
@@ -472,14 +462,14 @@ impl SetDisks {
         entry: Arc<GetObjectMetadataCacheEntry>,
         after_insert: impl FnOnce(),
     ) {
-        // Every publisher shares Moka's per-key compute lock. A partial snapshot
-        // cannot downgrade a full layout or extend its original TTL.
+        // Every publisher shares Moka's per-key compute lock. A conditional
+        // snapshot cannot downgrade a full layout or extend its original TTL.
         self.get_object_metadata_cache
             .entry(key.clone())
             .and_compute_with(|current| {
                 std::future::ready(
                     if !self.is_get_object_metadata_cache_generation_current(generation)
-                        || (entry.capability != GetObjectMetadataCacheCapability::Full
+                        || (entry.capability == GetObjectMetadataCacheCapability::ConditionOnly
                             && current
                                 .as_ref()
                                 .is_some_and(|existing| existing.value().capability == GetObjectMetadataCacheCapability::Full))
@@ -612,11 +602,6 @@ impl SetDisks {
         purpose: MetadataReadPurpose,
     ) -> Result<GetObjectFileInfo> {
         let vid = opts.version_id.clone().unwrap_or_default();
-        let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
-            GET_OBJECT_PATH_INTERNAL_META
-        } else {
-            GET_OBJECT_PATH_LEGACY_DUPLEX
-        };
         let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
 
         let metadata_cache_lookup_start = get_stage_timer_if_enabled(stage_metrics_enabled);
@@ -632,17 +617,12 @@ impl SetDisks {
             match self.lookup_cached_get_object_fileinfo(bucket, object).await {
                 MetadataCacheLookup::Hit(cached)
                     if cached.capability == GetObjectMetadataCacheCapability::Full
-                        || (cached.capability == GetObjectMetadataCacheCapability::QuorumSelected
-                            && read_data
-                            && allow_early_stop)
                         || (purpose == MetadataReadPurpose::GetObjectReader
-                            && cached.capability == GetObjectMetadataCacheCapability::ConditionOnly
                             && allow_early_stop
                             && crate::object_api::get_object_read_condition_is_terminal(
                                 crate::object_api::GetObjectReadMetadata::from_file_info(bucket, object, &cached.fi),
                             )) =>
                 {
-                    rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, "reached");
                     rustfs_io_metrics::record_get_object_metadata_cache_decision(
                         GET_OBJECT_PATH_SET_DISK,
                         GET_METADATA_CACHE_DECISION_HIT,
@@ -653,15 +633,7 @@ impl SetDisks {
                         GET_STAGE_METADATA_CACHE_LOOKUP,
                         metadata_cache_lookup_start,
                     );
-                    let snapshot = if cached.capability == GetObjectMetadataCacheCapability::QuorumSelected
-                        && non_inline_data_read_early_stop_allowed(read_data, bucket, object)
-                        && late_materialization_candidate_is_safe(&cached.fi)
-                    {
-                        GetObjectFileInfo::shared_with_late_metadata_refresh(cached)
-                    } else {
-                        GetObjectFileInfo::shared(cached)
-                    };
-                    return Ok(snapshot);
+                    return Ok(GetObjectFileInfo::shared(cached));
                 }
                 MetadataCacheLookup::Hit(_) => {
                     // Keep the conditional entry for other requests; this read
@@ -711,7 +683,7 @@ impl SetDisks {
         let allow_read_version_coalescing = purpose == MetadataReadPurpose::GetObjectReader
             && !crate::bucket::utils::is_meta_bucketname(bucket)
             && crate::runtime::global::get_metadata_read_version_coalescing_service_ready();
-        let metadata_read_result = if allow_read_version_coalescing {
+        let metadata_read = if allow_read_version_coalescing {
             Self::read_metadata_for_get_object(
                 &disks,
                 "",
@@ -723,7 +695,7 @@ impl SetDisks {
                 allow_early_stop,
                 self.default_parity_count,
             )
-            .await
+            .await?
         } else {
             Self::read_metadata_observed(
                 &disks,
@@ -737,19 +709,16 @@ impl SetDisks {
                 allow_early_stop,
                 self.default_parity_count,
             )
-            .await
-        };
-        let metadata_read = match metadata_read_result {
-            Ok(metadata_read) => metadata_read,
-            Err(err) => {
-                let err = StorageError::from(err);
-                rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, metadata_quorum_outcome(&err));
-                return Err(err);
-            }
+            .await?
         };
         let metadata_fanout_complete = metadata_read.is_complete();
         let condition_stopped = metadata_read.condition_stopped;
         let (mut parts_metadata, errs, metadata_fanout_diagnostics) = metadata_read.into_legacy();
+        let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
+            GET_OBJECT_PATH_INTERNAL_META
+        } else {
+            GET_OBJECT_PATH_LEGACY_DUPLEX
+        };
         metadata_fanout_diagnostics.record(metadata_metrics_path);
         // warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata);
         // warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs);
@@ -762,7 +731,6 @@ impl SetDisks {
         {
             Ok(v) => v,
             Err(e) => {
-                rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, metadata_quorum_outcome(&e));
                 // error!("Self::object_quorum_from_meta: {:?}, bucket: {}, object: {}", &e, bucket, object);
                 record_get_stage_duration_if_enabled(
                     GET_OBJECT_PATH_SET_DISK,
@@ -777,7 +745,6 @@ impl SetDisks {
         let write_quorum = usize::try_from(write_quorum)
             .map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?;
         if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
-            rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, "not_reached");
             error!("reduce_read_quorum_errs: {:?}, bucket: {}, object: {}", &err, bucket, object);
             record_get_stage_duration_if_enabled(
                 GET_OBJECT_PATH_SET_DISK,
@@ -788,17 +755,7 @@ impl SetDisks {
         }
 
         let (op_online_disks, mut fi, fileinfo_selection_quorum) =
-            match Self::select_valid_fileinfo(&disks, &parts_metadata, &errs, vid.as_str(), read_quorum, write_quorum) {
-                Ok(selection) => selection,
-                Err(err) => {
-                    let err = StorageError::from(err);
-                    rustfs_io_metrics::record_get_object_metadata_quorum_result(
-                        metadata_metrics_path,
-                        metadata_quorum_outcome(&err),
-                    );
-                    return Err(err);
-                }
-            };
+            Self::select_valid_fileinfo(&disks, &parts_metadata, &errs, vid.as_str(), read_quorum, write_quorum)?;
         let include_part_checksums =
             opts.include_part_checksums || opts.part_number.is_some() || opts.data_movement || opts.raw_data_movement_read;
         if include_part_checksums {
@@ -821,16 +778,7 @@ impl SetDisks {
                 "metadata_read_error",
             )
             .await;
-        } else if use_metadata_cache
-            && (metadata_fanout_complete
-                || condition_stopped
-                || (read_data
-                    && allow_early_stop
-                    && metadata_fanout_diagnostics.valid_responses() >= read_quorum
-                    && errs.iter().all(Option::is_none)
-                    && fileinfo_selection_quorum >= read_quorum
-                    && op_online_disks.iter().filter(|disk| disk.is_some()).count() >= read_quorum))
-        {
+        } else if use_metadata_cache && (metadata_fanout_complete || condition_stopped) {
             #[cfg(test)]
             metadata_cache_tests::wait_before_metadata_cache_publish(bucket, object).await;
             self.cache_get_object_fileinfo(
@@ -840,10 +788,8 @@ impl SetDisks {
                     &fi,
                     if metadata_fanout_complete {
                         GetObjectMetadataCacheCapability::Full
-                    } else if condition_stopped {
-                        GetObjectMetadataCacheCapability::ConditionOnly
                     } else {
-                        GetObjectMetadataCacheCapability::QuorumSelected
+                        GetObjectMetadataCacheCapability::ConditionOnly
                     },
                 ),
                 &parts_metadata,
@@ -2704,7 +2650,6 @@ fn is_get_object_metadata_cache_request_eligible(bucket: &str, opts: &ObjectOpti
 #[cfg(test)]
 mod metadata_cache_tests {
     use super::*;
-    use crate::set_disk::core::io_primitives::disk_call_counters;
     use rustfs_heal_contracts::heal_channel::HealAdmissionDropReason;
     use serial_test::serial;
     use std::sync::atomic::{AtomicUsize, Ordering};
@@ -4064,7 +4009,7 @@ mod metadata_cache_tests {
     }
 
     #[tokio::test]
-    #[serial(metadata_cache_early_stop)]
+    #[serial(metadata_cache_publish_barrier)]
     async fn metadata_cache_production_fanout_cannot_publish_after_invalidation() {
         // Isolated context: an ambient DistErasure window (another test's
         // SetupTypeGuard) would bypass metadata-cache publication entirely
@@ -4118,191 +4063,6 @@ mod metadata_cache_tests {
         );
     }
 
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    #[serial(metadata_cache_early_stop)]
-    async fn metadata_cache_publishes_quorum_selected_early_stop_result() {
-        use tokio::io::AsyncReadExt;
-
-        let isolated_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
-        isolated_ctx
-            .update_erasure_type(crate::layout::endpoints::SetupType::Erasure)
-            .await;
-        let (dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, isolated_ctx).await;
-        let bucket = "metadata-cache-early-stop";
-        let object = "objects/repeated-key";
-        let disks = set.disks.read().await.clone();
-        for disk in disks.iter().flatten() {
-            disk.make_volume(bucket)
-                .await
-                .expect("test bucket should be created on each disk");
-        }
-        let expected_body = vec![0x5a; 1024 * 1024];
-        set.put_object(
-            bucket,
-            object,
-            &mut PutObjReader::from_vec(expected_body.clone()),
-            &ObjectOptions::default(),
-        )
-        .await
-        .expect("test object should be written before enabling the slow-tail fault");
-
-        let calls = disk_call_counters::observe(object);
-        temp_env::async_with_vars(
-            [
-                ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
-                ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
-                ("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")),
-                ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("false")),
-                ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS", Some("500")),
-                ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS", Some("3")),
-                ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET", Some(bucket)),
-                ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX", Some("objects/")),
-            ],
-            async {
-                let first = tokio::time::timeout(
-                    Duration::from_millis(400),
-                    set.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, true),
-                )
-                .await
-                .expect("first metadata read should stop before the delayed peer")
-                .expect("read quorum should provide valid object metadata");
-                assert!(first.owned.is_some(), "the first lookup should resolve through disk metadata");
-                assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 4);
-                assert_eq!(calls.total(disk_call_counters::KIND_METADATA_SLOWTAIL_FAULT), 1);
-
-                let second = set
-                    .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, true)
-                    .await
-                    .expect("second lookup should use the quorum-published metadata cache entry");
-                assert!(second.shared_entry().is_some(), "the repeated GET should hit the published cache entry");
-                assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 4);
-
-                let cached = second.shared_entry().expect("second lookup should share cached metadata");
-                assert_eq!(cached.fi.name, object);
-                let cached_online_count = cached.online_disks.iter().filter(|disk| disk.is_some()).count();
-                assert_eq!(cached_online_count, 3, "the warm cache should contain three of four disks");
-                assert_eq!(cached.capability, GetObjectMetadataCacheCapability::QuorumSelected);
-
-                let data_dir = cached
-                    .fi
-                    .data_dir
-                    .expect("quorum cache retains the object data directory")
-                    .to_string();
-                let selected_disk_indices = cached
-                    .online_disks
-                    .iter()
-                    .enumerate()
-                    .filter_map(|(index, disk)| disk.as_ref().map(|_| index))
-                    .take(2)
-                    .collect::<Vec<_>>();
-                assert_eq!(selected_disk_indices.len(), 2, "the warmed cache should name enough selected shards");
-                for index in selected_disk_indices {
-                    tokio::fs::remove_file(dirs[index].path().join(bucket).join(object).join(&data_dir).join("part.1"))
-                        .await
-                        .expect("simulate loss of two cached selected shards");
-                }
-
-                let mut reader = set
-                    .get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
-                    .await
-                    .expect("a cached partial layout must recover through omitted disks");
-                let mut body = Vec::new();
-                reader
-                    .stream
-                    .read_to_end(&mut body)
-                    .await
-                    .expect("the GET should decode after refreshing the omitted disks");
-                assert_eq!(body, expected_body, "recovered body must be byte-exact");
-            },
-        )
-        .await;
-
-        let full_layout = set
-            .get_object_fileinfo_gated(bucket, object, &ObjectOptions::default(), false, false)
-            .await
-            .expect("read-before-write consumers must fetch a complete layout");
-        assert!(
-            full_layout.shared_entry().is_none(),
-            "a quorum-selected partial layout cannot satisfy mutation consumers"
-        );
-        assert!(
-            calls.total(disk_call_counters::KIND_READ_VERSION) > 4,
-            "mutation consumers must independently fan out after a quorum-only cache hit"
-        );
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    #[serial(metadata_cache_early_stop)]
-    async fn metadata_cache_early_stop_publication_is_fenced_by_invalidation() {
-        let isolated_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
-        isolated_ctx
-            .update_erasure_type(crate::layout::endpoints::SetupType::Erasure)
-            .await;
-        let (_dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, isolated_ctx).await;
-        let bucket = "metadata-cache-early-stop-fence";
-        let object = "objects/repeated-key";
-        let disks = set.disks.read().await.clone();
-        for disk in disks.iter().flatten() {
-            disk.make_volume(bucket)
-                .await
-                .expect("test bucket should be created on each disk");
-        }
-        set.put_object(
-            bucket,
-            object,
-            &mut PutObjReader::from_vec(vec![0x5a; 1024 * 1024]),
-            &ObjectOptions::default(),
-        )
-        .await
-        .expect("test object should be written before enabling the slow-tail fault");
-
-        temp_env::async_with_vars(
-            [
-                ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
-                ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
-                ("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")),
-                ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("false")),
-                ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS", Some("500")),
-                ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS", Some("3")),
-                ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET", Some(bucket)),
-                ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX", Some("objects/")),
-            ],
-            async {
-                let barrier = MetadataCachePublishBarrier::install(bucket, object);
-                let stale_generation = set
-                    .get_object_metadata_cache_generation(bucket, object)
-                    .expect("metadata cache generation should be active");
-                let reader_set = Arc::clone(&set);
-                let read = tokio::spawn(async move {
-                    reader_set
-                        .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, true)
-                        .await
-                });
-                barrier.wait_until_paused().await;
-                set.invalidate_get_object_metadata_cache(bucket, object).await;
-                barrier.release();
-
-                let snapshot = read
-                    .await
-                    .expect("early-stop metadata read should not panic")
-                    .expect("read quorum should resolve the selected FileInfo");
-                assert!(snapshot.owned.is_some());
-                assert!(
-                    set.get_object_metadata_cache
-                        .get(&GetObjectMetadataCacheKey::new(bucket, object, stale_generation))
-                        .await
-                        .is_none(),
-                    "early-stop publication must not retain an entry under the retired generation"
-                );
-                assert!(
-                    set.cached_get_object_fileinfo(bucket, object).await.is_none(),
-                    "invalidation must reject the quorum-selected early-stop publication"
-                );
-            },
-        )
-        .await;
-    }
-
     #[tokio::test]
     #[serial(metadata_cache_publish_barrier, body_cache_hook)]
     async fn conditional_inline_hedge_caches_only_terminal_metadata() {
@@ -5322,15 +5082,12 @@ mod metadata_cache_tests {
             .get_object_metadata_cache_generation("bucket-a", "object-a")
             .expect("metadata cache generation should be active");
         let first_key = GetObjectMetadataCacheKey::new("bucket-a", "object-a", generation);
-        let first_lookup = GetObjectMetadataCacheLookupKey::new("bucket-a", "object-a", generation);
         let second_key = GetObjectMetadataCacheKey {
             bucket: Arc::from("bucket-b"),
             object: Arc::from("object-b"),
             generation: generation.value,
             hash: generation.hash,
         };
-        let second_lookup = GetObjectMetadataCacheLookupKey::new("bucket-b", "object-b", generation);
-        let unrelated_lookup = GetObjectMetadataCacheLookupKey::new("bucket-c", "object-c", generation);
         let first_fi = valid_test_fileinfo("object-a");
         let second_fi = valid_test_fileinfo("object-b");
         let entry = |fi: FileInfo| {
@@ -5351,7 +5108,7 @@ mod metadata_cache_tests {
 
         assert_eq!(
             set.get_object_metadata_cache
-                .get(&first_lookup)
+                .get(&first_key)
                 .await
                 .expect("first colliding entry should remain addressable")
                 .fi
@@ -5360,14 +5117,13 @@ mod metadata_cache_tests {
         );
         assert_eq!(
             set.get_object_metadata_cache
-                .get(&second_lookup)
+                .get(&second_key)
                 .await
                 .expect("second colliding entry should remain addressable")
                 .fi
                 .name,
             "object-b"
         );
-        assert!(set.get_object_metadata_cache.get(&unrelated_lookup).await.is_none());
     }
 
     #[tokio::test]
@@ -5534,16 +5290,6 @@ mod tests {
     const CODEC_STREAMING_TEST_OBJECT: &str = "object";
     static CAPTURED_READ_REPAIR_CALLS: AtomicUsize = AtomicUsize::new(0);
 
-    #[test]
-    fn metadata_quorum_outcome_uses_bounded_error_categories() {
-        assert_eq!(metadata_quorum_outcome(&StorageError::ErasureReadQuorum), "not_reached");
-        assert_eq!(metadata_quorum_outcome(&StorageError::FileNotFound), "not_found");
-        assert_eq!(
-            metadata_quorum_outcome(&StorageError::Io(std::io::Error::other("disk unavailable"))),
-            "unavailable"
-        );
-    }
-
     fn capture_read_repair_submitter(
         _request: rustfs_heal_contracts::heal_channel::HealChannelRequest,
     ) -> ReadRepairAdmissionFuture {

crates/ecstore/src/store/heal.rs

@@ -404,17 +404,7 @@ impl ECStore {
         expected: uuid::Uuid,
         opts: &HealOpts,
     ) -> Result<HealResultItem> {
-        self.run_bucket_heal_at_incarnation(bucket, expected, opts, move |store, bucket, mut opts| async move {
-            if !opts.recreate && !opts.remove {
-                let mut missing = false;
-                for (pool, set) in store.heal_erasure_set_scopes(&opts).await? {
-                    missing |= store.pools[pool].get_disks(set).bucket_needs_reconstruction(&bucket).await?;
-                }
-                if missing {
-                    crate::bucket::metadata_sys::validate_bucket_reconstruction_in(&store.ctx, &bucket, expected).await?;
-                    opts.recreate = true;
-                }
-            }
+        self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
             store.heal_bucket(&bucket, &opts).await
         })
         .await
@@ -1461,178 +1451,6 @@ mod tests {
         .expect("minimal pool should build")
     }
 
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn admin_bucket_recovery_recreates_missing_directories_at_current_incarnation() {
-        let (root, store, shutdown) = multi_pool_heal_store().await;
-        let bucket = "admin-bucket-recovery";
-        store
-            .make_bucket(bucket, &MakeBucketOptions::default())
-            .await
-            .expect("create a committed bucket generation");
-        let incarnation = store
-            .bucket_incarnation_id_from_disk(bucket)
-            .await
-            .expect("persisted generation");
-        for disk in 1..4 {
-            tokio::fs::remove_dir(root.path().join(format!("pool0-disk{disk}/{bucket}")))
-                .await
-                .expect("remove only an empty fixture bucket directory");
-        }
-        store
-            .heal_bucket_at_incarnation(bucket, incarnation, &HealOpts::default())
-            .await
-            .expect("default administrator recovery should use committed generation authority");
-        for disk in 0..4 {
-            assert!(root.path().join(format!("pool0-disk{disk}/{bucket}")).is_dir());
-        }
-        assert_eq!(
-            store
-                .bucket_incarnation_id_from_disk(bucket)
-                .await
-                .expect("recovered generation"),
-            incarnation
-        );
-        shutdown.cancel();
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn admin_bucket_recovery_rejects_missing_corrupt_uncommitted_and_retired_authority() {
-        let (root, store, shutdown) = multi_pool_heal_store().await;
-        for state in ["missing", "corrupt", "uncommitted", "mismatched", "retired"] {
-            let bucket = format!("recovery-authority-{state}");
-            store
-                .make_bucket(&bucket, &MakeBucketOptions::default())
-                .await
-                .expect("create fixture generation");
-            let incarnation = store
-                .bucket_incarnation_id_from_disk(&bucket)
-                .await
-                .expect("fixture identity");
-            let (mut metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&store.ctx, &bucket)
-                .await
-                .expect("committed metadata");
-            assert!(persisted && metadata.bucket_creation_committed);
-            let path = metadata.save_file_path();
-            match state {
-                "missing" => delete_config(store.clone(), &path).await.expect("remove metadata authority"),
-                "corrupt" => save_config(store.clone(), &path, b"corrupt bucket metadata".to_vec())
-                    .await
-                    .expect("corrupt authority"),
-                "uncommitted" | "mismatched" => {
-                    if state == "uncommitted" {
-                        metadata.bucket_creation_committed = false;
-                    } else {
-                        metadata.bucket_incarnation_id = Uuid::new_v4();
-                    }
-                    metadata
-                        .save_with_store_committed(store.clone())
-                        .await
-                        .expect("inject inconsistent metadata authority");
-                }
-                "retired" => {
-                    // A returning disk may retain a directory after retirement was committed.
-                    crate::bucket::retirement::commit_retirement(store.clone(), &bucket, incarnation, &ObjectOptions::default())
-                        .await
-                        .expect("persist retirement evidence");
-                }
-                _ => unreachable!("fixed fixture states"),
-            }
-            for disk in 1..4 {
-                tokio::fs::remove_dir(root.path().join(format!("pool0-disk{disk}/{bucket}")))
-                    .await
-                    .expect("lose fixture directories");
-            }
-            assert!(
-                store
-                    .heal_bucket_at_incarnation(&bucket, incarnation, &HealOpts::default())
-                    .await
-                    .is_err(),
-                "must reject {state}"
-            );
-            for disk in 1..4 {
-                assert!(
-                    !root.path().join(format!("pool0-disk{disk}/{bucket}")).exists(),
-                    "{state} cannot authorize reconstruction"
-                );
-            }
-        }
-        shutdown.cancel();
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn admin_bucket_recovery_rejects_offline_disks_and_fully_missing_sets() {
-        for offline in [false, true] {
-            let (root, store, shutdown) = multi_pool_heal_store().await;
-            let bucket = "recovery-unproven-set";
-            store
-                .make_bucket(bucket, &MakeBucketOptions::default())
-                .await
-                .expect("fixture bucket");
-            let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("fixture identity");
-            let first_missing = usize::from(offline);
-            for disk in first_missing..4 {
-                tokio::fs::remove_dir(root.path().join(format!("pool0-disk{disk}/{bucket}")))
-                    .await
-                    .expect("remove fixture directories");
-            }
-            if offline {
-                let disk = store.pools[0].disk_set[0].disks.read().await[0]
-                    .clone()
-                    .expect("online fixture disk");
-                disk.close().await.expect("stop disk monitoring");
-                disk.force_offline_for_test();
-            }
-            assert!(
-                store
-                    .heal_bucket_at_incarnation(bucket, incarnation, &HealOpts::default())
-                    .await
-                    .is_err()
-            );
-            for disk in first_missing..4 {
-                assert!(!root.path().join(format!("pool0-disk{disk}/{bucket}")).exists());
-            }
-            shutdown.cancel();
-        }
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn admin_bucket_recovery_dry_run_preserves_missing_directories() {
-        let (root, store, shutdown) = multi_pool_heal_store().await;
-        let bucket = "recovery-dry-run";
-        store
-            .make_bucket(bucket, &MakeBucketOptions::default())
-            .await
-            .expect("fixture bucket");
-        let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("fixture identity");
-        for disk in 1..4 {
-            tokio::fs::remove_dir(root.path().join(format!("pool0-disk{disk}/{bucket}")))
-                .await
-                .expect("remove fixture directories");
-        }
-        store
-            .heal_bucket_at_incarnation(
-                bucket,
-                incarnation,
-                &HealOpts {
-                    dry_run: true,
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("dry run can inspect eligible recovery");
-        for disk in 1..4 {
-            assert!(
-                !root.path().join(format!("pool0-disk{disk}/{bucket}")).exists(),
-                "dry run must not reconstruct directories"
-            );
-        }
-        shutdown.cancel();
-    }
-
     #[tokio::test]
     #[serial_test::serial]
     async fn bucket_incarnation_heal_preserves_successor_shards_and_allows_fresh_repair() {

crates/ecstore/src/store/multipart.rs

@@ -32,42 +32,6 @@ fn data_movement_multipart_discovery_counts() -> &'static std::sync::Mutex<std::
     DATA_MOVEMENT_MULTIPART_DISCOVERY_COUNTS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
 }
 
-#[cfg(test)]
-type OrphanMultipartCleanupPause = (tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>);
-
-#[cfg(test)]
-static ORPHAN_MULTIPART_CLEANUP_PAUSES: std::sync::OnceLock<
-    std::sync::Mutex<std::collections::HashMap<String, OrphanMultipartCleanupPause>>,
-> = std::sync::OnceLock::new();
-
-#[cfg(test)]
-pub(crate) fn pause_orphan_multipart_cleanup_for_test(
-    bucket: &str,
-) -> (tokio::sync::oneshot::Receiver<()>, tokio::sync::oneshot::Sender<()>) {
-    let (arrived_tx, arrived_rx) = tokio::sync::oneshot::channel();
-    let (release_tx, release_rx) = tokio::sync::oneshot::channel();
-    let previous = ORPHAN_MULTIPART_CLEANUP_PAUSES
-        .get_or_init(Default::default)
-        .lock()
-        .expect("orphan cleanup pause mutex should not poison")
-        .insert(bucket.to_owned(), (arrived_tx, release_rx));
-    assert!(previous.is_none(), "each bucket has at most one orphan cleanup pause");
-    (arrived_rx, release_tx)
-}
-
-#[cfg(test)]
-async fn pause_orphan_multipart_cleanup(bucket: &str) {
-    let pause = ORPHAN_MULTIPART_CLEANUP_PAUSES
-        .get_or_init(Default::default)
-        .lock()
-        .expect("orphan cleanup pause mutex should not poison")
-        .remove(bucket);
-    if let Some((arrived, release)) = pause {
-        let _ = arrived.send(());
-        let _ = release.await;
-    }
-}
-
 fn decommission_multipart_target_clear_pending(opts: &ObjectOptions, target: Option<&ObjectInfo>) -> Result<bool> {
     let expected_mod_time = opts.mod_time.ok_or_else(|| Error::DecommissionCapacityBlocked {
         message: "multipart cleanup cannot prove exact target absence without a modification time".to_string(),
@@ -690,10 +654,7 @@ impl ECStore {
         opts: &ObjectOptions,
     ) -> Result<PartInfo> {
         check_put_object_part_args(bucket, object, upload_id)?;
-        let (mut opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
-        if let Some(guard) = bucket_lifecycle_guard {
-            opts.bucket_lifecycle_lock_owner = Some(Arc::new(guard));
-        }
+        let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
         if !self.single_pool() {
             opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
         }
@@ -789,12 +750,19 @@ impl ECStore {
         opts: &ObjectOptions,
     ) -> Result<()> {
         check_abort_multipart_args(bucket, object, upload_id)?;
-        let (opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
+        let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
+        let opts = &opts;
+
+        // TODO(backlog): defer DeleteUploadID to background for faster abort response
+
+        if self.single_pool() {
+            return self.pools[0].abort_multipart_upload(bucket, object, upload_id, opts).await;
+        }
 
         for pool_idx in self.existing_multipart_pool_order().await {
             let pool = &self.pools[pool_idx];
 
-            let err = match pool.abort_multipart_upload(bucket, object, upload_id, &opts).await {
+            let err = match pool.abort_multipart_upload(bucket, object, upload_id, opts).await {
                 Ok(_) => return Ok(()),
                 Err(err) => {
                     //
@@ -807,92 +775,7 @@ impl ECStore {
             }
         }
 
-        self.recover_orphan_multipart_upload(bucket, object, upload_id, opts, bucket_lifecycle_guard)
-            .await
-    }
-
-    async fn recover_orphan_multipart_upload(
-        &self,
-        bucket: &str,
-        object: &str,
-        upload_id: &str,
-        opts: ObjectOptions,
-        bucket_lifecycle_guard: Option<rustfs_lock::NamespaceLockGuard>,
-    ) -> Result<()> {
-        let invalid_upload = || StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned());
-        if self.ctx.lock_manager().is_disabled() {
-            return Err(invalid_upload());
-        }
-        let Some(bucket_lifecycle_guard) = bucket_lifecycle_guard else {
-            // A signal borrowed from an enclosing operation cannot keep its lock
-            // alive when the request waiter disappears during physical cleanup.
-            return Err(invalid_upload());
-        };
-        let Some(expected_incarnation) = opts.expected_bucket_incarnation_id.filter(|id| !id.is_nil()) else {
-            return Err(invalid_upload());
-        };
-        // Recovery is a side path of an idempotent Abort. Until the first
-        // deletion is issued, anything that stops staging absence from being
-        // proven (an offline disk, lock contention, partially overlapping lock
-        // domains, unreadable bucket metadata) keeps the upload reported as
-        // gone, exactly as before recovery existed; only failures inside the
-        // deletion task are request failures, because a retry can finish them.
-        let (metadata, persisted) =
-            match crate::bucket::metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await {
-                Ok(loaded) => loaded,
-                Err(error) => {
-                    debug!(bucket, object, upload_id, %error, "orphan multipart cleanup skipped: bucket metadata unavailable");
-                    return Err(invalid_upload());
-                }
-            };
-        if !persisted
-            || !metadata.bucket_creation_committed
-            || !metadata.bucket_incarnation_sidecar
-            || metadata.bucket_incarnation_id != expected_incarnation
-        {
-            return Err(invalid_upload());
-        }
-
-        // Lock order: bucket lifecycle, then upload locks in pool-index order.
-        // Prove metadata absence across every pool before deleting any staging.
-        let mut cleanup_guards = Vec::with_capacity(self.pools.len());
-        for pool in &self.pools {
-            let guard = match pool
-                .get_disks_by_key(object)
-                .lock_orphan_multipart_cleanup(bucket, object, upload_id, &opts, &cleanup_guards)
-                .await
-            {
-                Ok(guard) => guard,
-                Err(error) => {
-                    if !is_err_invalid_upload_id(&error) {
-                        debug!(bucket, object, upload_id, %error, "orphan multipart cleanup skipped: staging absence not proven");
-                    }
-                    return Err(invalid_upload());
-                }
-            };
-            cleanup_guards.push(guard);
-        }
-        let bucket = bucket.to_owned();
-        let object = object.to_owned();
-        let upload_id = upload_id.to_owned();
-        tokio::spawn(async move {
-            // Own both the real bucket guard and all upload guards until every
-            // disk deletion settles, including after cancellation of the waiter.
-            let _bucket_lifecycle_guard = bucket_lifecycle_guard;
-            #[cfg(test)]
-            pause_orphan_multipart_cleanup(&bucket).await;
-            let mut recovered = false;
-            for guard in &cleanup_guards {
-                recovered |= guard.delete(&bucket, &object, &opts).await?;
-            }
-            if recovered {
-                Ok(())
-            } else {
-                Err(StorageError::InvalidUploadID(bucket, object, upload_id))
-            }
-        })
-        .await
-        .map_err(|error| Error::other_with_context("orphan multipart cleanup task failed", error))?
+        Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
     }
 
     pub(crate) async fn abort_multipart_upload_for_data_movement(

crates/ecstore/src/store/peer.rs

@@ -18,11 +18,6 @@ use crate::disk::error::{DiskError, Result as DiskResult};
 use crate::disk::{DeleteOptions, Disk, RenameDataGuards, RenameDataResp};
 use crate::runtime::instance::{InstanceContext, NamespaceCommitGuard};
 use crate::runtime::sources as runtime_sources;
-use rustfs_io_metrics::{
-    PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT, PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME,
-    PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP, PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD,
-    PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL,
-};
 use tracing::{debug, error};
 
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
@@ -336,38 +331,15 @@ async fn rename_local_data_with_ctx(
     destination: (&str, &str),
     mut guards: RenameDataGuards,
 ) -> DiskResult<RenameDataResp> {
-    let handler_started = rustfs_io_metrics::put_stage_timer();
-    let lookup_started = rustfs_io_metrics::put_stage_timer();
     let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
-    let lookup_ms = lookup_started.map(|started| started.elapsed().as_secs_f64() * 1000.0);
-    if let Some(duration_ms) = lookup_ms {
-        rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP, duration_ms);
-    }
     let mutates_namespace = !is_meta_bucketname(source.0) || !is_meta_bucketname(destination.0);
-    let admit_started = rustfs_io_metrics::put_stage_timer();
     let owner = admit_local_disk(ctx, &disk, disk_id, mutates_namespace).await?;
-    let admit_ms = admit_started.map(|started| started.elapsed().as_secs_f64() * 1000.0);
-    if let Some(duration_ms) = admit_ms {
-        rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT, duration_ms);
-    }
     guards.namespace_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
-    let local_rename_started = rustfs_io_metrics::put_stage_timer();
     let result = disk
         .rename_data_borrowed_with_fence_observed(source.0, source.1, fi, destination.0, destination.1, guards)
         .await
         .result;
-    let local_rename_ms = local_rename_started.map(|started| started.elapsed().as_secs_f64() * 1000.0);
-    if let Some(duration_ms) = local_rename_ms {
-        rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME, duration_ms);
-    }
     drop(owner);
-    if let Some(handler_started) = handler_started {
-        let handler_ms = handler_started.elapsed().as_secs_f64() * 1000.0;
-        rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL, handler_ms);
-        let child_ms = lookup_ms.unwrap_or(0.0) + admit_ms.unwrap_or(0.0) + local_rename_ms.unwrap_or(0.0);
-        let overhead_ms = (handler_ms - child_ms).max(0.0);
-        rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD, overhead_ms);
-    }
     result
 }
 
@@ -1697,9 +1669,7 @@ mod tests {
     #[serial_test::serial]
     async fn target_ordinary_timeout_keeps_its_physical_namespace_owner() {
         use crate::disk::os::prepared_publication_test_hooks as hooks;
-        // Allow real fixture I/O to finish under load. Expire the operation
-        // with virtual time only after its physical publication is paused.
-        temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60"))], async {
+        temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("1"))], async {
             let ctx = Arc::new(InstanceContext::new());
             let store = super::super::tests::build_store_with_ctx(ctx.clone());
             let root = tempfile::tempdir().expect("root");
@@ -1733,7 +1703,7 @@ mod tests {
             .await
             .expect("bounded entry");
             tokio::time::pause();
-            tokio::time::advance(std::time::Duration::from_secs(61)).await;
+            tokio::time::advance(std::time::Duration::from_secs(2)).await;
             tokio::time::resume();
             let result = tokio::time::timeout(std::time::Duration::from_secs(5), &mut rename)
                 .await

crates/ecstore/tests/bucket_config_goldens.rs

@@ -0,0 +1,497 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Byte-level goldens for the 13 persisted bucket-configuration XML families
+//! (rustfs/backlog#2744).
+//!
+//! Responsible for: pinning the exact bytes that `deserialize` followed by
+//! `serialize` produces for every family in its three source shapes (AWS
+//! documentation shape, MinIO-written shape, RustFS-written shape), and the
+//! exact bytes RustFS writes for a typed value. Only bytes are compared; a
+//! structural comparison would let a codec replacement change the persisted
+//! encoding without failing here.
+//!
+//! Not responsible for: validating configurations, touching the bucket
+//! metadata store, or choosing a canonical shape.
+//!
+//! Upstream: the bucket-config `deserialize`/`serialize` pair, reached through the
+//! integration-test facade `tests/storage_api.rs`, and the s3s DTOs they are generic over. Downstream: the DTO/codec
+//! replacement (T1.4) and the persisted-bytes comparison (T3.7), which must
+//! keep every golden green without regenerating it. Regeneration is reserved
+//! to `scripts/gen_bucket_config_goldens.sh`, which sets
+//! `BUCKET_CONFIG_GOLDENS_WRITE`: in that mode every test writes its files
+//! and then fails on purpose, so a write run never prints a green line.
+
+mod storage_api;
+
+use s3s::dto::{
+    AbortIncompleteMultipartUpload, AccelerateConfiguration, BucketAccelerateStatus, BucketLifecycleConfiguration,
+    BucketLoggingStatus, BucketVersioningStatus, CORSConfiguration, CORSRule, Condition, DefaultRetention, DelMarkerExpiration,
+    DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination,
+    ErrorDocument, Event, ExcludedPrefix, ExistingObjectReplication, ExistingObjectReplicationStatus, ExpirationStatus,
+    FilterRule, FilterRuleName, IndexDocument, LifecycleExpiration, LifecycleRule, LifecycleRuleAndOperator, LifecycleRuleFilter,
+    LoggingEnabled, NoncurrentVersionExpiration, NotificationConfiguration, NotificationConfigurationFilter,
+    ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule, Payer, PublicAccessBlockConfiguration,
+    QueueConfiguration, Redirect, ReplicaModifications, ReplicaModificationsStatus, ReplicationConfiguration, ReplicationRule,
+    ReplicationRuleAndOperator, ReplicationRuleFilter, ReplicationRuleStatus, RequestPaymentConfiguration, RoutingRule,
+    S3KeyFilter, ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration,
+    ServerSideEncryptionRule, SourceSelectionCriteria, StorageClass, Tag, Tagging, Transition, TransitionStorageClass,
+    VersioningConfiguration, WebsiteConfiguration,
+};
+use s3s::xml;
+use std::path::{Path, PathBuf};
+use storage_api::bucket_config_codec::{deserialize, serialize};
+
+/// Set only by `scripts/gen_bucket_config_goldens.sh`. Every pair test then
+/// writes its `.out.xml` (and, for the `rustfs` shape, its `.in.xml`) and
+/// fails on purpose, so a write run can never be mistaken for a verification.
+const WRITE_ENV: &str = "BUCKET_CONFIG_GOLDENS_WRITE";
+
+fn goldens_dir() -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("tests")
+        .join("fixtures")
+        .join("bucket-config-goldens")
+}
+
+fn pair_path(family: &str, shape: &str, side: &str) -> PathBuf {
+    goldens_dir().join(family).join(format!("{shape}.{side}.xml"))
+}
+
+fn read(path: &Path) -> Vec<u8> {
+    std::fs::read(path).unwrap_or_else(|err| panic!("read {}: {err}", path.display()))
+}
+
+fn write(path: &Path, bytes: &[u8]) {
+    std::fs::write(path, bytes).unwrap_or_else(|err| panic!("write {}: {err}", path.display()));
+}
+
+fn contains(haystack: &[u8], needle: &[u8]) -> bool {
+    haystack.windows(needle.len()).any(|window| window == needle)
+}
+
+fn round_trip<T>(input: &[u8], label: &str) -> Vec<u8>
+where
+    T: for<'xml> xml::Deserialize<'xml> + xml::Serialize,
+{
+    let parsed: T = deserialize(input).unwrap_or_else(|err| panic!("{label}: deserialize failed: {err}"));
+    serialize(&parsed).unwrap_or_else(|err| panic!("{label}: serialize failed: {err}"))
+}
+
+/// Byte comparison with a readable report. Parsed values are never compared.
+fn assert_bytes_eq(actual: &[u8], expected: &[u8], context: &str) {
+    if actual != expected {
+        panic!(
+            "{context}\n--- expected ({} bytes) ---\n{}\n--- actual ({} bytes) ---\n{}\n",
+            expected.len(),
+            String::from_utf8_lossy(expected),
+            actual.len(),
+            String::from_utf8_lossy(actual),
+        );
+    }
+}
+
+/// Checks one `<family>/<shape>` pair. `typed` is the value RustFS writes for
+/// the `rustfs` shape; the `aws` and `minio` shapes pass `None`.
+fn check_pair<T>(family: &str, shape: &str, typed: Option<&T>)
+where
+    T: for<'xml> xml::Deserialize<'xml> + xml::Serialize,
+{
+    let in_path = pair_path(family, shape, "in");
+    let out_path = pair_path(family, shape, "out");
+    let label = format!("{family}/{shape}");
+
+    if std::env::var_os(WRITE_ENV).is_some() {
+        if let Some(value) = typed {
+            let written = serialize(value).unwrap_or_else(|err| panic!("{label}: typed value failed to serialize: {err}"));
+            write(&in_path, &written);
+        }
+        let input = read(&in_path);
+        write(&out_path, &round_trip::<T>(&input, &label));
+        panic!(
+            "{WRITE_ENV} is set: wrote {} without verifying anything; rerun without {WRITE_ENV}",
+            out_path.display()
+        );
+    }
+
+    let input = read(&in_path);
+    let expected = read(&out_path);
+    if let Some(value) = typed {
+        let written = serialize(value).unwrap_or_else(|err| panic!("{label}: typed value failed to serialize: {err}"));
+        assert_bytes_eq(
+            &written,
+            &input,
+            &format!(
+                "{label}: RustFS no longer writes the pinned bytes for the typed value ({})",
+                in_path.display()
+            ),
+        );
+    }
+    let actual = round_trip::<T>(&input, &label);
+    assert_bytes_eq(
+        &actual,
+        &expected,
+        &format!("{label}: deserialize->serialize bytes drifted from {}", out_path.display()),
+    );
+    let again = round_trip::<T>(&expected, &label);
+    assert_bytes_eq(
+        &again,
+        &expected,
+        &format!("{label}: the golden is not a fixed point of deserialize->serialize"),
+    );
+}
+
+/// The typed values behind every `rustfs.in.xml`: what RustFS writes today
+/// through `serialize` when it persists a configuration it built itself.
+/// MinIO-only members (`DelMarkerExpiration`, `ExcludeFolders`,
+/// `ExcludedPrefixes`, `DeleteReplication`) are included where RustFS accepts
+/// them, so the `rustfs` shape is the one that pins their encoding.
+mod rustfs_values {
+    use super::*;
+
+    fn tag(key: &str, value: &str) -> Tag {
+        Tag {
+            key: Some(key.to_owned()),
+            value: Some(value.to_owned()),
+        }
+    }
+
+    pub(super) fn notification() -> NotificationConfiguration {
+        NotificationConfiguration {
+            queue_configurations: Some(vec![QueueConfiguration {
+                events: vec![
+                    Event::from("s3:ObjectCreated:*".to_owned()),
+                    Event::from("s3:ObjectRemoved:*".to_owned()),
+                ],
+                filter: Some(NotificationConfigurationFilter {
+                    key: Some(S3KeyFilter {
+                        filter_rules: Some(vec![FilterRule {
+                            name: Some(FilterRuleName::from_static(FilterRuleName::PREFIX)),
+                            value: Some("uploads/".to_owned()),
+                        }]),
+                    }),
+                }),
+                id: Some("rustfs-webhook".to_owned()),
+                queue_arn: "arn:rustfs:sqs::primary:webhook".to_owned(),
+            }]),
+            ..Default::default()
+        }
+    }
+
+    pub(super) fn lifecycle() -> BucketLifecycleConfiguration {
+        BucketLifecycleConfiguration {
+            rules: vec![
+                LifecycleRule {
+                    abort_incomplete_multipart_upload: Some(AbortIncompleteMultipartUpload {
+                        days_after_initiation: Some(3),
+                    }),
+                    del_marker_expiration: Some(DelMarkerExpiration { days: Some(7) }),
+                    expiration: Some(LifecycleExpiration {
+                        days: Some(30),
+                        ..Default::default()
+                    }),
+                    filter: Some(LifecycleRuleFilter {
+                        prefix: Some("tmp/".to_owned()),
+                        ..Default::default()
+                    }),
+                    id: Some("expire-tmp".to_owned()),
+                    noncurrent_version_expiration: Some(NoncurrentVersionExpiration {
+                        newer_noncurrent_versions: Some(2),
+                        noncurrent_days: Some(7),
+                    }),
+                    noncurrent_version_transitions: None,
+                    prefix: None,
+                    status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                    transitions: None,
+                },
+                LifecycleRule {
+                    abort_incomplete_multipart_upload: None,
+                    del_marker_expiration: None,
+                    expiration: None,
+                    filter: Some(LifecycleRuleFilter {
+                        and: Some(LifecycleRuleAndOperator {
+                            prefix: Some("archive/".to_owned()),
+                            tags: Some(vec![tag("tier", "cold")]),
+                            ..Default::default()
+                        }),
+                        ..Default::default()
+                    }),
+                    id: Some("tier-cold".to_owned()),
+                    noncurrent_version_expiration: None,
+                    noncurrent_version_transitions: None,
+                    prefix: None,
+                    status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                    transitions: Some(vec![Transition {
+                        days: Some(90),
+                        storage_class: Some(TransitionStorageClass::from_static("COLDTIER")),
+                        ..Default::default()
+                    }]),
+                },
+            ],
+            ..Default::default()
+        }
+    }
+
+    pub(super) fn object_lock() -> ObjectLockConfiguration {
+        ObjectLockConfiguration {
+            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            rule: Some(ObjectLockRule {
+                default_retention: Some(DefaultRetention {
+                    days: Some(30),
+                    mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE)),
+                    years: None,
+                }),
+            }),
+        }
+    }
+
+    pub(super) fn versioning() -> VersioningConfiguration {
+        VersioningConfiguration {
+            exclude_folders: Some(true),
+            excluded_prefixes: Some(vec![
+                ExcludedPrefix {
+                    prefix: Some("tmp/".to_owned()),
+                },
+                ExcludedPrefix {
+                    prefix: Some("cache/".to_owned()),
+                },
+            ]),
+            mfa_delete: None,
+            status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
+        }
+    }
+
+    pub(super) fn encryption() -> ServerSideEncryptionConfiguration {
+        ServerSideEncryptionConfiguration {
+            rules: vec![ServerSideEncryptionRule {
+                apply_server_side_encryption_by_default: Some(ServerSideEncryptionByDefault {
+                    kms_master_key_id: Some("rustfs-bucket-key".to_owned()),
+                    sse_algorithm: ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS),
+                }),
+                blocked_encryption_types: None,
+                bucket_key_enabled: Some(false),
+            }],
+        }
+    }
+
+    pub(super) fn tagging() -> Tagging {
+        Tagging {
+            tag_set: vec![tag("team", "storage"), tag("env", "staging")],
+        }
+    }
+
+    pub(super) fn replication() -> ReplicationConfiguration {
+        ReplicationConfiguration {
+            role: String::new(),
+            rules: vec![ReplicationRule {
+                delete_marker_replication: Some(DeleteMarkerReplication {
+                    status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
+                }),
+                delete_replication: Some(DeleteReplication {
+                    status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
+                }),
+                destination: Destination {
+                    bucket: "arn:rustfs:replication::0b6f3c2a-5e6a-4d2b-9c1e-7a8f9d0e1b2c:interop-dr".to_owned(),
+                    storage_class: Some(StorageClass::from_static(StorageClass::STANDARD)),
+                    ..Default::default()
+                },
+                existing_object_replication: Some(ExistingObjectReplication {
+                    status: ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED),
+                }),
+                filter: Some(ReplicationRuleFilter {
+                    and: Some(ReplicationRuleAndOperator {
+                        prefix: Some("docs/".to_owned()),
+                        tags: Some(vec![tag("replicate", "yes")]),
+                    }),
+                    ..Default::default()
+                }),
+                id: Some("rustfs-dr".to_owned()),
+                prefix: None,
+                priority: Some(1),
+                source_selection_criteria: Some(SourceSelectionCriteria {
+                    replica_modifications: Some(ReplicaModifications {
+                        status: ReplicaModificationsStatus::from_static(ReplicaModificationsStatus::ENABLED),
+                    }),
+                    sse_kms_encrypted_objects: None,
+                }),
+                status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
+            }],
+        }
+    }
+
+    pub(super) fn cors() -> CORSConfiguration {
+        CORSConfiguration {
+            cors_rules: vec![CORSRule {
+                allowed_headers: Some(vec!["Authorization".to_owned(), "Content-Type".to_owned()]),
+                allowed_methods: vec!["GET".to_owned(), "PUT".to_owned()],
+                allowed_origins: vec!["https://app.example.internal".to_owned()],
+                expose_headers: Some(vec!["ETag".to_owned(), "x-amz-request-id".to_owned()]),
+                id: Some("rustfs-app".to_owned()),
+                max_age_seconds: Some(3600),
+            }],
+        }
+    }
+
+    pub(super) fn logging() -> BucketLoggingStatus {
+        BucketLoggingStatus {
+            logging_enabled: Some(LoggingEnabled {
+                target_bucket: "rustfs-access-logs".to_owned(),
+                target_grants: None,
+                target_object_key_format: None,
+                target_prefix: "buckets/interop/".to_owned(),
+            }),
+        }
+    }
+
+    pub(super) fn website() -> WebsiteConfiguration {
+        WebsiteConfiguration {
+            error_document: Some(ErrorDocument {
+                key: "404.html".to_owned(),
+            }),
+            index_document: Some(IndexDocument {
+                suffix: "index.html".to_owned(),
+            }),
+            redirect_all_requests_to: None,
+            routing_rules: Some(vec![RoutingRule {
+                condition: Some(Condition {
+                    http_error_code_returned_equals: None,
+                    key_prefix_equals: Some("legacy/".to_owned()),
+                }),
+                redirect: Redirect {
+                    host_name: None,
+                    http_redirect_code: Some("301".to_owned()),
+                    protocol: None,
+                    replace_key_prefix_with: Some("current/".to_owned()),
+                    replace_key_with: None,
+                },
+            }]),
+        }
+    }
+
+    pub(super) fn accelerate() -> AccelerateConfiguration {
+        AccelerateConfiguration {
+            status: Some(BucketAccelerateStatus::from_static(BucketAccelerateStatus::SUSPENDED)),
+        }
+    }
+
+    pub(super) fn request_payment() -> RequestPaymentConfiguration {
+        RequestPaymentConfiguration {
+            payer: Payer::from_static(Payer::BUCKET_OWNER),
+        }
+    }
+
+    pub(super) fn public_access_block() -> PublicAccessBlockConfiguration {
+        PublicAccessBlockConfiguration {
+            block_public_acls: Some(true),
+            block_public_policy: Some(false),
+            ignore_public_acls: Some(true),
+            restrict_public_buckets: Some(false),
+        }
+    }
+}
+
+/// One test per `<family>/<shape>` pair: 13 families x 3 shapes = 39.
+mod bucket_config_goldens {
+    use super::*;
+
+    macro_rules! golden {
+        ($name:ident, $family:literal, $shape:literal, $ty:ty) => {
+            #[test]
+            fn $name() {
+                check_pair::<$ty>($family, $shape, None);
+            }
+        };
+        ($name:ident, $family:literal, rustfs = $value:expr) => {
+            #[test]
+            fn $name() {
+                check_pair($family, "rustfs", Some(&$value));
+            }
+        };
+    }
+
+    golden!(notification_aws, "notification", "aws", NotificationConfiguration);
+    golden!(notification_minio, "notification", "minio", NotificationConfiguration);
+    golden!(notification_rustfs, "notification", rustfs = rustfs_values::notification());
+
+    golden!(lifecycle_aws, "lifecycle", "aws", BucketLifecycleConfiguration);
+    golden!(lifecycle_minio, "lifecycle", "minio", BucketLifecycleConfiguration);
+    golden!(lifecycle_rustfs, "lifecycle", rustfs = rustfs_values::lifecycle());
+
+    golden!(object_lock_aws, "object_lock", "aws", ObjectLockConfiguration);
+    golden!(object_lock_minio, "object_lock", "minio", ObjectLockConfiguration);
+    golden!(object_lock_rustfs, "object_lock", rustfs = rustfs_values::object_lock());
+
+    golden!(versioning_aws, "versioning", "aws", VersioningConfiguration);
+    golden!(versioning_minio, "versioning", "minio", VersioningConfiguration);
+    golden!(versioning_rustfs, "versioning", rustfs = rustfs_values::versioning());
+
+    golden!(encryption_aws, "encryption", "aws", ServerSideEncryptionConfiguration);
+    golden!(encryption_minio, "encryption", "minio", ServerSideEncryptionConfiguration);
+    golden!(encryption_rustfs, "encryption", rustfs = rustfs_values::encryption());
+
+    golden!(tagging_aws, "tagging", "aws", Tagging);
+    golden!(tagging_minio, "tagging", "minio", Tagging);
+    golden!(tagging_rustfs, "tagging", rustfs = rustfs_values::tagging());
+
+    /// `replication/aws.in.xml` is the unknown-element sample from
+    /// `crates/replication/src/config.rs`
+    /// (`s3_xml_parser_discards_unknown_replication_elements_before_validation`).
+    /// Beyond the byte golden, the pair proves the root-level `FutureTopLevel`
+    /// element is skipped on read and never re-emitted on write.
+    #[test]
+    fn replication_aws() {
+        check_pair::<ReplicationConfiguration>("replication", "aws", None);
+        let input = read(&pair_path("replication", "aws", "in"));
+        let output = read(&pair_path("replication", "aws", "out"));
+        assert!(
+            contains(&input, b"<FutureTopLevel>future</FutureTopLevel>"),
+            "the sample must still carry the unknown element"
+        );
+        assert!(
+            !contains(&output, b"FutureTopLevel"),
+            "an unknown element must not be re-emitted: {}",
+            String::from_utf8_lossy(&output)
+        );
+    }
+    golden!(replication_minio, "replication", "minio", ReplicationConfiguration);
+    golden!(replication_rustfs, "replication", rustfs = rustfs_values::replication());
+
+    golden!(cors_aws, "cors", "aws", CORSConfiguration);
+    golden!(cors_minio, "cors", "minio", CORSConfiguration);
+    golden!(cors_rustfs, "cors", rustfs = rustfs_values::cors());
+
+    golden!(logging_aws, "logging", "aws", BucketLoggingStatus);
+    golden!(logging_minio, "logging", "minio", BucketLoggingStatus);
+    golden!(logging_rustfs, "logging", rustfs = rustfs_values::logging());
+
+    golden!(website_aws, "website", "aws", WebsiteConfiguration);
+    golden!(website_minio, "website", "minio", WebsiteConfiguration);
+    golden!(website_rustfs, "website", rustfs = rustfs_values::website());
+
+    golden!(accelerate_aws, "accelerate", "aws", AccelerateConfiguration);
+    golden!(accelerate_minio, "accelerate", "minio", AccelerateConfiguration);
+    golden!(accelerate_rustfs, "accelerate", rustfs = rustfs_values::accelerate());
+
+    golden!(request_payment_aws, "request_payment", "aws", RequestPaymentConfiguration);
+    golden!(request_payment_minio, "request_payment", "minio", RequestPaymentConfiguration);
+    golden!(request_payment_rustfs, "request_payment", rustfs = rustfs_values::request_payment());
+
+    golden!(public_access_block_aws, "public_access_block", "aws", PublicAccessBlockConfiguration);
+    golden!(public_access_block_minio, "public_access_block", "minio", PublicAccessBlockConfiguration);
+    golden!(
+        public_access_block_rustfs,
+        "public_access_block",
+        rustfs = rustfs_values::public_access_block()
+    );
+}

crates/ecstore/tests/cleanup_isolation_test.rs

@@ -1,174 +0,0 @@
-// Copyright 2026 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#![cfg(target_os = "linux")]
-
-use bytes::Bytes;
-mod storage_api;
-
-use std::time::Duration;
-use storage_api::cleanup_isolation::{DeleteOptions, DiskAPI, DiskError, DiskOption, Endpoint, new_disk};
-
-#[tokio::test]
-async fn cleanup_disk_and_periodic_gc_use_affinity_without_changing_foreground() {
-    const CHILD: &str = "RUSTFS_CLEANUP_LINUX_TEST_CHILD";
-    let foreground = rustix::thread::sched_getaffinity(None).unwrap();
-    let cpu = (0..rustix::thread::CpuSet::MAX_CPU)
-        .find(|cpu| foreground.is_set(*cpu))
-        .expect("the test process must have an allowed CPU");
-    if std::env::var_os(CHILD).is_none() {
-        for mode in ["custom", "default", "none", "restricted"] {
-            if mode == "default" && (!foreground.is_set(1) || !foreground.is_set(2)) {
-                continue;
-            }
-            let output = tokio::task::spawn_blocking(move || {
-                let mut command = std::process::Command::new(std::env::current_exe().unwrap());
-                command
-                    .args([
-                        "--exact",
-                        "cleanup_disk_and_periodic_gc_use_affinity_without_changing_foreground",
-                        "--nocapture",
-                    ])
-                    .env(CHILD, mode)
-                    .env("RUSTFS_CLEANUP_ISOLATE_ENABLE", "true")
-                    .env("RUSTFS_PUT_RENAME_TAIL_CLEANUP_MAX_PENDING", "4")
-                    .env("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKERS", "1")
-                    .env("RUSTFS_CLEANUP_DISK_MAX_PENDING", "4")
-                    .env("RUSTFS_CLEANUP_DISK_WORKERS", "1")
-                    .env("RUSTFS_CLEANUP_GC_WORKERS", "1")
-                    .env("RUSTFS_CLEANUP_BLOCKING_THREADS", "1")
-                    .env("RUSTFS_CLEANUP_ASYNC_THREADS", "1")
-                    .env_remove("RUSTFS_CLEANUP_CPUS")
-                    .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP")
-                    .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER")
-                    .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP");
-                match mode {
-                    "custom" => {
-                        command.env("RUSTFS_CLEANUP_CPUS", cpu.to_string());
-                    }
-                    "none" => {
-                        command.env("RUSTFS_CLEANUP_CPUS", "none");
-                    }
-                    _ => {}
-                }
-                command.output().unwrap()
-            })
-            .await
-            .unwrap();
-            assert!(
-                output.status.success(),
-                "{}\n{}",
-                String::from_utf8_lossy(&output.stdout),
-                String::from_utf8_lossy(&output.stderr)
-            );
-            assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed"));
-        }
-        return;
-    }
-
-    let mode = std::env::var(CHILD).unwrap();
-    if mode == "restricted" {
-        let mut restricted = rustix::thread::CpuSet::new();
-        restricted.set(cpu);
-        rustix::thread::sched_setaffinity(None, &restricted).expect("restrict initialization to one allowed CPU");
-    }
-    let dir = tempfile::tempdir().unwrap();
-    let disk = new_disk(&Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(), &DiskOption::default())
-        .await
-        .unwrap();
-    let bucket = "cleanup-affinity";
-    disk.make_volume(bucket).await.unwrap();
-    disk.write_all(bucket, "old/part.1", Bytes::from_static(b"garbage"))
-        .await
-        .unwrap();
-    let receipt = disk
-        .write_all(bucket, "old/.rustfs-old-data-cleanup-receipt.json", Bytes::from_static(b"receipt"))
-        .await;
-    if mode == "restricted" {
-        assert!(
-            receipt
-                .expect_err("unavailable default CPUs must reject cleanup")
-                .to_string()
-                .contains("allowed CPU set")
-        );
-        assert_eq!(disk.read_all(bucket, "old/part.1").await.unwrap(), Bytes::from_static(b"garbage"));
-        return;
-    }
-    receipt.unwrap();
-    let expected_mask = match mode.as_str() {
-        "default" => "1-2".to_owned(),
-        "none" => std::fs::read_to_string("/proc/thread-self/status")
-            .unwrap()
-            .lines()
-            .find_map(|line| line.strip_prefix("Cpus_allowed_list:"))
-            .unwrap()
-            .trim()
-            .to_owned(),
-        _ => cpu.to_string(),
-    };
-    disk.write_all(bucket, "live/part.1", Bytes::from_static(b"keep"))
-        .await
-        .unwrap();
-    disk.delete(
-        bucket,
-        "old",
-        DeleteOptions {
-            recursive: true,
-            ..Default::default()
-        },
-    )
-    .await
-    .unwrap();
-    assert!(matches!(disk.read_all(bucket, "old/part.1").await, Err(DiskError::FileNotFound)));
-
-    // One async worker and one lazily started blocking worker. Inspect the
-    // real process threads, including the threads executing filesystem work.
-    let mut cleanup_threads = 0;
-    for entry in std::fs::read_dir("/proc/self/task").unwrap() {
-        let task = entry.unwrap().path();
-        let Ok(name) = std::fs::read_to_string(task.join("comm")) else {
-            continue;
-        };
-        if name.trim() != "rustfs-cleanup" {
-            continue;
-        }
-        cleanup_threads += 1;
-        let status = std::fs::read_to_string(task.join("status")).unwrap();
-        let mask = status
-            .lines()
-            .find_map(|line| line.strip_prefix("Cpus_allowed_list:"))
-            .unwrap()
-            .trim();
-        assert_eq!(mask, expected_mask);
-    }
-    assert_eq!(cleanup_threads, 2, "async and blocking workers must both use the configured runtime");
-    assert_eq!(rustix::thread::sched_getaffinity(None).unwrap(), foreground);
-
-    let trash = dir.path().join(".rustfs.sys/tmp/.trash");
-    assert!(std::fs::read_dir(&trash).unwrap().next().is_some());
-    // Advance the foreground timer that dispatches the five-minute periodic
-    // scan; the dedicated runtime then performs the actual physical removal.
-    tokio::time::pause();
-    tokio::time::advance(Duration::from_secs(301)).await;
-    tokio::time::resume();
-    tokio::time::timeout(Duration::from_secs(20), async {
-        while std::fs::read_dir(&trash).unwrap().next().is_some() {
-            tokio::time::sleep(Duration::from_millis(5)).await;
-        }
-    })
-    .await
-    .expect("periodic GC must physically remove trash");
-    assert_eq!(disk.read_all(bucket, "live/part.1").await.unwrap(), Bytes::from_static(b"keep"));
-    assert_eq!(rustix::thread::sched_getaffinity(None).unwrap(), foreground);
-}

crates/ecstore/tests/fd_relative_io_test.rs

@@ -1,283 +0,0 @@
-// Copyright 2026 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#![cfg(all(target_os = "linux", feature = "test-util"))]
-
-mod storage_api;
-
-use bytes::Bytes;
-use rustfs_filemeta::{FileInfo, ObjectPartInfo};
-use std::{path::Path, time::Duration};
-use storage_api::fd_relative::{
-    DiskAPI, DiskError, DiskOption, DiskStore, Endpoint, LocalPublicationPause, LocalPublicationStage, ReadOptions, new_disk,
-};
-use time::OffsetDateTime;
-use uuid::Uuid;
-
-const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp";
-
-async fn disk_at(path: &Path) -> DiskStore {
-    new_disk(
-        &Endpoint::try_from(path.to_str().expect("UTF-8 test path")).expect("endpoint"),
-        &DiskOption::default(),
-    )
-    .await
-    .expect("local disk")
-}
-
-fn metadata(data: Bytes, inline: bool, directory: Uuid) -> FileInfo {
-    let mut fi = FileInfo::new("object", 1, 0);
-    fi.erasure.index = 1;
-    fi.version_id = Some(Uuid::nil());
-    fi.data_dir = Some(directory);
-    fi.size = i64::try_from(data.len()).expect("test data size");
-    fi.parts = vec![ObjectPartInfo {
-        number: 1,
-        size: data.len(),
-        actual_size: fi.size,
-        ..Default::default()
-    }];
-    fi.mod_time = Some(OffsetDateTime::now_utc());
-    fi.data = inline.then_some(data);
-    if inline {
-        fi.set_inline_data();
-    }
-    fi
-}
-
-async fn put(disk: &DiskStore, staging: &str, data: Bytes, inline: bool) -> Uuid {
-    let directory = Uuid::new_v4();
-    if !inline {
-        disk.write_all(RUSTFS_META_TMP_BUCKET, &format!("{staging}/{directory}/part.1"), data.clone())
-            .await
-            .expect("stage shard");
-    }
-    disk.rename_data(RUSTFS_META_TMP_BUCKET, staging, metadata(data, inline, directory), "bucket", "object")
-        .await
-        .expect("commit object");
-    directory
-}
-
-#[tokio::test]
-async fn pinned_root_reads_and_writes_original_disk_after_path_replacement() {
-    let temp = tempfile::tempdir().expect("test directory");
-    let root = temp.path().join("disk");
-    std::fs::create_dir(&root).expect("disk root");
-    let disk = disk_at(&root).await;
-    disk.make_volume("bucket").await.expect("bucket");
-    disk.write_all("bucket", "metadata", Bytes::from_static(b"original"))
-        .await
-        .expect("initial write");
-    let moved = temp.path().join("original");
-    std::fs::rename(&root, &moved).expect("move mount path");
-    std::fs::create_dir_all(root.join("bucket")).expect("replacement disk");
-    std::fs::write(root.join("bucket/metadata"), b"replacement").expect("replacement sentinel");
-    assert_eq!(
-        disk.read_all("bucket", "metadata").await.expect("pinned read"),
-        Bytes::from_static(b"original")
-    );
-    disk.write_all("bucket", "metadata", Bytes::from_static(b"updated"))
-        .await
-        .expect("pinned write");
-    assert_eq!(std::fs::read(moved.join("bucket/metadata")).expect("original disk"), b"updated");
-    assert_eq!(std::fs::read(root.join("bucket/metadata")).expect("replacement disk"), b"replacement");
-}
-
-#[tokio::test]
-async fn relative_reads_and_writes_reject_symlink_escape() {
-    let temp = tempfile::tempdir().expect("test directory");
-    let disk = disk_at(temp.path()).await;
-    disk.make_volume("bucket").await.expect("bucket");
-    let outside = tempfile::tempdir().expect("outside directory");
-    std::fs::write(outside.path().join("sentinel"), b"preserved").expect("sentinel");
-    std::os::unix::fs::symlink(outside.path(), temp.path().join("bucket/link")).expect("directory link");
-    std::os::unix::fs::symlink(outside.path().join("sentinel"), temp.path().join("bucket/file-link")).expect("file link");
-    for path in ["link/sentinel", "file-link"] {
-        assert!(matches!(disk.read_all("bucket", path).await, Err(DiskError::InvalidPath)));
-        assert!(matches!(
-            disk.write_all("bucket", path, Bytes::from_static(b"unsafe")).await,
-            Err(DiskError::InvalidPath)
-        ));
-    }
-    assert_eq!(std::fs::read(outside.path().join("sentinel")).expect("outside bytes"), b"preserved");
-}
-
-#[tokio::test]
-async fn inline_and_non_inline_overwrites_remain_readable_after_reopen() {
-    for inline in [true, false] {
-        let temp = tempfile::tempdir().expect("test directory");
-        let disk = disk_at(temp.path()).await;
-        disk.make_volume("bucket").await.expect("bucket");
-        put(&disk, "first", Bytes::from_static(b"old bytes"), inline).await;
-        let directory = put(&disk, "second", Bytes::from_static(b"new bytes"), inline).await;
-        drop(disk);
-        let disk = disk_at(temp.path()).await;
-        let fi = disk
-            .read_version(
-                "bucket",
-                "bucket",
-                "object",
-                "",
-                &ReadOptions {
-                    read_data: true,
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("committed metadata after reopen");
-        assert_eq!(fi.data_dir, Some(directory));
-        if inline {
-            assert_eq!(fi.data, Some(Bytes::from_static(b"new bytes")));
-        } else {
-            assert_eq!(
-                disk.read_all("bucket", &format!("object/{directory}/part.1"))
-                    .await
-                    .expect("shard bytes"),
-                Bytes::from_static(b"new bytes")
-            );
-        }
-    }
-}
-
-#[tokio::test]
-async fn rename_rejects_symlinked_destination_metadata() {
-    for inline in [true, false] {
-        let temp = tempfile::tempdir().expect("disk directory");
-        let outside = tempfile::tempdir().expect("outside directory");
-        let disk = disk_at(temp.path()).await;
-        disk.make_volume("bucket").await.expect("bucket");
-        put(&disk, "seed", Bytes::from_static(b"old object"), inline).await;
-        let destination = temp.path().join("bucket/object/xl.meta");
-        let original = std::fs::read(&destination).expect("valid existing metadata");
-        let external_metadata = outside.path().join("xl.meta");
-        std::fs::write(&external_metadata, &original).expect("external valid metadata");
-        std::fs::remove_file(&destination).expect("replace destination metadata");
-        std::os::unix::fs::symlink(&external_metadata, &destination).expect("metadata escape link");
-
-        let directory = Uuid::new_v4();
-        let data = Bytes::from_static(b"replacement");
-        if !inline {
-            disk.write_all(RUSTFS_META_TMP_BUCKET, &format!("replacement/{directory}/part.1"), data.clone())
-                .await
-                .expect("stage non-inline data");
-        }
-        let result = disk
-            .rename_data(
-                RUSTFS_META_TMP_BUCKET,
-                "replacement",
-                metadata(data, inline, directory),
-                "bucket",
-                "object",
-            )
-            .await;
-        assert!(
-            matches!(result, Err(DiskError::InvalidPath)),
-            "symlinked metadata must be rejected: {result:?}"
-        );
-        assert_eq!(std::fs::read_link(&destination).expect("link must remain"), external_metadata);
-        assert_eq!(std::fs::read(&external_metadata).expect("external metadata must survive"), original);
-    }
-}
-
-#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn prepared_publication_rejects_source_or_directory_replacement() {
-    for replace_directory in [false, true] {
-        let temp = tempfile::tempdir().expect("test directory");
-        let disk = disk_at(temp.path()).await;
-        disk.make_volume("bucket").await.expect("bucket");
-        put(&disk, "first", Bytes::from_static(b"old bytes"), true).await;
-        let old = std::fs::read(temp.path().join("bucket/object/xl.meta")).expect("old metadata");
-        let mut pause = LocalPublicationPause::install(&disk, "bucket", "object/xl.meta", LocalPublicationStage::PreparedRename)
-            .expect("publication pause");
-        let writer = disk.clone();
-        let task = tokio::spawn(async move {
-            writer
-                .rename_data(
-                    RUSTFS_META_TMP_BUCKET,
-                    "second",
-                    metadata(Bytes::from_static(b"new bytes"), true, Uuid::new_v4()),
-                    "bucket",
-                    "object",
-                )
-                .await
-        });
-        tokio::time::timeout(Duration::from_secs(30), pause.entered())
-            .await
-            .expect("publication reached pause")
-            .expect("pause entered");
-        if replace_directory {
-            std::fs::rename(temp.path().join("bucket/object"), temp.path().join("detached")).expect("detach object");
-            std::fs::create_dir(temp.path().join("bucket/object")).expect("replacement directory");
-            std::fs::write(temp.path().join("bucket/object/xl.meta"), &old).expect("replacement metadata");
-        } else {
-            let src = temp.path().join(RUSTFS_META_TMP_BUCKET).join("second/xl.meta");
-            std::fs::rename(&src, src.with_extension("detached")).expect("detach staged metadata");
-            std::fs::write(src, b"unrelated").expect("replacement staged file");
-        }
-        drop(pause);
-        assert!(task.await.expect("writer task").is_err());
-        assert_eq!(
-            std::fs::read(temp.path().join("bucket/object/xl.meta")).expect("old metadata survives"),
-            old
-        );
-    }
-}
-
-#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn cancelled_waiter_keeps_publication_serialized_until_physical_completion() {
-    let temp = tempfile::tempdir().expect("test directory");
-    let disk = disk_at(temp.path()).await;
-    disk.make_volume("bucket").await.expect("bucket");
-    put(&disk, "first", Bytes::from_static(b"old bytes"), true).await;
-    let mut pause = LocalPublicationPause::install(&disk, "bucket", "object/xl.meta", LocalPublicationStage::PreparedRename)
-        .expect("publication pause");
-    let writer = disk.clone();
-    let pending = tokio::spawn(async move { put(&writer, "shared-stage", Bytes::from_static(b"cancelled waiter"), true).await });
-    tokio::time::timeout(Duration::from_secs(30), pause.entered())
-        .await
-        .expect("publication reached pause")
-        .expect("pause entered");
-    pending.abort();
-    assert!(pending.await.expect_err("waiter aborted").is_cancelled());
-    let writer = disk.clone();
-    let (started_tx, started_rx) = tokio::sync::oneshot::channel();
-    let mut retry = tokio::spawn(async move {
-        let _ = started_tx.send(());
-        put(&writer, "shared-stage", Bytes::from_static(b"latest bytes"), true).await
-    });
-    started_rx.await.expect("retry started");
-    assert!(
-        tokio::time::timeout(Duration::from_millis(200), &mut retry).await.is_err(),
-        "a retry must not publish while the cancelled writer still owns physical IO"
-    );
-    drop(pause);
-    tokio::time::timeout(Duration::from_secs(30), retry)
-        .await
-        .expect("retry completed")
-        .expect("retry task");
-    let fi = disk
-        .read_version(
-            "bucket",
-            "bucket",
-            "object",
-            "",
-            &ReadOptions {
-                read_data: true,
-                ..Default::default()
-            },
-        )
-        .await
-        .expect("latest metadata");
-    assert_eq!(fi.data, Some(Bytes::from_static(b"latest bytes")));
-}

crates/ecstore/tests/fixtures/bucket-config-goldens/README.md

@@ -0,0 +1,58 @@
+# Persisted bucket-configuration XML goldens (rustfs/backlog#2744)
+
+Byte-level goldens for the 13 XML families that `BucketMetadata` persists as
+`*_config_xml`. They pin what `rustfs_ecstore::api::bucket::utils::deserialize`
+followed by `serialize` emits today, so that the DTO/codec replacement (T1.4)
+and the persisted-bytes comparison (T3.7) can be proven byte-identical: an
+older binary must keep reading what a newer one writes, and the reverse.
+
+Generated at commit: `6b1554003ebf8f2037ffb7da9c9b906527e758da` (2026-10-07T17:40:56Z)
+
+Checked by `crates/ecstore/tests/bucket_config_goldens.rs`
+(`cargo nextest run -p rustfs-ecstore bucket_config_goldens`, 39 tests).
+Generated once by `scripts/gen_bucket_config_goldens.sh`, which records the
+commit above and refuses to run again unless `--force` is passed. Do not
+regenerate after a DTO or codec replacement: at that point a diff here is the
+finding, not noise.
+
+## Layout
+
+`<family>/<shape>.in.xml` is an input; `<family>/<shape>.out.xml` is the exact
+bytes of `serialize(deserialize(in))` on the generating commit. The test
+asserts `bytes == out` (never structural equality), that `out` is a fixed point
+of the same round trip, and for the `rustfs` shape that `serialize(typed
+value)` still equals `rustfs.in.xml`.
+
+| Shape | Source |
+| --- | --- |
+| `aws` | Written for this corpus from the public S3 XML schema in the shape of the AWS documentation examples: XML declaration, `xmlns` on the root, indented. No AWS prose is copied. |
+| `minio` | The bytes a MinIO server persists. Seven families are carved verbatim out of `../minio/bucket_metadata.blob.hex` (MinIO `RELEASE.2025-07-23T15-54-02Z`); the rest are hand-shaped as compact Go `encoding/xml` output with the S3 namespace on the root, because MinIO does not persist those families. |
+| `rustfs` | The bytes RustFS writes today through `serialize` for a typed value (`rustfs_values` in the test). This shape carries the MinIO-only members RustFS accepts (`DelMarkerExpiration`, `ExcludeFolders`, `ExcludedPrefixes`, `DeleteReplication`). |
+
+| Family | Root element | `minio.in.xml` provenance |
+| --- | --- | --- |
+| `notification` | `NotificationConfiguration` | MinIO fixture |
+| `lifecycle` | `LifecycleConfiguration` | MinIO fixture (carries the MinIO `ExpiryUpdatedAt` extension) |
+| `object_lock` | `ObjectLockConfiguration` | MinIO fixture |
+| `versioning` | `VersioningConfiguration` | MinIO fixture |
+| `encryption` | `ServerSideEncryptionConfiguration` | MinIO fixture |
+| `tagging` | `Tagging` | MinIO fixture |
+| `replication` | `ReplicationConfiguration` | MinIO fixture |
+| `cors` | `CORSConfiguration` | hand-shaped |
+| `logging` | `BucketLoggingStatus` | hand-shaped |
+| `website` | `WebsiteConfiguration` | hand-shaped |
+| `accelerate` | `AccelerateConfiguration` | hand-shaped |
+| `request_payment` | `RequestPaymentConfiguration` | hand-shaped |
+| `public_access_block` | `PublicAccessBlockConfiguration` | hand-shaped |
+
+## Unknown elements
+
+`replication/aws.in.xml` is the unknown-element sample from
+`crates/replication/src/config.rs`
+(`s3_xml_parser_discards_unknown_replication_elements_before_validation`),
+re-indented to two spaces. Its golden proves the root-level `FutureTopLevel`
+element is skipped on read and never re-emitted on write, which is the
+persisted-configuration exception recorded in the root `AGENTS.md`. On the
+generating commit every one of the 13 root elements skips unknown children;
+nested members (`Rule`, `Filter`, `Destination`, ...) reject them. A
+replacement codec must keep that boundary where it is.

crates/ecstore/tests/fixtures/bucket-config-goldens/accelerate/aws.in.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<AccelerateConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Status>Enabled</Status>
+</AccelerateConfiguration>

crates/ecstore/tests/fixtures/bucket-config-goldens/accelerate/aws.out.xml

@@ -0,0 +1 @@
+<AccelerateConfiguration><Status>Enabled</Status></AccelerateConfiguration>

crates/ecstore/tests/fixtures/bucket-config-goldens/accelerate/minio.in.xml

@@ -0,0 +1 @@
+<AccelerateConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></AccelerateConfiguration>

crates/ecstore/tests/fixtures/bucket-config-goldens/accelerate/minio.out.xml

@@ -0,0 +1 @@
+<AccelerateConfiguration><Status>Suspended</Status></AccelerateConfiguration>

crates/ecstore/tests/fixtures/bucket-config-goldens/accelerate/rustfs.in.xml

@@ -0,0 +1 @@
+<AccelerateConfiguration><Status>Suspended</Status></AccelerateConfiguration>

crates/ecstore/tests/fixtures/bucket-config-goldens/accelerate/rustfs.out.xml

@@ -0,0 +1 @@
+<AccelerateConfiguration><Status>Suspended</Status></AccelerateConfiguration>

crates/ecstore/tests/fixtures/bucket-config-goldens/cors/aws.in.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <CORSRule>
+    <ID>AllowWebApp</ID>
+    <AllowedOrigin>https://www.example.com</AllowedOrigin>
+    <AllowedMethod>PUT</AllowedMethod>
+    <AllowedMethod>POST</AllowedMethod>
+    <AllowedMethod>DELETE</AllowedMethod>
+    <AllowedHeader>*</AllowedHeader>
+    <ExposeHeader>x-amz-server-side-encryption</ExposeHeader>
+    <MaxAgeSeconds>3000</MaxAgeSeconds>
+  </CORSRule>
+  <CORSRule>
+    <AllowedOrigin>*</AllowedOrigin>
+    <AllowedMethod>GET</AllowedMethod>
+  </CORSRule>
+</CORSConfiguration>

crates/ecstore/tests/fixtures/bucket-config-goldens/cors/aws.out.xml

@@ -0,0 +1 @@
+<CORSConfiguration><CORSRule><AllowedHeader>*</AllowedHeader><AllowedMethod>PUT</AllowedMethod><AllowedMethod>POST</AllowedMethod><AllowedMethod>DELETE</AllowedMethod><AllowedOrigin>https://www.example.com</AllowedOrigin><ExposeHeader>x-amz-server-side-encryption</ExposeHeader><ID>AllowWebApp</ID><MaxAgeSeconds>3000</MaxAgeSeconds></CORSRule><CORSRule><AllowedMethod>GET</AllowedMethod><AllowedOrigin>*</AllowedOrigin></CORSRule></CORSConfiguration>

crates/ecstore/tests/fixtures/bucket-config-goldens/cors/minio.in.xml

@@ -0,0 +1 @@
+<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><CORSRule><ID>console</ID><AllowedOrigin>https://console.example.internal</AllowedOrigin><AllowedMethod>GET</AllowedMethod><AllowedMethod>HEAD</AllowedMethod><AllowedHeader>Authorization</AllowedHeader><AllowedHeader>Content-Type</AllowedHeader><ExposeHeader>ETag</ExposeHeader><MaxAgeSeconds>600</MaxAgeSeconds></CORSRule></CORSConfiguration>

crates/ecstore/tests/fixtures/bucket-config-goldens/cors/minio.out.xml

@@ -0,0 +1 @@
+<CORSConfiguration><CORSRule><AllowedHeader>Authorization</AllowedHeader><AllowedHeader>Content-Type</AllowedHeader><AllowedMethod>GET</AllowedMethod><AllowedMethod>HEAD</AllowedMethod><AllowedOrigin>https://console.example.internal</AllowedOrigin><ExposeHeader>ETag</ExposeHeader><ID>console</ID><MaxAgeSeconds>600</MaxAgeSeconds></CORSRule></CORSConfiguration>