chore(gateway): merge main into integration

Sync main 84e24e767 into gateway/integration. Ports main-side additions to
the integration boundaries: the tier deletion diagnostics admin handlers move
to the owned admin request/response types, and the new lifecycle rule checks
in ecstore use the persisted lifecycle types instead of s3s DTOs. Keeps the
gateway git source in deny.toml while dropping the smithy-rs pin main retired,
and refreshes the test-count baseline for the merged tree.

.agents/references/implementation.md

@@ -18,7 +18,6 @@ These implementation, style, and quality rules apply to all contributions, regar
 - Do not include agent, tool, contributor, account, or organization names in branch names.
 - Push to the user-requested remote or the repository's configured push remote. Do not hard-code or infer a remote from an account name.
 - Check free space before artifact-heavy builds, tests, coverage, or downloads. Re-check before a broad gate when space is tight.
-- Before handoff, remove the Cargo target directories and temporary worktrees the task created unless they stay intentionally or hold unmerged work. Identify build output by Cargo's markers (`.rustc_info.json`, `CACHEDIR.TAG`), not by directory name: `target` is also a source directory here.
 - Remove only task-owned temporary/build artifacts. Never delete another task's worktree or uncommitted data.
 - At handoff, mention disk or cleanup details only when they affected execution or artifacts/worktrees remain intentionally.
 - For Docker verification, ensure the tested image contains the intended binary. `buildx build` without `--load` leaves the image in the build cache; use the Makefile's loaded image and rebuild without cache only when stale artifacts are evidenced.

.config/ci-plan.json

@@ -1,6 +1,6 @@
 {
   "schema_version": 1,
-  "rollout": "enforced",
+  "rollout": "shadow",
   "core_packages": ["rustfs", "rustfs-ecstore", "rustfs-filemeta", "rustfs-iam", "rustfs-policy"],
   "connect_packages": [
     "rustfs-credentials", "rustfs-config", "rustfs-common", "rustfs-crypto",

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
-sha256-darwin=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
+sha256-linux=eb33067d90ee66c0a97e8f3658e46c152e47f897d2fbfdc6cb5ad93cfa55414b
+sha256-darwin=eb33067d90ee66c0a97e8f3658e46c152e47f897d2fbfdc6cb5ad93cfa55414b

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=7c705942d3a9e583ebaa85845daa21c4f81fdbf36def24db931aaef38636c53e
-sha256-linux=5c33dc395233671cb77452f8fb12900ecdbeb368632222b96835d310bce1aab8
+sha256-darwin=4fb92e88763982eabafd74d05eb3d269387624a447e427a8c6f84914e5b03bfe
+sha256-linux=bd0db7e846ce24f056a097655db8fe2265201913260d0e16feb38d99dcb954a7

.config/e2e-repl-nightly-selection.txt

@@ -1 +1 @@
-sha256=e4ee3898b840a8374cb02c64342cc15f8d7ecd1becaf4b4a2a4657290f9e7ff1
+sha256=4eac269f9a59b2f9be4e7a87ea6aa38454f5ebe10ede35c9b8d4218137d2cd35

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=00f8a56b75990b487a594b97c36610b94476a6944ba00473e37187ea654d42bf
+sha256=879f421f1115d44f930e229510a2e533cfd5ebd4effe38730b82ef61f4fd7a86

.config/make/lint-fmt.mak

@@ -70,6 +70,11 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
 	@echo "📦 Checking s3s footprint ratchet..."
 	./scripts/check_s3s_footprint.sh
 
+.PHONY: test-count-ratchet-check
+test-count-ratchet-check: ## Check no workspace crate lost tests or gained #[ignore] against the committed baseline
+	@echo "🧪 Checking per-crate test-count ratchet..."
+	./scripts/check_test_count_ratchet.sh
+
 .PHONY: fips-wording-check
 fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities
 	@echo "📣 Checking cryptographic capability wording guard..."

.config/make/pre-commit.mak

@@ -20,11 +20,11 @@ planning-docs-check: ## Check that no planning-type documents are committed
 	./scripts/check_no_planning_docs.sh
 
 .PHONY: pre-commit
-pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
+pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
 	@echo "✅ All pre-commit checks passed!"
 
 .PHONY: pre-pr
-pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
+pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
 	@echo "✅ All pre-PR checks passed!"
 
 .PHONY: dev-check

.config/make/tests.mak

@@ -51,7 +51,9 @@ script-tests: ## Run shell script tests
 	./scripts/test_python_bin.sh
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_check_doc_paths.py
 	./scripts/check_embedded_secrets.sh --self-test
+	./scripts/check_s3s_footprint.sh --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
@@ -61,7 +63,6 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_connect_release_boundary.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_test_artifact.py
-	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_test_shards.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_merge_queue.py
 	$(RUSTFS_PYTHON_BIN) -m unittest discover -s scripts/ci -p 'test_*.py'
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test

.config/nextest.toml

@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
 # processes cannot exhaust those budgets. Keep deadlines and assertions intact.
 # The short HTTP fixtures are tracked in rustfs/backlog#2714.
 [[profile.default.overrides]]
-filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
+filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
 threads-required = "num-test-threads"
 
 # This cancellation probe hashes the serving executable before acquiring its
@@ -356,6 +356,12 @@ threads-required = "num-test-threads"
 filter = 'package(rustfs-ecstore) & test(=store::init::tests::dispatch_manifest_rollback_bounded_concurrency_reaches_tail_behind_slow_member)'
 threads-required = "num-test-threads"
 
+# These short-deadline real-disk progress probes need the test run's capacity
+# for scanner and object-lock handoffs; keep their assertions and deadlines.
+[[profile.default.overrides]]
+filter = '(package(rustfs-scanner) & (test(=scanner::tests::cycle_persist_failure::later_cycle_post_commit_failure_reloads_durable_state) | test(=scanner::tests::running_main_loop_catches_up_pause_cleared_after_startup_observe))) | (package(rustfs-ecstore) & test(=store::init::tests::restored_transitioned_delete_causally_enqueues_free_version))'
+threads-required = "num-test-threads"
+
 # ---------------------------------------------------------------------------
 # ci profile — the strict CI gate (ci.yml `cargo nextest run --profile ci`)
 # ---------------------------------------------------------------------------
@@ -424,6 +430,12 @@ threads-required = "num-test-threads"
 filter = 'package(rustfs-scanner) & (test(/^scanner::backlog::tests::native_/) | test(=scanner_folder::tests::mrf_ownership::mrf_ownership_manager_completion_preserves_scanner_pending) | test(/^scanner_io::tests::service_cohort::/))'
 threads-required = "num-test-threads"
 
+# Keep short-deadline real-disk progress probes from competing with unrelated
+# tests for scanner and object-lock capacity. Assertions and deadlines stay on.
+[[profile.ci.overrides]]
+filter = '(package(rustfs-scanner) & (test(=scanner::tests::cycle_persist_failure::later_cycle_post_commit_failure_reloads_durable_state) | test(=scanner::tests::running_main_loop_catches_up_pause_cleared_after_startup_observe))) | (package(rustfs-ecstore) & test(=store::init::tests::restored_transitioned_delete_causally_enqueues_free_version))'
+threads-required = "num-test-threads"
+
 # Emit this expensive fixture's stage timings in CI and use tmpfs when safe.
 [[profile.ci.overrides]]
 filter = 'package(rustfs-ecstore) & test(=store::init::tests::decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page)'
@@ -548,7 +560,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
 test-group = 'embedded-test-ports'
 
 [[profile.ci.overrides]]
-filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
+filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
 threads-required = "num-test-threads"
 
 # Preserve the cancellation probe's startup budget and assertions in CI too.
@@ -698,6 +710,14 @@ test-group = 'ecstore-serial-flaky'
 # the PR-lane e2e coverage for scanner usage snapshots consumed by quota and
 # admin surfaces. It uses the same single-node, random-port, isolated-temp-dir
 # fixture as the existing smoke modules.
+#
+# Dual-stack differential (rustfs/backlog#2740): `dual_stack_differential::`
+# starts two single-node servers from the same binary, one per
+# RUSTFS_S3_STACK value, on random ports with isolated temp dirs, and runs one
+# SDK script covering every operation of `impl S3 for FS` against both. Its
+# comparator unit tests need no server. It meets the admission criteria
+# unchanged; the module clause admits a new differential test without another
+# filter edit.
 [profile.e2e-smoke]
 default-filter = """
     package(e2e_test) & (
@@ -706,6 +726,7 @@ default-filter = """
         | test(/^reliant::lifecycle::/)
         | test(/^conditional_delete_objects_test::/)
         | test(/^reliant::tiering::/)
+        | test(/^dual_stack_differential::/)
         | test(/^upgrade_compatibility_test::upgrade_write_readiness_tests::/)
         | test(/^on_demand_migration::(get_basic_test::(get_miss_pulls_inline_and_serves_locally_afterwards|head_miss_answers_from_the_source_without_persisting)|interaction_test::test_odm_admin_config_is_redacted_and_status_counts_match_the_source)$/)
     )
@@ -827,7 +848,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
 # ---------------------------------------------------------------------------
 # e2e-distributed profile — 4-node 4-disk Actions suite
 # ---------------------------------------------------------------------------
-# Nightly and manual-dispatch lane owned by
+# Storage-sensitive PR / nightly / dispatch lane owned by
 # .github/workflows/e2e-distributed.yml.
 # Each case starts four rustfs processes (and for site replication, two
 # clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
@@ -960,12 +981,6 @@ test-group = 'e2e-inline-boundaries'
 filter = 'package(e2e_test) & test(/^heal_erasure_disk_rebuild_test::/)'
 test-group = 'e2e-cluster-nightly'
 
-# The multi-pool crash fixture seeds three nodes and 36 drives. Reserve this
-# nextest run's workers so other multi-node tests do not starve its deadline.
-[[profile.e2e-full.overrides]]
-filter = 'package(e2e_test) & test(=heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_pool_after_background_target_crash)'
-threads-required = "num-test-threads"
-
 [[profile.e2e-full.overrides]]
 filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
 test-group = 'e2e-vault'
@@ -976,12 +991,6 @@ test-group = 'e2e-vault'
 filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
 threads-required = "num-test-threads"
 
-# The mc fixture uploads hundreds of objects concurrently before measuring listing.
-# Reserve the runner so other server fixtures do not starve its disk operations.
-[[profile.e2e-full.overrides]]
-filter = 'package(e2e_test) & test(=mc_mirror_small_bucket_test::test_mc_mirror_small_bucket_completes_without_list_timeout)'
-threads-required = "num-test-threads"
-
 # Fixed, small contract for every executable server change. These existing
 # scenarios exercise write/read, anonymous denial and retained bytes on restart.
 [profile.core-contract]

.config/release-validation-plan.json

@@ -1,77 +0,0 @@
-{
-  "schema_version": 1,
-  "plan_id": "rustfs-release-validation",
-  "description": "Versioned certification plan for every public RustFS release exit (rustfs/backlog#2662, R01). preview, prerelease and stable all require this explicit plan. Scripts/release/release_gate.py fail-closes publication on any missing, duplicate, stale, substituted or non-PASS required item.",
-  "release_types": ["preview", "prerelease", "stable"],
-  "targets": [
-    {"os": "linux", "arch": "x86_64", "abi": "musl"},
-    {"os": "linux", "arch": "aarch64", "abi": "musl"},
-    {"os": "linux", "arch": "x86_64", "abi": "gnu"},
-    {"os": "linux", "arch": "aarch64", "abi": "gnu"},
-    {"os": "macos", "arch": "aarch64", "abi": "none"},
-    {"os": "windows", "arch": "x86_64", "abi": "msvc"}
-  ],
-  "targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
-  "features": ["pyroscope"],
-  "trusted_workflows": [
-    {"repository": "rustfs/s3chaos", "workflow_id": "release-gate.yml"}
-  ],
-  "trusted_workflows_basis": "The shared evidence envelope producer lives in rustfs/s3chaos (G01, fault/release_gate.rs) and its plan/live split workflow is .github/workflows/release-gate.yml there. Only results from these identities may enter a release certification; PR-uploaded JSON never gains release authority. R02 must extend this list when additional verifier workflows land, never widen it to untrusted sources.",
-  "lanes": [
-    {
-      "lane": "identity",
-      "harness_repository": "rustfs/s3chaos",
-      "harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
-      "harness_basis": "Fixed G01 evidence-envelope producer revision (rustfs/s3chaos PR #102 head recorded in rustfs/backlog#2658). R02 re-pins this when the live verifier ships.",
-      "timeout_seconds": 1800,
-      "environment_capabilities": ["process-exec"],
-      "cases": ["artifact-identity", "startup"],
-      "applies_to": {
-        "release_types": ["preview", "prerelease", "stable"],
-        "platforms": "all"
-      },
-      "applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
-    },
-    {
-      "lane": "upgrade",
-      "harness_repository": "rustfs/s3chaos",
-      "harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
-      "harness_basis": "Same fixed envelope producer as the identity lane; the workflow SHA and the product build SHA are bound separately and need not be equal.",
-      "timeout_seconds": 5400,
-      "environment_capabilities": ["process-exec", "previous-release-artifact"],
-      "cases": [
-        "direct-upgrade",
-        "mixed-version-upgrade",
-        "bucket-config-upgrade",
-        "bucket-config-rollback",
-        "multipart-layout-upgrade",
-        "multipart-layout-baseline"
-      ],
-      "cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
-      "applies_to": {
-        "release_types": ["prerelease", "stable"],
-        "platforms": [{"os": "linux", "arch": "x86_64", "abi": "gnu"}]
-      },
-      "applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
-    }
-  ],
-  "upgrade_sources": {
-    "basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
-    "sources": [
-      {
-        "version": "1.0.1",
-        "role": "N-1",
-        "artifact_name": "rustfs-linux-x86_64-gnu-v1.0.1.zip",
-        "artifact_sha256": "9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5"
-      },
-      {
-        "version": "1.0.0-rc.5",
-        "role": "N-2",
-        "artifact_name": "rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip",
-        "artifact_sha256": "3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d"
-      }
-    ],
-    "rollback_supported": false,
-    "rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
-  }
-}

.config/s3s-edge-allowlist.txt

@@ -0,0 +1,37 @@
+# Files that may still reference s3s (`s3s::` or `s3s_sigv4`) when Phase 1 of
+# the gateway migration ends (rustfs/backlog#2734 task T0.6, rustfs/backlog#2741).
+# Consumed only by `scripts/check_s3s_footprint.sh --mode allowlist`.
+#
+# One entry per line: a repository-relative file path or a git `:(glob)`
+# pathspec. `**` matches across directories, `*` stays within one path
+# component, and a directory entry matches everything under it. Blank lines
+# and lines starting with `#` are ignored; an entry may not contain whitespace,
+# so a note belongs on its own comment line.
+#
+# Every Rust file listed by `git grep -l '\bs3s\(::\|_sigv4\)' -- '*.rs'` that
+# no entry matches fails the allowlist mode. Task T1.9 switches CI to that
+# mode; until then `--dry-run` prints the files still outside this list, which
+# is the Phase 1 tracking list.
+#
+# Never add crates/** other than crates/e2e_test: the engine crates leave s3s
+# entirely, and the script rejects such an entry. Only the s3s-facing edge of
+# the rustfs binary and the e2e oracle keep it.
+
+# rustfs binary: the s3s-facing edge that survives Phase 1
+rustfs/src/storage/ecfs.rs
+rustfs/src/storage/s3_compat/**
+rustfs/src/storage/access.rs
+rustfs/src/auth.rs
+rustfs/src/error.rs
+rustfs/src/storage_api.rs
+rustfs/src/app/storage_api.rs
+rustfs/src/admin/router.rs
+rustfs/src/app/metadata_route.rs
+rustfs/src/server/http.rs
+rustfs/src/server/layer.rs
+rustfs/src/server/hybrid.rs
+rustfs/src/protocols/client.rs
+
+# e2e oracle: the test target that verifies S3 behaviour against s3s
+crates/e2e_test/src/fake_s3_target/**
+crates/e2e_test/src/**

.config/s3s-footprint-baseline.txt

@@ -0,0 +1,17 @@
+# Exact baseline for the direct s3s footprint, consumed only by
+# scripts/check_s3s_footprint.sh (rustfs/backlog#1677; rustfs/backlog#2734 T0.4).
+#
+# Each value is the exact measurement of the tree, not a ceiling. CI fails when a
+# measured counter differs from its line here in either direction:
+#   - a larger measurement means new code widened the s3s surface: route it
+#     through the gateway abstractions instead; never raise a value here;
+#   - a smaller measurement means the PR shrank the surface: lower the matching
+#     value to the measured number in the same PR.
+#
+# files           Rust files referencing s3s paths, repo-wide minus crates/e2e_test
+# s3_error_lines  lines holding s3_error!, same scope, minus those the script
+#                 header's rule clears (rustfs_s3_types's macro); fails closed
+# ecstore_files   Rust files referencing s3s paths under crates/ecstore/src
+files=65
+s3_error_lines=289
+ecstore_files=0

.config/test-count-baseline.txt

@@ -0,0 +1,64 @@
+# Per-crate test-count baseline consumed by scripts/check_test_count_ratchet.sh
+# (rustfs/backlog#2734 section 1 rule 3; T0.3 in rustfs/backlog#2738).
+#
+# One row per workspace member: <package>, tests, ignored, tab-separated.
+# tests counts #[test], #[tokio::test], #[tokio::test(...)] and #[test_case(...)]
+# attribute lines under the package directory; ignored counts #[ignore] and
+# #[ignore = "..."]. Both columns are exact: CI fails when a crate's tests fall
+# below or rise above its row, or when its ignored count changes. Raise a row in
+# the PR that adds tests; lowering one is a reviewed statement that tests were
+# intentionally removed. Regenerate with:
+#   scripts/check_test_count_ratchet.sh --update
+e2e_test	993	20
+rustfs	6078	64
+rustfs-audit	70	0
+rustfs-checksums	26	0
+rustfs-common	36	0
+rustfs-concurrency	54	0
+rustfs-config	43	0
+rustfs-connect-symbol-catalog	6	0
+rustfs-credentials	24	0
+rustfs-crypto	67	0
+rustfs-data-usage	56	0
+rustfs-ecstore	7415	2
+rustfs-extension-schema	23	0
+rustfs-filemeta	353	0
+rustfs-heal	1033	3
+rustfs-heal-contracts	8	0
+rustfs-iam	401	0
+rustfs-io-core	49	0
+rustfs-io-metrics	154	0
+rustfs-keystone	29	0
+rustfs-kms	715	25
+rustfs-license	2	0
+rustfs-lifecycle	155	0
+rustfs-lock	143	0
+rustfs-log-analyzer	90	0
+rustfs-madmin	210	0
+rustfs-notify	139	0
+rustfs-object-capacity	92	0
+rustfs-object-data-cache	140	0
+rustfs-obs	385	0
+rustfs-policy	734	0
+rustfs-protocols	748	1
+rustfs-protos	73	0
+rustfs-replication	209	0
+rustfs-rio	206	0
+rustfs-rio-v2	34	6
+rustfs-s3-client	69	0
+rustfs-s3-ops	8	0
+rustfs-s3-types	103	0
+rustfs-s3select-api	207	0
+rustfs-s3select-query	189	2
+rustfs-scanner	1261	20
+rustfs-scanner-metrics	90	0
+rustfs-security-governance	20	0
+rustfs-signer	31	0
+rustfs-storage-api	34	0
+rustfs-targets	603	26
+rustfs-test-utils	1	0
+rustfs-tls-runtime	43	0
+rustfs-trusted-proxies	68	0
+rustfs-utils	247	0
+rustfs-zip	30	0
+zstd	11	0

.github/actions/quick-checks/action.yml

@@ -96,6 +96,10 @@ runs:
       shell: bash
       run: ./scripts/check_s3s_footprint.sh
 
+    - name: Check per-crate test-count ratchet
+      shell: bash
+      run: ./scripts/check_test_count_ratchet.sh
+
     - name: Check cryptographic capability wording
       shell: bash
       run: ./scripts/check_fips_wording.sh

.github/dependabot.yml

@@ -32,8 +32,6 @@ updates:
       - "overtrue"
       - "majinghe"
     ignore:
-      - dependency-name: "pulsar"
-        versions: [ "6.9.x" ]
       - dependency-name: "object_store"
         versions: [ "0.13.x" ]
       - dependency-name: "libunftp"

.github/scheduled-validations.json

@@ -296,32 +296,6 @@
       "origin": "baseline",
       "alert_contract": "scheduled-validation"
     },
-    {
-      "workflow": ".github/workflows/rustfs-validation-sweep.yml",
-      "name": "RustFS Validation Sweep",
-      "roles": [
-        "periodic",
-        "manual"
-      ],
-      "events": [
-        "schedule",
-        "workflow_dispatch"
-      ],
-      "cadence": "nightly",
-      "timezone": "UTC",
-      "schedule": [
-        "30 19 * * *"
-      ],
-      "owner": "@overtrue",
-      "policy": "active",
-      "monitor": "workflow",
-      "max_age_hours": 60,
-      "result_owner": "rustfs-validation-sweep.yml",
-      "disposition": "canonical validation cases executed in rotating batches",
-      "failure_owner": "workflow",
-      "origin": "baseline",
-      "alert_contract": "scheduled-validation"
-    },
     {
       "workflow": ".github/workflows/btf-prototype-reference.yml",
       "name": "BTF prototype reference",
@@ -769,10 +743,12 @@
       "workflow": ".github/workflows/e2e-distributed.yml",
       "name": "e2e-distributed",
       "roles": [
+        "pr",
         "periodic",
         "manual"
       ],
       "events": [
+        "pull_request",
         "workflow_dispatch",
         "schedule"
       ],
@@ -847,11 +823,12 @@
       "workflow": ".github/workflows/e2e-upgrade.yml",
       "name": "Upgrade Compatibility",
       "roles": [
+        "pr",
         "periodic",
-        "manual",
-        "release"
+        "manual"
       ],
       "events": [
+        "pull_request",
         "push",
         "schedule",
         "workflow_dispatch"

.github/workflows/build.yml

@@ -814,18 +814,13 @@ jobs:
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
-          # Determine upload path based on build type. Release and prerelease
-          # builds only stage their bytes here: the official
-          # artifacts/rustfs/release/ path is written solely by the
-          # certification-gated promote-release-artifacts job, from these same
-          # staged objects. Development builds keep the uncertified dev channel
-          # and never touch official or latest channels.
+          # Determine upload path based on build type
           if [[ "$BUILD_TYPE" == "development" ]]; then
             R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/dev/"
             echo "📤 Uploading development build to R2 dev directory"
           else
-            R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
-            echo "📤 Staging release build for certification-gated promotion"
+            R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
+            echo "📤 Uploading release build to R2 release directory"
           fi
 
           # Upload all rustfs zip files to R2 using S3-compatible API
@@ -874,221 +869,6 @@ jobs:
             echo "This run does not trigger Docker publishing."
           fi
 
-  # Aggregate the parent candidate batch. This job always collects results and
-  # retains its evidence, including on failure, and every publication job
-  # below requires its success. Missing, stale, substituted or non-PASS
-  # evidence fails closed here: only the trusted verification sources
-  # (rustfs/s3chaos release-gate envelopes, rustfs/backlog#2658) can satisfy
-  # it, so a PR-uploaded JSON set never gains release authority.
-  release-certification:
-    name: Release Certification
-    needs: [ build-check, build-rustfs ]
-    if: ${{ !cancelled() && needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') }}
-    runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
-    timeout-minutes: 30
-    permissions:
-      contents: read
-      actions: read
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-
-      - name: Download all build artifacts
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          path: ./artifacts
-          pattern: rustfs-*
-          merge-multiple: true
-
-      - name: Write candidate and collect trusted validation results
-        env:
-          # Cross-repo read of the trusted verification run. Falls back to the
-          # workflow token, which cannot read rustfs/s3chaos; a missing token
-          # therefore fail-closes instead of certifying.
-          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
-          PRODUCT_COMMIT: ${{ github.sha }}
-          VERSION: ${{ needs.build-check.outputs.version }}
-          BUILD_TYPE: ${{ needs.build-check.outputs.build_type }}
-          FEATURES: ${{ needs.build-check.outputs.build_type == 'development' && '' || 'pyroscope' }}
-        shell: bash
-        run: |
-          set -euo pipefail
-
-          mkdir -p results release-gate-output
-          PLAN_SHA256=$(python3 -c "import hashlib,json,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('.config/release-validation-plan.json'))))")
-
-          python3 - "$PLAN_SHA256" > candidate.json <<'PY'
-          import hashlib, json, os, sys, zipfile
-
-          plan_sha256 = sys.argv[1]
-          profile = {"release": "stable", "prerelease": "prerelease", "preview": "preview"}.get(os.environ["BUILD_TYPE"])
-          if profile is None:
-              raise SystemExit("release certification only covers preview, prerelease and stable builds")
-
-          def sha256(path):
-              digest = hashlib.sha256()
-              with open(path, "rb") as handle:
-                  for chunk in iter(lambda: handle.read(1024 * 1024), b""):
-                      digest.update(chunk)
-              return digest.hexdigest()
-
-          deliverables = []
-          for name in sorted(os.listdir("artifacts")):
-              if not name.startswith("rustfs-") or not name.endswith(".zip"):
-                  continue
-              path = os.path.join("artifacts", name)
-              members = zipfile.ZipFile(path).namelist()
-              binary_name = "rustfs.exe" if "rustfs.exe" in members else "rustfs"
-              with zipfile.ZipFile(path) as archive:
-                  binary_bytes = archive.read(binary_name)
-              import tempfile
-              with tempfile.NamedTemporaryFile() as handle:
-                  handle.write(binary_bytes)
-                  handle.flush()
-                  binary_sha = sha256(handle.name)
-              deliverables.append({
-                  "kind": "zip",
-                  "name": name,
-                  "location": "https://github.com/rustfs/rustfs/releases/download/{}/{}".format(os.environ["VERSION"], name),
-                  "sha256": sha256(path),
-                  "size": os.path.getsize(path),
-                  "binary_sha256": binary_sha,
-              })
-          if not deliverables:
-              raise SystemExit("no built deliverables found to certify")
-          candidate = {
-              "schema_version": 1,
-              "product_commit": os.environ["PRODUCT_COMMIT"],
-              "version": os.environ["VERSION"],
-              "release_profile": profile,
-              "features": [f for f in os.environ.get("FEATURES", "").split(",") if f],
-              "plan_sha256": plan_sha256,
-              "validation_batch": {"run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"])},
-              "deliverables": deliverables,
-          }
-          json.dump(candidate, sys.stdout, indent=2, sort_keys=True)
-          print()
-          PY
-
-          # Trusted verification evidence only: the release-gate envelopes of
-          # rustfs/s3chaos for this exact product commit. Absence blocks below.
-          TRUSTED_REPO="rustfs/s3chaos"
-          TRUSTED_WORKFLOW="release-gate.yml"
-          TRUSTED_RUN=$(gh api "repos/${TRUSTED_REPO}/actions/workflows/${TRUSTED_WORKFLOW}/runs?head_sha=${PRODUCT_COMMIT}&status=success&per_page=1" --jq '.workflow_runs[0].id // empty' || true)
-          if [[ -z "$TRUSTED_RUN" ]]; then
-            echo "❌ No trusted verification run of ${TRUSTED_REPO}/${TRUSTED_WORKFLOW} for ${PRODUCT_COMMIT}; refusing to certify" >&2
-          else
-            gh run download "$TRUSTED_RUN" -R "$TRUSTED_REPO" -n release-validation-evidence -D results || \
-              echo "❌ Trusted verification run ${TRUSTED_RUN} published no release-validation-evidence artifact" >&2
-          fi
-
-          python3 scripts/release/release_gate.py validate \
-            --plan .config/release-validation-plan.json \
-            --candidate candidate.json \
-            --results results \
-            --output release-gate-output
-
-      - name: Upload certification evidence
-        if: always()
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: release-certification
-          path: |
-            candidate.json
-            results/
-            release-gate-output/
-          if-no-files-found: warn
-          retention-days: 30
-
-  # The only writer of the official R2 release path. Promotion re-checks the
-  # exact bytes about to be published against the passed certification list
-  # and copies the staged objects byte-for-byte: no rebuild-after-verify.
-  promote-release-artifacts:
-    name: Promote Release Artifacts
-    needs: [ build-check, build-rustfs, release-certification ]
-    # Target-channel lock for the official R2 release path.
-    concurrency:
-      group: release-channel-r2-release
-      cancel-in-progress: false
-    if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    permissions:
-      contents: read
-      actions: read
-    env:
-      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
-      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
-      R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
-      R2_BUCKET: ${{ secrets.R2_BUCKET }}
-      AWS_EC2_METADATA_DISABLED: true
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-
-      - name: Download build and certification artifacts
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          path: ./artifacts
-          pattern: rustfs-*
-          merge-multiple: true
-
-      - name: Download certification evidence
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: release-certification
-          path: ./certification
-
-      - name: Verify and promote staged release artifacts
-        shell: bash
-        run: |
-          set -euo pipefail
-
-          if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
-            echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
-            exit 1
-          fi
-
-          for zip_file in artifacts/rustfs-*.zip; do
-            [[ -f "$zip_file" ]] || continue
-            name="$(basename "$zip_file")"
-            python3 scripts/release/release_gate.py verify-promotion \
-              --plan .config/release-validation-plan.json \
-              --candidate certification/candidate.json \
-              --summary certification/release-gate-output \
-              --artifact "$zip_file" \
-              --artifact-name "$name"
-          done
-
-          if ! command -v aws >/dev/null 2>&1; then
-            sudo apt-get update && sudo apt-get install -y awscli
-          fi
-          export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
-          export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
-          export AWS_DEFAULT_REGION="auto"
-
-          STAGING_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
-          RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
-          for zip_file in artifacts/rustfs-*.zip; do
-            [[ -f "$zip_file" ]] || continue
-            name="$(basename "$zip_file")"
-            staged="$(mktemp)"
-            aws s3 cp "${STAGING_PATH}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
-            staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
-            built_sha="$(sha256sum "$zip_file" | awk '{print $1}')"
-            if [[ "$staged_sha" != "$built_sha" ]]; then
-              echo "❌ Staged object ${name} does not match the verified bytes" >&2
-              exit 1
-            fi
-            aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
-            rm -f "$staged"
-            echo "✅ Promoted ${name} (${built_sha})"
-          done
-
   # Create GitHub Release for every valid release tag, including previews
   create-release:
     name: Create GitHub Release
@@ -1248,10 +1028,6 @@ jobs:
   update-latest-version:
     name: Update Latest Version
     needs: [ build-check, publish-release ]
-    # Target-channel lock for the latest.json version pointer.
-    concurrency:
-      group: release-channel-oss-latest
-      cancel-in-progress: false
     if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -1292,45 +1068,6 @@ jobs:
           rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
           chmod +x "$OSSUTIL_BIN"
 
-          # Candidate/pointer ordering: an older candidate must never overwrite
-          # a newer pointer, even if its promotion job finishes later.
-          CURRENT_LATEST="${RUNNER_TEMP}/current-latest.json"
-          if "$OSSUTIL_BIN" cp oss://rustfs-version/latest.json "$CURRENT_LATEST" 2>/dev/null; then
-            python3 - "$CURRENT_LATEST" "$VERSION" <<'PY'
-          import json
-          import re
-          import sys
-
-          def key(version):
-              match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$", version)
-              if not match:
-                  raise SystemExit(3)
-              core = tuple(int(part) for part in match.groups()[:3])
-              prerelease = match.group(4)
-              if prerelease is None:
-                  return core + (1,)
-              return core + (0, tuple(int(p) if p.isdigit() else p for p in re.split(r"[.-]", prerelease)))
-
-          try:
-              current = json.load(open(sys.argv[1], encoding="utf-8")).get("version", "")
-              proposed = sys.argv[2]
-              if current and key(current) > key(proposed):
-                  raise SystemExit(1)
-          except SystemExit:
-              raise
-          except (OSError, ValueError, TypeError):
-              raise SystemExit(3)
-          PY
-            guard_status=$?
-            if [[ "$guard_status" -eq 1 ]]; then
-              echo "Current latest.json points at a newer version; refusing to downgrade" >&2
-              exit 0
-            elif [[ "$guard_status" -ne 0 ]]; then
-              echo "Existing latest.json is unparsable; refusing to overwrite it blindly" >&2
-              exit 1
-            fi
-          fi
-
           # Create latest.json
           cat > latest.json << EOF
           {
@@ -1347,59 +1084,16 @@ jobs:
 
           echo "✅ Updated latest.json for ${RELEASE_TYPE} release $VERSION"
 
-  # Publish release (remove draft status). Requires the release-certification
-  # aggregate: without a complete legal success there is no public write.
+  # Publish release (remove draft status)
   publish-release:
     name: Publish Release
-    needs: [ build-check, create-release, upload-release-assets, release-certification ]
-    # Target-channel lock: concurrent releases serialize on this channel so an
-    # older candidate cannot interleave its publish with a newer one.
-    concurrency:
-      group: release-channel-github-release
-      cancel-in-progress: false
+    needs: [ build-check, create-release, upload-release-assets ]
     if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
     runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
     timeout-minutes: 30
     permissions:
       contents: write
-      actions: read
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-
-      - name: Download certification evidence
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: release-certification
-          path: ./certification
-
-      - name: Verify release assets against certification
-        env:
-          GH_TOKEN: ${{ github.token }}
-        shell: bash
-        run: |
-          set -euo pipefail
-          TAG="${{ needs.build-check.outputs.version }}"
-          mkdir -p "$RUNNER_TEMP/promotion-assets"
-          gh release download "$TAG" --dir "$RUNNER_TEMP/promotion-assets" --pattern '*.zip'
-          found=0
-          for zip_file in "$RUNNER_TEMP"/promotion-assets/*.zip; do
-            [[ -f "$zip_file" ]] || continue
-            found=1
-            python3 scripts/release/release_gate.py verify-promotion \
-              --plan .config/release-validation-plan.json \
-              --candidate certification/candidate.json \
-              --summary certification/release-gate-output \
-              --artifact "$zip_file" \
-              --artifact-name "$(basename "$zip_file")"
-          done
-          if [[ "$found" -ne 1 ]]; then
-            echo "No release zip assets to verify against the certification" >&2
-            exit 1
-          fi
-
       - name: Publish release
         env:
           GH_TOKEN: ${{ github.token }}

.github/workflows/cache-warm.yml

@@ -96,7 +96,7 @@ env:
   RUST_BACKTRACE: 1
 
 jobs:
-  # Readers: workspace-lint-and-build, workspace-tests (workspace and ILM), build-rustfs-debug-binary,
+  # Readers: test-and-lint (workspace and ILM), build-rustfs-debug-binary,
   # e2e-tests, e2e-full.
   warm-ci-dev:
     name: Warm ci-dev
@@ -146,7 +146,7 @@ jobs:
           if-no-files-found: error
 
       # --all-targets covers the test binaries nextest builds, including
-      # e2e_test, which the workspace run excludes. The second build adds
+      # e2e_test, which test-and-lint's own run excludes. The second build adds
       # the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
       # and that no lint lane enables.
       - name: Build ci-dev superset

.github/workflows/ci.yml

@@ -16,7 +16,9 @@ name: Continuous Integration
 
 on:
   push:
-    branches: [ main ]
+    # gateway/integration receives the daily merge of main as a direct push
+    # (rustfs/backlog#2734), so its pushes must prove the result like main's.
+    branches: [ main, gateway/integration ]
     paths-ignore:
       - "**.md"
       - "docs/**"
@@ -44,11 +46,6 @@ on:
   schedule:
     - cron: "11 0 * * *" # Nightly 00:11 UTC, after the 22:41 UTC cache-warm run
   workflow_dispatch:
-    inputs:
-      rehearse_workspace_shards:
-        description: Compare one full execution with at most two resource-safe shards of the same archive
-        type: boolean
-        default: false
 
 permissions:
   contents: read
@@ -92,7 +89,6 @@ jobs:
       connect: ${{ steps.scope.outputs.connect || 'true' }}
       packages: ${{ steps.scope.outputs.packages }}
       full_validation: ${{ steps.scope.outputs.full_validation || 'true' }}
-      rehearse_shards: ${{ steps.scope.outputs.rehearse_shards || 'false' }}
     steps:
       - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
@@ -103,7 +99,6 @@ jobs:
         env:
           CI_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }}
           CI_EVENT_ACTION: ${{ github.event.action }}
-          CI_REHEARSE_SHARDS: ${{ inputs.rehearse_workspace_shards && 'true' || 'false' }}
         run: |
           if [[ "$GITHUB_EVENT_NAME" != "pull_request" ]]; then
             python3 -I scripts/ci_gate.py select
@@ -113,9 +108,9 @@ jobs:
             printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
             echo "Base CI policy unavailable; running the full matrix."
           fi
-          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
-            printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
-            echo "Base policy does not cover this workflow contract; retaining the full matrix."
+          if ! grep -q '^plan=' "$GITHUB_OUTPUT"; then
+            printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' >> "$GITHUB_OUTPUT"
+            echo "Legacy policy has no verification plan; retaining the full matrix."
           fi
       - name: Retain verification plan
         if: steps.scope.outputs.plan != ''
@@ -160,7 +155,7 @@ jobs:
       - name: Run shared quick checks
         uses: ./.github/actions/quick-checks
 
-  workspace-lint-and-build:
+  test-and-lint:
     name: Workspace Lint and Build
     if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
     needs: [ quick-checks, classify-changes ]
@@ -193,13 +188,13 @@ jobs:
 
       - name: Prepare test evidence
         run: |
-          mkdir -p artifacts/workspace
+          mkdir -p artifacts/test-and-lint
           {
             echo "run_id=${GITHUB_RUN_ID}"
             echo "job=${GITHUB_JOB}"
             echo "runner=${RUNNER_NAME}"
             echo "started_at=$(date --utc --iso-8601=seconds)"
-          } > artifacts/workspace/run-metadata.txt
+          } > artifacts/test-and-lint/run-metadata.txt
 
       # Clippy gates every event, pull requests included. When it ran only after
       # merge, a lint regression merged freely and then failed every main run
@@ -224,8 +219,7 @@ jobs:
       - name: Upload workspace test binaries
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: workspace-test-binaries
-          overwrite: true
+          name: workspace-test-binaries-${{ github.run_attempt }}
           path: ${{ runner.temp }}/workspace-tests.tar.zst*
           compression-level: 0
           if-no-files-found: error
@@ -233,11 +227,11 @@ jobs:
 
       - name: Run documentation tests
         run: |
-          mkdir -p artifacts/workspace
+          mkdir -p artifacts/test-and-lint
           set +e
           timeout --verbose --signal=TERM --kill-after=30s 15m \
             python3 scripts/ci_test_artifact.py cargo doc --configuration workspace \
-            2>&1 | tee artifacts/workspace/doctest.log
+            2>&1 | tee artifacts/test-and-lint/doctest.log
           status=${PIPESTATUS[0]}
           {
             echo "command=cargo test --all --exclude e2e_test --doc"
@@ -246,7 +240,7 @@ jobs:
             echo
             echo "Remaining test-related processes:"
             pgrep -af 'cargo|rustdoc|target/.*/deps/' || true
-          } > artifacts/workspace/doctest-diagnostics.txt
+          } > artifacts/test-and-lint/doctest-diagnostics.txt
           exit "${status}"
 
       - name: Upload test reports and diagnostics
@@ -255,7 +249,7 @@ jobs:
         with:
           name: junit-test-and-lint-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
-            artifacts/workspace
+            artifacts/test-and-lint
           retention-days: 3
           if-no-files-found: error
 
@@ -268,13 +262,11 @@ jobs:
 
   workspace-tests:
     name: Workspace tests
-    needs: [ workspace-lint-and-build, classify-changes ]
+    needs: [ test-and-lint, classify-changes ]
     # The archive retains compile-time fixture paths and the large test set
     # requires the same runner image until the hosted-runtime pilot passes.
     runs-on: sm-standard-4
     timeout-minutes: 90
-    outputs:
-      workspace_shards: ${{ steps.shards.outputs.workspace_shards || '[]' }}
     env:
       CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
     steps:
@@ -286,21 +278,21 @@ jobs:
         run: chmod go-w "$(realpath "$HOME")"
       - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: workspace-test-binaries
+          name: workspace-test-binaries-${{ github.run_attempt }}
           path: ${{ runner.temp }}
       - name: Verify workspace archive identity
         run: python3 scripts/ci_test_artifact.py verify --configuration workspace --archive "$RUNNER_TEMP/workspace-tests.tar.zst"
 
       - name: Extract and list workspace tests
         run: |
-          mkdir -p artifacts/workspace
+          mkdir -p artifacts/test-and-lint
           cargo nextest list --profile ci --archive-file "$RUNNER_TEMP/workspace-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json \
-            > artifacts/workspace/core-test-listing.json
-          python3 scripts/check_test_wiring.py --check-core artifacts/workspace/core-test-listing.json
+            > artifacts/test-and-lint/core-test-listing.json
+          python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json
 
       - name: Run nextest tests
         run: |
-          mkdir -p artifacts/workspace
+          mkdir -p artifacts/test-and-lint
           rm -f target/nextest/ci/junit.xml
           ./scripts/ci/resource_sampler.sh start nextest
           trap './scripts/ci/resource_sampler.sh stop' EXIT
@@ -309,7 +301,7 @@ jobs:
             cargo nextest run --profile ci --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json \
             --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
             --status-level all --final-status-level all \
-            2>&1 | tee artifacts/workspace/nextest.log
+            2>&1 | tee artifacts/test-and-lint/nextest.log
           status=${PIPESTATUS[0]}
           {
             echo "command=workspace archive execution"
@@ -321,13 +313,13 @@ jobs:
             echo
             echo "Kernel OOM / kill events:"
             dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
-          } > artifacts/workspace/nextest-diagnostics.txt
+          } > artifacts/test-and-lint/nextest-diagnostics.txt
           exit "${status}"
 
       - name: Verify workspace execution completeness
         run: |
-          python3 scripts/ci_test_artifact.py evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
-          ./scripts/check_migration_gate_count.sh evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
+          python3 scripts/ci_test_artifact.py evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
+          ./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
 
       - name: Run ignored ILM integration tests serially
         if: needs.classify-changes.outputs.mode == 'full'
@@ -358,98 +350,14 @@ jobs:
           name: workspace-test-evidence-${{ github.run_attempt }}
           path: |
             target/nextest/ci/junit.xml
-            artifacts/workspace
+            artifacts/test-and-lint
             artifacts/ilm-integration
           retention-days: 3
           if-no-files-found: error
 
-      - name: Prepare bounded shards from this candidate's observed durations
-        id: shards
-        if: needs.classify-changes.outputs.rehearse_shards == 'true'
-        run: |
-          python3 scripts/ci_test_shards.py prepare \
-            --listing artifacts/workspace/core-test-listing.json \
-            --junit target/nextest/ci/junit.xml --output artifacts/workspace-shard-plan
-      - name: Upload same-run shard plan
-        if: needs.classify-changes.outputs.rehearse_shards == 'true'
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: workspace-shard-plan-${{ github.run_attempt }}
-          path: artifacts/workspace-shard-plan
-          retention-days: 3
-          if-no-files-found: error
-
-  workspace-shards:
-    name: Workspace shards (${{ matrix.shard }})
-    if: needs.classify-changes.outputs.rehearse_shards == 'true'
-    needs: [classify-changes, workspace-tests]
-    # Reuse the producer image and checkout path for the full RustFS archive.
-    runs-on: sm-standard-4
-    timeout-minutes: 90
-    strategy:
-      fail-fast: false
-      max-parallel: 2
-      matrix:
-        shard: ${{ fromJSON(needs.workspace-tests.outputs.workspace_shards || '[]') }}
-    env:
-      CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
-    steps:
-      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-      - uses: ./.github/actions/test-runtime
-      - name: Protect Connect test home
-        run: chmod go-w "$(realpath "$HOME")"
-      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: workspace-test-binaries-${{ github.run_attempt }}
-          path: ${{ runner.temp }}
-      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: workspace-shard-plan-${{ github.run_attempt }}
-          path: artifacts/workspace-shard-plan
-      - name: Verify and execute this shard without compilation
-        env:
-          SHARD_INDEX: ${{ matrix.shard }}
-        run: |
-          python3 scripts/ci_test_artifact.py verify --configuration workspace --archive "$RUNNER_TEMP/workspace-tests.tar.zst"
-          python3 scripts/ci_test_shards.py execute --plan artifacts/workspace-shard-plan \
-            --archive "$RUNNER_TEMP/workspace-tests.tar.zst" --index "$SHARD_INDEX" --output artifacts/workspace-shard-result
-      - name: Retain shard evidence including failures
-        if: always()
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: workspace-shard-result-${{ matrix.shard }}-${{ github.run_attempt }}
-          path: artifacts/workspace-shard-result
-          retention-days: 3
-          if-no-files-found: error
-
-  workspace-shard-evidence:
-    name: Workspace shard completeness
-    if: always() && needs.classify-changes.outputs.rehearse_shards == 'true'
-    needs: [classify-changes, workspace-tests, workspace-shards]
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: workspace-shard-plan-${{ github.run_attempt }}
-          path: artifacts/workspace-shard-plan
-      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          pattern: workspace-shard-result-*-${{ github.run_attempt }}
-          path: artifacts/workspace-shards
-      - name: Require the exact union of successful shard executions
-        run: |
-          python3 scripts/ci_test_shards.py verify --plan artifacts/workspace-shard-plan --evidence artifacts/workspace-shards
-          ./scripts/check_migration_gate_count.sh evidence artifacts/workspace-shard-plan/listing.json artifacts/workspace-shards/combined.xml
-
   connect-feature-boundaries:
     name: Connect Feature Boundaries
-    if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
+    if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && (github.event_name != 'pull_request' || github.event.action != 'closed')
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
@@ -497,72 +405,13 @@ jobs:
       - name: Check offline enrollment E2E root boundary
         run: ./scripts/check_offline_enrollment_e2e.sh
 
-  fips-provider:
-    name: FIPS provider boundaries
-    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
-    needs: [ quick-checks, classify-changes ]
-    # This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
-    runs-on: sm-standard-4
-    timeout-minutes: 90
-    strategy:
-      fail-fast: false
-      matrix:
-        mode: [default, fips]
-    env:
-      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
-      CARGO_BUILD_JOBS: "2"
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          rust-version: stable
-          cache-shared-key: ci-fips-provider
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
-
-      - name: Install FIPS build tools
-        if: matrix.mode == 'fips'
-        run: sudo apt-get install -y cmake golang clang libclang-dev
-
-      - name: Verify provider installation
-        shell: bash
-        run: |
-          args=()
-          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
-            args+=(--features fips)
-          fi
-          cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
-          cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
-          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
-          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
-          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
-            cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
-            cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
-            cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
-          fi
-          tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
-          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
-            tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
-          fi
-          for test_name in "${tests[@]}"; do
-            cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
-          done
-          cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
-
   test-and-lint-rio-v2:
     name: Test and Lint (rio-v2)
-    # Specialized boundary, hardware and alternate-feature lanes run only on
-    # the nightly schedule and manual dispatch. They do not gate a merge: the
-    # nightly run reports a regression within a day, and repeating these builds
-    # on every PR revision and merge group bought no earlier signal worth the
-    # runner occupancy. Workspace lint and tests, the core contract, the debug
-    # build, E2E smoke and the S3 lanes remain the pull-request gate.
-    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
+    # Alternate feature configurations run on the nightly schedule, merge
+    # groups and manual dispatch. The default workspace remains the PR gate and
+    # the per-push main baseline: repeating four feature builds on every merge
+    # bought nothing the nightly run does not report within a day.
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
@@ -599,12 +448,12 @@ jobs:
           # --profile ci so the quarantine list (and its junit flaky markers)
           # covers this leg too; the default profile is the local no-retry
           # profile and silently ignored quarantined flakes here (rustfs#6703).
-          mkdir -p artifacts/workspace
+          mkdir -p artifacts/test-and-lint
           rm -f target/nextest/ci/junit.xml
           ./scripts/ci/resource_sampler.sh start nextest-rio-v2
           trap './scripts/ci/resource_sampler.sh stop' EXIT
           cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
-            2>&1 | tee artifacts/workspace/nextest.log
+            2>&1 | tee artifacts/test-and-lint/nextest.log
           cargo test -p rustfs --doc --features rio-v2
 
       - name: Upload rio-v2 test reports and diagnostics
@@ -614,14 +463,14 @@ jobs:
           name: junit-test-and-lint-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
             target/nextest/ci/junit.xml
-            artifacts/workspace
+            artifacts/test-and-lint
           retention-days: 3
           if-no-files-found: error
 
   test-and-lint-protocols:
     name: "Test and Lint (${{ matrix.features.name }})"
     # Same cadence as test-and-lint-rio-v2 above.
-    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
@@ -690,7 +539,7 @@ jobs:
           path: |
             target/nextest/ci/junit.xml
             artifacts/protocol-tests
-            artifacts/workspace
+            artifacts/test-and-lint
           retention-days: 3
           if-no-files-found: warn
 
@@ -766,8 +615,7 @@ jobs:
       - name: Upload debug binary
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: rustfs-debug-binary
-          overwrite: true
+          name: rustfs-debug-binary-${{ github.run_attempt }}
           path: |
             target/debug/rustfs
             target/debug/rustfs.e2e.json
@@ -786,8 +634,7 @@ jobs:
       - name: Upload E2E test binaries
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: e2e-test-binaries
-          overwrite: true
+          name: e2e-test-binaries-${{ github.run_attempt }}
           path: ${{ runner.temp }}/e2e-tests.tar.zst*
           compression-level: 0
           if-no-files-found: error
@@ -826,8 +673,7 @@ jobs:
       - name: Upload debug binary
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: rustfs-debug-binary-rio-v2
-          overwrite: true
+          name: rustfs-debug-binary-rio-v2-${{ github.run_attempt }}
           path: |
             target/debug/rustfs
             target/debug/rustfs.e2e.json
@@ -842,7 +688,7 @@ jobs:
     # job had neither, so each closed/merged PR really ran the whole io_uring
     # suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
     # 30662728539) and kept the cancellation run in progress for minutes.
-    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
+    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
     needs: [ quick-checks, classify-changes ]
     # GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
     # a container, applies no seccomp filter that would block io_uring_setup — so
@@ -923,11 +769,11 @@ jobs:
       - uses: ./.github/actions/test-runtime
       - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: rustfs-debug-binary
+          name: rustfs-debug-binary-${{ github.run_attempt }}
           path: target/debug
       - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: e2e-test-binaries
+          name: e2e-test-binaries-${{ github.run_attempt }}
           path: ${{ runner.temp }}
       - name: Run core storage and authorization contract
         env:
@@ -956,6 +802,8 @@ jobs:
     # Retain the producer image and fixture paths until the runtime pilot passes.
     runs-on: sm-standard-4
     timeout-minutes: 30
+    env:
+      RUSTFS_S3_STACK: legacy
     steps:
       - name: Checkout repository
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -968,7 +816,7 @@ jobs:
       - name: Download E2E test binaries
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: e2e-test-binaries
+          name: e2e-test-binaries-${{ github.run_attempt }}
           path: ${{ runner.temp }}
 
       - name: Verify E2E archive identity
@@ -992,7 +840,7 @@ jobs:
       - name: Download debug binary
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: rustfs-debug-binary
+          name: rustfs-debug-binary-${{ github.run_attempt }}
           path: target/debug
 
       - name: Make binary executable
@@ -1037,7 +885,7 @@ jobs:
           path: |
             ${{ runner.temp }}/rustfs-e2e-smoke-logs/
             ${{ runner.temp }}/rustfs-e2e-smoke-list.json
-            artifacts/workspace
+            artifacts/test-and-lint
           if-no-files-found: warn
 
       - name: Upload e2e smoke JUnit report
@@ -1076,18 +924,122 @@ jobs:
           path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
           retention-days: 3
 
+  # The same e2e-smoke suite against the same debug binary with the gateway
+  # S3 stack selected (rustfs/backlog#2734). Report-only: it is outside the
+  # Test and Lint aggregate and becomes required at T2.10. The test-binary
+  # download, identity, membership, run and completeness steps must stay
+  # identical to e2e-tests; scripts/ci_gate.py enforces that.
+  e2e-tests-gateway:
+    name: e2e-smoke (gateway, report-only until T2.10)
+    needs: [ build-rustfs-debug-binary ]
+    runs-on: sm-standard-2
+    timeout-minutes: 30
+    continue-on-error: true
+    env:
+      RUSTFS_S3_STACK: gateway
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Install verification runtime
+        uses: ./.github/actions/test-runtime
+
+      - name: Download E2E test binaries
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: e2e-test-binaries-${{ github.run_attempt }}
+          path: ${{ runner.temp }}
+
+      - name: Verify E2E archive identity
+        run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
+
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+
+      - name: Install awscurl
+        run: |
+          python3 -m pip install --user --upgrade pip "awscurl==0.44"
+          echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
+
+      - name: Verify awscurl
+        run: test -x "$AWSCURL_PATH"
+
+      # Download after the cache restore so the freshly built binary from the
+      # build job always wins over anything restored into target/debug.
+      - name: Download debug binary
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: rustfs-debug-binary-${{ github.run_attempt }}
+          path: target/debug
+
+      - name: Make binary executable
+        run: chmod +x ./target/debug/rustfs
+
+      # Build the e2e test graph once. The archive is reused by the smoke
+      # selection guard, security exact-count check, and run below, avoiding a
+      # second compile of the same e2e_test target on cold runners (backlog#1645).
+      - name: Verify e2e smoke test membership
+        env:
+          NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
+          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
+        run: |
+          cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json > "${NEXTEST_LISTING}"
+          python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
+          ./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
+
+      # PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
+      # profile.e2e-smoke default-filter in .config/nextest.toml is the single
+      # wiring mechanism for e2e tests in CI — extend that filter instead of
+      # adding new e2e jobs here. Each test spawns its own rustfs server on a
+      # random port and reuses the downloaded debug binary above.
+      - name: Run e2e smoke suite
+        env:
+          NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
+          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
+          SAMPLER_INTERVAL_SECS: "15"
+        run: |
+          ./scripts/ci/resource_sampler.sh start e2e-smoke
+          trap './scripts/ci/resource_sampler.sh stop' EXIT
+          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
+            --status-level all --final-status-level all --failure-output final
+
+      - name: Verify e2e smoke execution completeness
+        run: python3 scripts/ci_test_artifact.py evidence "$RUNNER_TEMP/rustfs-e2e-smoke-list.json" target/nextest/e2e-smoke/junit.xml
+
+      - name: Upload e2e smoke diagnostics
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: e2e-smoke-gateway-diagnostics-${{ github.run_number }}-${{ github.run_attempt }}
+          path: |
+            ${{ runner.temp }}/rustfs-e2e-smoke-logs/
+            ${{ runner.temp }}/rustfs-e2e-smoke-list.json
+          if-no-files-found: warn
+
+      - name: Upload e2e smoke JUnit report
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: e2e-smoke-gateway-junit-${{ github.run_number }}-${{ github.run_attempt }}
+          path: target/nextest/e2e-smoke/junit.xml
+          if-no-files-found: warn
+
   e2e-full:
     name: End-to-End Tests (full merge gate)
     # Merge gate only (backlog#1149 ci-5): the never-automated user-visible
     # suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
     # security-boundary, ... — via the e2e-full nextest profile. Too heavy for
-    # every PR, so it is gated to main pushes, the merge queue, and manual
-    # dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
-    # owned by other lanes (see .config/nextest.toml profile.e2e-full).
+    # every PR, so it is gated to main and gateway/integration pushes, the merge
+    # queue, and manual dispatch. protocols / the 7 cluster suites / replication
+    # / #[ignore] are owned by other lanes (see .config/nextest.toml profile.e2e-full).
     if: >-
       github.event_name == 'workflow_dispatch' ||
       github.event_name == 'merge_group' ||
-      (github.event_name == 'push' && github.ref == 'refs/heads/main')
+      (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/gateway/integration'))
     needs: [ build-rustfs-debug-binary ]
     # Retain the producer image and fixture paths until the runtime pilot passes.
     runs-on: sm-standard-4
@@ -1104,7 +1056,7 @@ jobs:
       - name: Download E2E test binaries
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: e2e-test-binaries
+          name: e2e-test-binaries-${{ github.run_attempt }}
           path: ${{ runner.temp }}
 
       - name: Verify E2E archive identity
@@ -1166,7 +1118,7 @@ jobs:
       - name: Download debug binary
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: rustfs-debug-binary
+          name: rustfs-debug-binary-${{ github.run_attempt }}
           path: target/debug
 
       - name: Make binary executable
@@ -1278,7 +1230,7 @@ jobs:
       - name: Download debug binary
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: rustfs-debug-binary-rio-v2
+          name: rustfs-debug-binary-rio-v2-${{ github.run_attempt }}
           path: target/debug
 
       - name: Make binary executable
@@ -1329,7 +1281,7 @@ jobs:
       - name: Download debug binary
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: rustfs-debug-binary
+          name: rustfs-debug-binary-${{ github.run_attempt }}
           path: target/debug
 
       - name: Make binary executable
@@ -1403,7 +1355,7 @@ jobs:
       - name: Download debug binary
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: rustfs-debug-binary
+          name: rustfs-debug-binary-${{ github.run_attempt }}
           path: target/debug
 
       - name: Make binary executable
@@ -1448,13 +1400,10 @@ jobs:
       - classify-changes
       - typos
       - quick-checks
-      - workspace-lint-and-build
+      - test-and-lint
       - workspace-tests
-      - workspace-shards
-      - workspace-shard-evidence
       - core-contract
       - connect-feature-boundaries
-      - fips-provider
       - test-and-lint-rio-v2
       - test-and-lint-protocols
       - build-rustfs-debug-binary
@@ -1482,11 +1431,10 @@ jobs:
     needs:
       - classify-changes
       - connect-feature-boundaries
-      - fips-provider
       - required-checks
       - typos
       - quick-checks
-      - workspace-lint-and-build
+      - test-and-lint
       - workspace-tests
       - core-contract
       - test-and-lint-rio-v2

.github/workflows/docker.yml

@@ -329,75 +329,13 @@ jobs:
           echo "  - Create latest: $create_latest"
           echo "  - Source ref: $source_ref"
 
-  # Consume the trusted release certification for the exact version this
-  # workflow wraps and verify the certified input digests. A manual dispatch
-  # without that certification fails here and pushes no official tag; only
-  # results from the trusted verification sources carry release authority.
-  release-certification:
-    name: Release Certification
-    needs: build-check
-    if: needs.build-check.outputs.should_build == 'true'
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    permissions:
-      contents: read
-      actions: read
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-          ref: ${{ needs.build-check.outputs.source_ref }}
-
-      - name: Download and verify release certification
-        env:
-          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
-          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
-          VERSION: ${{ needs.build-check.outputs.version }}
-        shell: bash
-        run: |
-          set -euo pipefail
-          mkdir -p certification
-          run_id="$TRIGGERING_RUN_ID"
-          if [[ -z "$run_id" ]]; then
-            run_id="$(gh api --method GET \
-              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-              -f branch="$VERSION" -f status=success -F per_page=1 \
-              --jq '.workflow_runs[0].id // empty' || true)"
-          fi
-          if [[ -z "$run_id" ]]; then
-            echo "No certified Build and Release run found for ${VERSION}; failing closed" >&2
-            exit 1
-          fi
-          if ! gh run download "$run_id" -n release-certification -D certification; then
-            echo "Build run ${run_id} published no release-certification artifact; failing closed" >&2
-            exit 1
-          fi
-          python3 scripts/release/release_gate.py verify-promotion \
-            --plan .config/release-validation-plan.json \
-            --candidate certification/candidate.json \
-            --summary certification/release-gate-output \
-            --artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
-            --artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
-
-          # Input digest binding: every zip this image wraps must be the
-          # certified byte-for-byte content, never a mutable latest pointer.
-          while IFS=$'\t' read -r name location expected; do
-            echo "Verifying certified input ${name}"
-            actual="$(curl -fsSL "$location" | sha256sum | awk '{print $1}')"
-            if [[ "$actual" != "$expected" ]]; then
-              echo "Certified input ${name} does not match its recorded digest" >&2
-              exit 1
-            fi
-          done < <(jq -r '.deliverables[] | select(.name | contains("linux")) | [.name, .location, .sha256] | @tsv' certification/candidate.json)
-
   # Build multi-arch Docker images
   # Strategy: Build images using pre-built binaries from dl.rustfs.com
   # Supports both release and dev channel binaries based on build context
   # Only runs when should_build is true (which includes workflow success check)
   build-docker:
     name: Build Docker Images
-    needs: [ build-check, release-certification ]
+    needs: build-check
     if: needs.build-check.outputs.should_build == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -505,14 +443,6 @@ jobs:
           # Output tags
           echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
 
-          # Staging identity for this run's promotion: official tags are only
-          # created after the staged image verifies against the certification,
-          # and they point at this same digest (no rebuild-after-verify).
-          STAGING_TAG="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${VARIANT_SUFFIX}"
-          STAGING_TAGS="${{ env.REGISTRY_DOCKERHUB }}:${STAGING_TAG},${{ env.REGISTRY_GHCR }}:${STAGING_TAG},${{ env.REGISTRY_QUAY }}:${STAGING_TAG}"
-          echo "staging_tags=$STAGING_TAGS" >> "$GITHUB_OUTPUT"
-          echo "staging_tag=$STAGING_TAG" >> "$GITHUB_OUTPUT"
-
           # Generate labels
           LABELS="org.opencontainers.image.title=RustFS"
           LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
@@ -537,9 +467,7 @@ jobs:
           file: ${{ matrix.file }}
           platforms: ${{ env.DOCKER_PLATFORMS }}
           push: ${{ needs.build-check.outputs.should_push == 'true' }}
-          # Push to staging tags only: official tags are created by the
-          # certification-gated promotion step below from this same digest.
-          tags: ${{ steps.meta.outputs.staging_tags }}
+          tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           # No layer cache. This build compiles nothing — it downloads a
           # release zip and runs apk/apt — so the cache could only save the
@@ -565,114 +493,6 @@ jobs:
           no-cache: false
           pull: true
 
-      # A later-built wrapper: the staged image must complete this platform's
-      # own binary identity verification before any official tag is written.
-      # It may not inherit an unverified package's conclusion.
-      - name: Verify staged image against certification
-        if: needs.build-check.outputs.should_push == 'true'
-        env:
-          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
-          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
-          VERSION: ${{ needs.build-check.outputs.version }}
-        shell: bash
-        run: |
-          set -euo pipefail
-          mkdir -p certification
-          run_id="$TRIGGERING_RUN_ID"
-          if [[ -z "$run_id" ]]; then
-            run_id="$(gh api --method GET \
-              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-              -f branch="$VERSION" -f status=success -F per_page=1 \
-              --jq '.workflow_runs[0].id // empty' || true)"
-          fi
-          gh run download "$run_id" -n release-certification -D certification
-
-          STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
-          case "${{ matrix.variant }}" in
-            musl) libc="musl" ;;
-            glibc) libc="gnu" ;;
-          esac
-          for platform in linux/amd64 linux/arm64; do
-            case "$platform" in
-              linux/amd64) arch_sub="x86_64" ;;
-              linux/arm64) arch_sub="aarch64" ;;
-            esac
-            docker pull --platform "$platform" "$STAGING_REF"
-            container="$(docker create --platform "$platform" "$STAGING_REF")"
-            docker cp "$container:/usr/bin/rustfs" "$RUNNER_TEMP/staged-rustfs"
-            docker rm "$container" >/dev/null
-            actual="$(sha256sum "$RUNNER_TEMP/staged-rustfs" | awk '{print $1}')"
-            expected="$(jq -r --arg needle "linux-${arch_sub}-${libc}" \
-              '[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
-              certification/candidate.json)"
-            if [[ -z "$expected" || "$actual" != "$expected" ]]; then
-              echo "Staged image binary for ${platform} does not match the certified binary digest" >&2
-              exit 1
-            fi
-            python3 scripts/release/release_gate.py verify-promotion \
-              --plan .config/release-validation-plan.json \
-              --candidate certification/candidate.json \
-              --summary certification/release-gate-output \
-              --artifact "$RUNNER_TEMP/staged-rustfs" \
-              --artifact-role binary
-          done
-
-      - name: Promote staged image to official tags
-        if: needs.build-check.outputs.should_push == 'true'
-        shell: bash
-        run: |
-          set -euo pipefail
-          STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
-          IFS=',' read -r -a official_tags <<< "${{ steps.meta.outputs.tags }}"
-          for tag in "${official_tags[@]}"; do
-            docker buildx imagetools create -t "$tag" "$STAGING_REF"
-            echo "Promoted $tag -> $STAGING_REF"
-          done
-
-      # Downstream consumers (Helm) verify this record instead of inheriting
-      # an unverified wrapper's conclusion.
-      - name: Record image promotion
-        if: needs.build-check.outputs.should_push == 'true'
-        shell: bash
-        env:
-          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
-          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
-          VERSION: ${{ needs.build-check.outputs.version }}
-        run: |
-          set -euo pipefail
-          mkdir -p certification
-          run_id="$TRIGGERING_RUN_ID"
-          if [[ -z "$run_id" ]]; then
-            run_id="$(gh api --method GET \
-              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-              -f branch="$VERSION" -f status=success -F per_page=1 \
-              --jq '.workflow_runs[0].id // empty' || true)"
-          fi
-          gh run download "$run_id" -n release-certification -D certification
-          STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
-          digest="$(docker buildx imagetools inspect "$STAGING_REF" --format '{{json .Manifest}}' | jq -r '.digest')"
-          candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
-          jq -n \
-            --arg version "$VERSION" \
-            --arg digest "$digest" \
-            --arg staging_ref "$STAGING_REF" \
-            --arg candidate_sha256 "$candidate_sha" \
-            --arg tags "${{ steps.meta.outputs.tags }}" \
-            --slurpfile candidate certification/candidate.json \
-            '{schema_version: 1, version: $version, image_digest: $digest, staging_ref: $staging_ref,
-              promoted_tags: ($tags | split(",")), candidate_sha256: $candidate_sha256,
-              binary_sha256s: ($candidate[0].deliverables | map(.binary_sha256) | unique)}' \
-            > image-promotion-record.json
-
-      - name: Upload image promotion record
-        if: needs.build-check.outputs.should_push == 'true'
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: image-promotion-record${{ matrix.suffix }}
-          path: image-promotion-record.json
-          if-no-files-found: error
-          retention-days: 30
-
   # Note: Manifest creation is no longer needed as we only build one variant
   # Multi-arch manifests are automatically created by docker/build-push-action
 

.github/workflows/e2e-distributed.yml

@@ -17,8 +17,8 @@
 # Each selected test starts a real localhost cluster via
 # `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
 # case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
-# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
-# and manual dispatches execute the same fail-closed suite.
+# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
+# nightly runs, and manual dispatches all execute the same fail-closed suite.
 # Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
 #
 # Isolated pool filesystems: expand/decommission/rebalance cases require
@@ -33,6 +33,32 @@
 name: e2e-distributed
 
 on:
+  pull_request:
+    paths:
+      - "Cargo.lock"
+      - "Cargo.toml"
+      - ".config/nextest.toml"
+      - ".config/e2e-distributed-selection.txt"
+      - ".github/workflows/e2e-distributed.yml"
+      - "crates/audit/**"
+      - "crates/common/**"
+      - "crates/config/**"
+      - "crates/e2e_test/**"
+      - "crates/ecstore/**"
+      - "crates/filemeta/**"
+      - "crates/heal/**"
+      - "crates/iam/**"
+      - "crates/lock/**"
+      - "crates/madmin/**"
+      - "crates/notify/**"
+      - "crates/replication/**"
+      - "crates/s3-client/**"
+      - "crates/s3-ops/**"
+      - "crates/s3-types/**"
+      - "crates/scanner/**"
+      - "crates/storage-api/**"
+      - "crates/utils/**"
+      - "rustfs/**"
   workflow_dispatch:
     inputs:
       filter:

.github/workflows/e2e-upgrade.yml

@@ -12,19 +12,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Single-node upgrade and rollback lane.
-#
-# The weekly Monday schedule, version tags, and workflow_dispatch run this
-# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
-
 name: Upgrade Compatibility
 
 on:
+  pull_request:
+    paths:
+      - ".github/workflows/e2e-upgrade.yml"
+      - "crates/e2e_test/src/common.rs"
+      - "crates/e2e_test/src/distributed/**"
+      - "crates/e2e_test/src/fake_s3_target/**"
+      - "crates/e2e_test/src/lib.rs"
+      - "crates/e2e_test/src/replication_extension_test.rs"
+      - "crates/e2e_test/src/rpc_fault_proxy.rs"
+      - "crates/e2e_test/src/upgrade_compatibility_test.rs"
+      - "crates/ecstore/**"
+      - "crates/filemeta/**"
+      - "crates/heal/**"
+      - "crates/kms/**"
+      - "crates/storage-api/**"
+      - "rustfs/**"
+      - "Cargo.lock"
   push:
     tags:
       - "[0-9]*.[0-9]*.[0-9]*"
   schedule:
-    # 03:17 UTC every Monday.
     - cron: "17 3 * * 1"
   workflow_dispatch:
 
@@ -33,7 +44,7 @@ permissions:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: false
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:
   CARGO_TERM_COLOR: always

.github/workflows/helm-package.yml

@@ -30,102 +30,9 @@ permissions:
   contents: read
 
 jobs:
-  # Consume the same trusted release certification as every other publication
-  # exit and verify the image promotion record of the Docker workflow. The
-  # chart wraps no binary of its own, so it may only reference an image whose
-  # platform verification already passed; it must not inherit an unverified
-  # package's conclusion. A run without that evidence fails closed here and
-  # never reaches the chart repository.
-  release-certification:
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    permissions:
-      contents: read
-      actions: read
-    outputs:
-      version: ${{ steps.version.outputs.raw_tag }}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-
-      - name: Normalize release version
-        id: version
-        env:
-          RAW_INPUT: ${{ github.event.inputs.version }}
-          RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
-        run: |
-          set -euo pipefail
-          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
-            RAW="$RAW_INPUT"
-          else
-            RAW="$RAW_BRANCH"
-          fi
-          case "$RAW" in
-            refs/tags/*) RAW_TAG="${RAW#refs/tags/}" ;;
-            *) RAW_TAG="$RAW" ;;
-          esac
-          echo "raw_tag=$RAW_TAG" >> "$GITHUB_OUTPUT"
-
-      - name: Download and verify release certification and image record
-        env:
-          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
-          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
-          RAW_TAG: ${{ steps.version.outputs.raw_tag }}
-        shell: bash
-        run: |
-          set -euo pipefail
-          mkdir -p certification image-record
-
-          build_run="$TRIGGERING_RUN_ID"
-          if [[ -z "$build_run" ]]; then
-            build_run="$(gh api --method GET \
-              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-              -f branch="$RAW_TAG" -f status=success -F per_page=1 \
-              --jq '.workflow_runs[0].id // empty' || true)"
-          fi
-          if [[ -z "$build_run" ]]; then
-            echo "No certified Build and Release run found for ${RAW_TAG}; failing closed" >&2
-            exit 1
-          fi
-          if ! gh run download "$build_run" -n release-certification -D certification; then
-            echo "Build run ${build_run} published no release-certification artifact; failing closed" >&2
-            exit 1
-          fi
-          python3 scripts/release/release_gate.py verify-promotion \
-            --plan .config/release-validation-plan.json \
-            --candidate certification/candidate.json \
-            --summary certification/release-gate-output \
-            --artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
-            --artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
-
-          docker_run="$(gh api --method GET \
-            "repos/${GITHUB_REPOSITORY}/actions/workflows/docker.yml/runs" \
-            -f branch="$RAW_TAG" -f status=success -F per_page=1 \
-            --jq '.workflow_runs[0].id // empty' || true)"
-          if [[ -z "$docker_run" ]] || ! gh run download "$docker_run" -n image-promotion-record -D image-record; then
-            echo "No image promotion record for ${RAW_TAG}; a chart may not wrap an unverified image" >&2
-            exit 1
-          fi
-          candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
-          record_sha="$(jq -r '.candidate_sha256 // empty' image-record/image-promotion-record.json)"
-          if [[ "$record_sha" != "$candidate_sha" ]]; then
-            echo "Image promotion record does not belong to this certification" >&2
-            exit 1
-          fi
-          overlap="$(jq -r --slurpfile candidate certification/candidate.json \
-            '[.binary_sha256s[]] - ($candidate[0].deliverables | map(.binary_sha256)) | length' \
-            image-record/image-promotion-record.json)"
-          if [[ "$overlap" -ne 0 ]]; then
-            echo "Image promotion record carries binary digests outside the certification" >&2
-            exit 1
-          fi
-
   build-helm-package:
     runs-on: ubuntu-latest
     timeout-minutes: 30
-    needs: [ release-certification ]
     if: |
       (github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
       (

.github/workflows/nightly-gnu.yml

@@ -87,206 +87,18 @@ jobs:
           install-build-packaging-tools: 'false'
           install-test-tools: 'false'
 
-      - name: Download static console assets
-        shell: bash
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-        run: |
-          # Packaging tools are off on this lane, so setup does not install unzip.
-          if ! command -v unzip >/dev/null 2>&1; then
-            sudo apt-get update -qq
-            sudo apt-get install -y -qq unzip
-          fi
-          if ! command -v unzip >/dev/null 2>&1; then
-            echo "unzip is required to extract console assets" >&2
-            exit 1
-          fi
-
-          mkdir -p ./rustfs/static
-
-          verify_sha256() {
-            local expected="$1"
-            local file="$2"
-            local actual=""
-            local actual_line=""
-
-            if command -v sha256sum >/dev/null 2>&1; then
-              actual_line=$(sha256sum -- "$file") || return 1
-            elif command -v shasum >/dev/null 2>&1; then
-              actual_line=$(shasum -a 256 -- "$file") || return 1
-            else
-              echo "No SHA-256 verification tool found" >&2
-              return 1
-            fi
-            actual="${actual_line%%[[:space:]]*}"
-
-            if [ "$actual" = "$expected" ]; then
-              echo "SHA256 verified OK: $actual"
-              return 0
-            else
-              echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
-              return 1
-            fi
-          }
-
-          download_console_assets() {
-            local curl_bin="curl"
-            local python_bin="python3"
-            local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
-            local console_json="console-release.json"
-            local console_url
-            local console_sha256
-            local curl_auth_args=()
-            console_tag=""
-
-            chmod +w ./rustfs/static/LICENSE || true
-
-            if ! command -v "$python_bin" >/dev/null 2>&1; then
-              python_bin="python"
-            fi
-            if ! command -v "$python_bin" >/dev/null 2>&1; then
-              echo "No Python interpreter found for release metadata parsing" >&2
-              return 1
-            fi
-
-            if [[ -n "${GITHUB_TOKEN:-}" ]]; then
-              curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
-            fi
-
-            "$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
-              -o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
-
-            read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
-          import json
-          import re
-          import sys
-
-          with open(sys.argv[1], encoding="utf-8") as handle:
-              release = json.load(handle)
-
-          tag = release.get("tag_name", "") or "unknown"
-
-          for asset in release.get("assets", []):
-              name = asset.get("name", "")
-              digest = asset.get("digest", "")
-              url = asset.get("browser_download_url", "")
-              if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
-                  sha256 = digest.split(":", 1)[1]
-                  if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
-                      raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
-                  sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
-                  break
-          else:
-              raise SystemExit("no console zip asset with sha256 digest found")
-          PY
-            ) || return 1
-
-            if [[ -z "$console_url" || -z "$console_sha256" ]]; then
-              echo "Console release metadata is missing URL or SHA-256 digest" >&2
-              return 1
-            fi
-
-            echo "Console release: ${console_tag}"
-            echo "Downloading console asset: ${console_url}"
-            "$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
-            verify_sha256 "$console_sha256" console.zip || return 2
-            unzip -o console.zip -d ./rustfs/static || return 2
-          }
-
-          if download_console_assets; then
-            rm -f console.zip console-release.json
-          else
-            status=$?
-            rm -f console.zip console-release.json
-            if [[ "$status" -eq 2 ]]; then
-              echo "Console asset integrity verification failed" >&2
-            fi
-            echo "Failed to download verified console assets" >&2
-            exit 1
-          fi
-
-          if [[ ! -s ./rustfs/static/index.html ]]; then
-            echo "Console asset archive is missing static/index.html" >&2
-            exit 1
-          fi
-
-          asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
-          echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
-
       - name: Build RustFS
         # Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
         # nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
-        shell: bash
-        run: |
-          # Force rebuild by touching build.rs so a restored cache cannot
-          # reuse a rustfs artifact compiled before these console assets existed.
-          touch rustfs/build.rs
-          cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
+        run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
 
       - name: Verify glibc symbol baseline
         run: |
           python3 scripts/release/check_glibc.py 2.31 \
-            target/x86_64-unknown-linux-gnu/release/rustfs
+            target/x86_64-unknown-linux-gnu/release/rustfs \
+            target/x86_64-unknown-linux-gnu/release/rustfs-cli
           target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
 
-      - name: Verify console
-        shell: bash
-        run: |
-          set -euo pipefail
-
-          find_free_port() {
-            python3 - <<'PY'
-          import socket
-
-          with socket.socket() as sock:
-              sock.bind(("127.0.0.1", 0))
-              print(sock.getsockname()[1])
-          PY
-          }
-
-          binary="target/x86_64-unknown-linux-gnu/release/rustfs"
-          data_dir="$(mktemp -d)"
-          log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
-          server_pid=""
-          trap '
-            if [[ -n "${server_pid:-}" ]]; then
-              kill "$server_pid" 2>/dev/null || true
-              wait "$server_pid" 2>/dev/null || true
-            fi
-            rm -rf "$data_dir"
-          ' EXIT
-
-          if [[ ! -x "$binary" ]]; then
-            echo "rustfs binary not found: ${binary}" >&2
-            exit 1
-          fi
-
-          api_port="$(find_free_port)"
-          console_port="$(find_free_port)"
-          console_url="http://127.0.0.1:${console_port}/rustfs/console/"
-
-          "$binary" server \
-            --address "127.0.0.1:${api_port}" \
-            --console-enable \
-            --console-address "127.0.0.1:${console_port}" \
-            --access-key console-smoke \
-            --secret-key console-smoke-secret \
-            "$data_dir" >"$log_file" 2>&1 &
-          server_pid=$!
-
-          response=""
-          for _ in {1..40}; do
-            response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
-            if [[ "$response" == 200\ text/html* ]]; then
-              exit 0
-            fi
-            sleep 0.25
-          done
-
-          echo "Console endpoint did not return 200 text/html: $response" >&2
-          cat "$log_file"
-          exit 1
-
       - name: Build DEB package
         id: deb
         shell: bash
@@ -534,12 +346,12 @@ jobs:
           path: ${{ steps.publish.outputs.candidate_file }}
           if-no-files-found: error
 
+      # Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
+      # engineers can download and install the nightly directly. The branch is
+      # a single-commit orphan rewritten on every build, which keeps the repo
+      # small while the latest files stay reachable at stable raw URLs.
       # Publish the deb/rpm pair as assets of the rolling `nightly` release on
       # rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
-      # The dispatch below runs only after this upload and the immutable R2
-      # candidate both succeed. It does not replace rustfs-functional-chain.yml,
-      # which still starts from this workflow's workflow_run completion.
-      # kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
       - name: Publish packages to auto-testing release assets
         env:
           ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -548,90 +360,6 @@ jobs:
           DEB_DATE: ${{ steps.deb.outputs.deb_date }}
           BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
         run: bash scripts/release/publish_nightly_assets.sh
-
-      # Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
-      # PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
-      # or fine-grained Contents write (the permission GitHub checks for
-      # POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
-      # step, and therefore this job, so the scheduled failure alert fires.
-      # GitHub allows at most 10 top-level client_payload properties, so the
-      # schema 2 candidate object is nested under `candidate` and the release
-      # fields sit beside it.
-      - name: Dispatch nightly candidate to auto-testing
-        if: ${{ steps.publish.outputs.candidate_file != '' }}
-        env:
-          DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
-          CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
-          DEB_DATE: ${{ steps.deb.outputs.deb_date }}
-          SOURCE_REPO: ${{ github.repository }}
-        shell: bash
-        run: |
-          set -euo pipefail
-          : "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
-          : "${CANDIDATE_FILE:?nightly candidate manifest is required}"
-          : "${DEB_DATE:?DEB_DATE is required}"
-          [[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
-          [[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
-
-          payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
-          jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
-            --arg event_type "nightly-candidate-ready" \
-            --arg source_repo "${SOURCE_REPO}" \
-            --arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
-            --arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
-            --arg channel "nightly" \
-            --arg deb_date "${DEB_DATE}" \
-            --arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
-            '{
-              event_type: $event_type,
-              source_repo: $source_repo,
-              latest_alias_url: $latest_alias_url,
-              auto_testing_release: $auto_testing_release,
-              channel: $channel,
-              deb_date: $deb_date,
-              nightly_workflow: $nightly_workflow,
-              candidate: $candidate[0]
-            }' > "${payload}"
-          if ! jq -e '
-            .event_type == "nightly-candidate-ready"
-            and .source_repo == "rustfs/rustfs"
-            and .channel == "nightly"
-            and .deb_date == env.DEB_DATE
-            and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
-            and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
-            and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
-            and .candidate.schema == 2
-            and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
-            and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
-          ' "${payload}" >/dev/null; then
-            echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
-            exit 1
-          fi
-
-          body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
-          jq -n --slurpfile client_payload "${payload}" \
-            '{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
-          response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
-          status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
-            -o "${response}" -w '%{http_code}' \
-            -X POST \
-            -H "Authorization: Bearer ${DISPATCH_TOKEN}" \
-            -H "Accept: application/vnd.github+json" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            -H "Content-Type: application/json" \
-            --data-binary @"${body}" \
-            https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
-            echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
-            exit 1
-          }
-          if [[ "${status}" != "204" ]]; then
-            echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
-            if [[ -s "${response}" ]]; then
-              cat "${response}" >&2
-            fi
-            exit 1
-          fi
-          echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
   # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
   #
   # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and

.github/workflows/package.yml

@@ -225,57 +225,10 @@ jobs:
           echo "  Build run ID: $BUILD_RUN_ID"
           echo "  Build run number: $RUN_NUMBER"
 
-  # Consume the trusted release certification of the resolved build run and
-  # verify the certified input digests before any package is wrapped. A run
-  # without that certification fails closed here; packages built later must
-  # complete their own verification and cannot inherit an unverified
-  # conclusion.
-  release-certification:
-    name: Release Certification
-    needs: resolve
-    if: needs.resolve.result == 'success'
-    runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
-    timeout-minutes: 30
-    permissions:
-      contents: read
-      actions: read
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-
-      - name: Download and verify release certification
-        env:
-          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
-          BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
-          BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
-        shell: bash
-        run: |
-          set -euo pipefail
-          mkdir -p certification
-          if [[ "$BUILD_TYPE" == "development" ]]; then
-            # Development artifacts are explicitly uncertified and never touch
-            # official or latest channels; only the staging/dev paths remain.
-            jq -n '{schema_version: 1, certified: false, reason: "development artifacts are explicitly uncertified"}' \
-              > certification/development-uncertified.json
-            exit 0
-          fi
-          if ! gh run download "$BUILD_RUN_ID" -n release-certification -D certification; then
-            echo "Build run ${BUILD_RUN_ID} published no release-certification artifact; failing closed" >&2
-            exit 1
-          fi
-          python3 scripts/release/release_gate.py verify-promotion \
-            --plan .config/release-validation-plan.json \
-            --candidate certification/candidate.json \
-            --summary certification/release-gate-output \
-            --artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
-            --artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
-
   # Build DEB and RPM packages for each architecture and libc variant
   package:
     name: Package (${{ matrix.arch }}-${{ matrix.libc }})
-    needs: [ resolve, release-certification ]
+    needs: resolve
     runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
     timeout-minutes: 30
     strategy:
@@ -541,55 +494,6 @@ jobs:
           echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ RPM built: $RPM_FILE"
 
-      - name: Download certification evidence
-        if: needs.resolve.outputs.build_type != 'development'
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: release-certification
-          path: ./certification
-
-      # Later-built wrapper verification: the binary inside each package must
-      # match the certified binary digest for this platform before anything is
-      # uploaded, and the package itself stays on staging until promotion.
-      - name: Verify wrapped binary against certification
-        if: needs.resolve.outputs.build_type != 'development'
-        shell: bash
-        env:
-          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
-          RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
-          ARTIFACT_NAME: ${{ matrix.artifact_name }}
-        run: |
-          set -euo pipefail
-          expected="$(jq -r --arg needle "$ARTIFACT_NAME" \
-            '[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
-            certification/candidate.json)"
-          if [[ -z "$expected" ]]; then
-            echo "No certified binary digest recorded for ${ARTIFACT_NAME}" >&2
-            exit 1
-          fi
-          if ! command -v rpm2cpio >/dev/null 2>&1; then
-            sudo apt-get update && sudo apt-get install -y rpm2cpio
-          fi
-          if ! command -v cpio >/dev/null 2>&1; then
-            sudo apt-get update && sudo apt-get install -y cpio
-          fi
-          for package_file in "$DEB_FILE" "$RPM_FILE"; do
-            case "$package_file" in
-              *.deb)
-                dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
-                ;;
-              *.rpm)
-                rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
-                ;;
-            esac
-            python3 scripts/release/release_gate.py verify-promotion \
-              --plan .config/release-validation-plan.json \
-              --candidate certification/candidate.json \
-              --summary certification/release-gate-output \
-              --artifact "$RUNNER_TEMP/wrapped-rustfs" \
-              --artifact-role binary
-          done
-
       - name: Upload packages to artifacts
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
@@ -627,14 +531,10 @@ jobs:
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
-          # Build-phase writes stay on staging for release types: the official
-          # packages/release and packages/latest paths are written only by the
-          # certification-gated promote-packages job, from these same staged
-          # objects. Development packages keep the uncertified dev channel.
           if [[ "$BUILD_TYPE" == "development" ]]; then
             R2_PREFIX="artifacts/rustfs/packages/dev"
           else
-            R2_PREFIX="artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}"
+            R2_PREFIX="artifacts/rustfs/packages/release"
           fi
           R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
 
@@ -649,6 +549,18 @@ jobs:
 
           echo "✅ Upload complete"
 
+          # Also upload as latest for release/prerelease
+          if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
+            LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
+            for f in "$DEB_FILE" "$RPM_FILE"; do
+              if [[ -n "$f" && -f "$f" ]]; then
+                echo "Uploading latest: $(basename "$f")"
+                aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+              fi
+            done
+            echo "✅ Latest packages updated"
+          fi
+
       - name: Upload packages to GitHub Release
         if: needs.resolve.outputs.tag != ''
         env:
@@ -702,114 +614,6 @@ jobs:
 
           echo "✅ GitHub release assets updated"
 
-  # The only writer of the official packages/release and packages/latest R2
-  # paths. Promotion re-checks each package's wrapped binary against the
-  # passed certification list and copies the staged objects byte-for-byte:
-  # no rebuild-after-verify, no mutable latest resolution downstream.
-  promote-packages:
-    name: Promote Packages
-    needs: [ resolve, package, release-certification ]
-    # Target-channel lock for the official package paths.
-    concurrency:
-      group: release-channel-r2-packages
-      cancel-in-progress: false
-    if: needs.resolve.outputs.tag != '' && (needs.resolve.outputs.build_type == 'release' || needs.resolve.outputs.build_type == 'prerelease')
-    runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
-    timeout-minutes: 30
-    permissions:
-      contents: read
-      actions: read
-    env:
-      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
-      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
-      R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
-      R2_BUCKET: ${{ secrets.R2_BUCKET }}
-      AWS_EC2_METADATA_DISABLED: true
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-
-      - name: Download packages and certification
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          path: ./packages
-          pattern: packages-*
-          merge-multiple: true
-
-      - name: Download certification evidence
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: release-certification
-          path: ./certification
-
-      - name: Verify and promote staged packages
-        shell: bash
-        run: |
-          set -euo pipefail
-
-          if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
-            echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
-            exit 1
-          fi
-
-          STAGING_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
-          RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/release/"
-          LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
-
-          if ! command -v aws >/dev/null 2>&1; then
-            sudo apt-get update && sudo apt-get install -y awscli
-          fi
-          export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
-          export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
-          export AWS_DEFAULT_REGION="auto"
-
-          if ! command -v rpm2cpio >/dev/null 2>&1; then
-            sudo apt-get install -y rpm2cpio
-          fi
-          if ! command -v cpio >/dev/null 2>&1; then
-            sudo apt-get install -y cpio
-          fi
-
-          promoted=0
-          for package_file in packages/*.deb packages/*.rpm; do
-            [[ -f "$package_file" ]] || continue
-            name="$(basename "$package_file")"
-            case "$package_file" in
-              *.deb)
-                dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
-                ;;
-              *.rpm)
-                rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
-                ;;
-            esac
-            python3 scripts/release/release_gate.py verify-promotion \
-              --plan .config/release-validation-plan.json \
-              --candidate certification/candidate.json \
-              --summary certification/release-gate-output \
-              --artifact "$RUNNER_TEMP/wrapped-rustfs" \
-              --artifact-role binary
-
-            staged="$(mktemp)"
-            aws s3 cp "${STAGING_PREFIX}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
-            staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
-            built_sha="$(sha256sum "$package_file" | awk '{print $1}')"
-            if [[ "$staged_sha" != "$built_sha" ]]; then
-              echo "❌ Staged package ${name} does not match the verified bytes" >&2
-              exit 1
-            fi
-            aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
-            aws s3 cp "$staged" "${LATEST_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
-            rm -f "$staged"
-            promoted=$((promoted + 1))
-            echo "✅ Promoted ${name} (${built_sha})"
-          done
-          if [[ "$promoted" -eq 0 ]]; then
-            echo "No staged packages found to promote" >&2
-            exit 1
-          fi
-
   # Summary
   summary:
     name: Summary

.github/workflows/rustfs-fault-tolerance-test.yml

@@ -197,21 +197,21 @@ jobs:
           jq -e --arg testing_sha "${AUTO_TESTING_SHA}" '
             .schema_version == 1 and
             .auto_testing_sha == $testing_sha and
-            .selected_scenarios == ["A", "B", "C", "C2", "D", "E", "F"] and
+            .selected_scenarios == ["A", "B", "C", "C2", "D", "E"] and
             .complete == true and
-            .counts.expected == 54 and
-            .counts.seen == 54 and
+            .counts.expected == 44 and
+            .counts.seen == 44 and
             .counts.harness_errors == 0 and
-            (.expected_cases | length) == 54 and
-            (.expected_cases | unique | length) == 54 and
-            (.seen_cases | length) == 54 and
-            (.seen_cases | unique | length) == 54 and
+            (.expected_cases | length) == 44 and
+            (.expected_cases | unique | length) == 44 and
+            (.seen_cases | length) == 44 and
+            (.seen_cases | unique | length) == 44 and
             (.seen_cases | sort) == (.expected_cases | sort) and
             (.missing_cases | length) == 0 and
             (.harness_errors | length) == 0 and
-            (.cases | length) == 54 and
+            (.cases | length) == 44 and
             ([.cases[].case_id] | sort) == (.expected_cases | sort) and
-            ([.cases[].case_id] | unique | length) == 54 and
+            ([.cases[].case_id] | unique | length) == 44 and
             all(.cases[]; .verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED") and
             .counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and
             .counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length)

.github/workflows/rustfs-functional-chain.yml

@@ -281,20 +281,12 @@ jobs:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
           PERFORMANCE_OPTIONAL: ${{ contains(fromJSON(steps.selection.outputs.optional_suites), 'performance') }}
         run: |
-          # TEMPORARY: the pf-testing environment is broken, so the
-          # performance lane is disabled until it is repaired. Forcing
-          # offline keeps the chain on its designed skip path: the lane job
-          # is skipped, aggregate receives --allow-skipped performance
-          # (required=false in scheduled-validations.json), and the report
-          # records the optional-lane skip with its reason.
-          # Restore the probe below once the environment is healthy again.
-          # if python3 scripts/check_functional_runners.py pf-testing; then
-          #   echo "performance_ready=online" >> "$GITHUB_OUTPUT"
-          # else
-          #   echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
-          #   test "$PERFORMANCE_OPTIONAL" = true
-          # fi
-          echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
+          if python3 scripts/check_functional_runners.py pf-testing; then
+            echo "performance_ready=online" >> "$GITHUB_OUTPUT"
+          else
+            echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
+            test "$PERFORMANCE_OPTIONAL" = true
+          fi
 
   validation:
     if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && (inputs.validation_case != '' || inputs.validation_tools_only || inputs.validation_native_tools || inputs.validation_shared_tools || inputs.validation_manager_tools || inputs.validation_cleanup_run_id != '' || inputs.validation_cleanup_retained_evidence) }}

.github/workflows/rustfs-heal-test.yml

@@ -55,8 +55,7 @@ env:
 jobs:
   heal-test:
     runs-on: smoke-testing
-    # 90-minute heal run (see the step below) plus setup, report and cleanup.
-    timeout-minutes: 120
+    timeout-minutes: 60
     # Standalone manual run, or one link of the nightly functional chain
     # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
@@ -155,7 +154,7 @@ jobs:
           ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Run heal test (write -> outage -> heal -> verify)
-        timeout-minutes: 90
+        timeout-minutes: 45
         id: test
         # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true

.github/workflows/rustfs-validation-sweep.yml

@@ -1,457 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: RustFS Validation Sweep
-
-# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
-# on the shared functional fleet. Everything with a lane executor is swept in
-# rotating nightly batches so the whole set is covered roughly weekly; a
-# single case or one module can be dispatched manually. That is the
-# implemented catalog (66 today) plus the eight planned cases whose dedicated
-# executors already exist and have live PASS evidence - fresh nightly
-# evidence feeds their acceptance review. heal.* cases stay on the
-# interactive validation flow: they require reviewed native/tool collectors
-# built by separate validation-tools runs before fleet execution.
-
-on:
-  schedule:
-    # After the nightly functional chain settles (chain starts ~16:48 UTC and
-    # runs for about two hours).
-    - cron: "30 19 * * *"
-  workflow_dispatch:
-    inputs:
-      case:
-        description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
-        type: string
-        required: false
-        default: ''
-      module:
-        description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
-        type: string
-        required: false
-        default: ''
-      limit:
-        description: 'Maximum cases per sweep (0 = run everything selected)'
-        type: number
-        required: false
-        default: 12
-      build_run_id:
-        description: 'Successful nightly package run; empty resolves the latest'
-        type: string
-        required: false
-        default: ''
-      build_run_attempt:
-        description: 'Attempt of that package run'
-        type: string
-        required: false
-        default: '1'
-
-permissions:
-  contents: read
-
-concurrency:
-  group: rustfs-validation-sweep
-  cancel-in-progress: false
-
-env:
-  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
-  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
-  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
-  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
-  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
-  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-
-jobs:
-  sweep:
-    if: github.repository == 'rustfs/rustfs'
-    runs-on: smoke-testing
-    timeout-minutes: 150
-    steps:
-      - name: Checkout chain tooling
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-
-      - name: Resolve nightly candidate
-        id: candidate
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          BUILD_RUN_ID: ${{ inputs.build_run_id }}
-          BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
-          CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
-        run: |
-          set -euo pipefail
-          # Scheduled runs carry no inputs and the resolver only derives a
-          # build from a workflow_run payload (which a schedule cannot
-          # provide), so resolve the newest successful nightly build
-          # explicitly before invoking it.
-          if [ -z "${BUILD_RUN_ID:-}" ]; then
-            LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
-            BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
-            BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
-            export BUILD_RUN_ID BUILD_RUN_ATTEMPT
-            echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
-          fi
-          python3 scripts/resolve_functional_candidate.py
-          python3 - <<'PY'
-          import json
-          import os
-          from pathlib import Path
-          root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
-          root.mkdir(parents=True, exist_ok=True)
-          chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
-          manifest = chain["candidate"]["manifest"]
-          (root / "candidate.json").write_text(json.dumps({
-              "package_url": manifest["package_url"],
-              "package_sha256": manifest["package_sha256"],
-              "source_sha": manifest["source_sha"],
-          }, sort_keys=True) + "\n")
-          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
-              output.write("testing_sha=" + chain["testing_sha"] + "\n")
-              output.write("package_url=" + manifest["package_url"] + "\n")
-              output.write("source_sha=" + manifest["source_sha"] + "\n")
-          PY
-
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: ${{ steps.candidate.outputs.testing_sha }}
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          path: auto-testing
-          persist-credentials: false
-
-      - name: Prepare validation Python runtime
-        # The pool / ilm / replication validation runners require a managed
-        # Python, exactly like the chain's validation job provides.
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
-        with:
-          python-version: "3.12"
-
-      - name: Install validation Python requirements
-        # Case scripts call python3 and import boto3 at startup. Install the
-        # functional chain's client into a per-run env so that interpreter
-        # sees it without changing the runner's shared Python.
-        run: |
-          set -euo pipefail
-          validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          test ! -e "${validation_python}"
-          python3 -m venv "${validation_python}"
-          "${validation_python}/bin/python" -m pip install \
-            -r auto-testing/scripts/validation_ilm/requirements.txt
-          printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
-
-      - name: Select sweep cases
-        id: select
-        env:
-          SELECTED_CASE: ${{ inputs.case }}
-          SELECTED_MODULE: ${{ inputs.module }}
-          CASE_LIMIT: ${{ inputs.limit }}
-          SWEEP_EVENT: ${{ github.event_name }}
-          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
-        run: |
-          set -euo pipefail
-          python3 - <<'PY'
-          import datetime
-          import json
-          import os
-          from pathlib import Path
-          sweep_root = Path(os.environ["SWEEP_ROOT"])
-
-          catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
-          def sweepable(case):
-              status = case.get("source_status")
-              if status == "implemented":
-                  return True
-              # The remaining planned non-heal cases route to the lane
-              # executors and already have live PASS evidence; sweeping them
-              # nightly produces fresh evidence for their acceptance review.
-              return status == "planned" and not case["id"].startswith("heal.")
-
-          selected = [(case["id"], module["id"]) for module in catalog["modules"]
-                      for case in module["cases"]
-                      if sweepable(case)]
-          if not selected:
-              raise SystemExit("catalog has no sweepable cases")
-
-          one_case = os.environ.get("SELECTED_CASE", "").strip()
-          module_filter = os.environ.get("SELECTED_MODULE", "").strip()
-          known = {case_id for case_id, _ in selected}
-          modules = {module_id for _, module_id in selected}
-          if one_case:
-              if one_case not in known:
-                  raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
-              selected = [(one_case, dict(selected)[one_case])]
-          elif module_filter:
-              if module_filter not in modules:
-                  raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
-              selected = [pair for pair in selected if pair[1] == module_filter]
-          elif os.environ.get("SWEEP_EVENT") == "schedule":
-              # Stateless rotation: every night take one window of the catalog
-              # so a full sweep needs len(selected)/limit nights and no state
-              # has to be carried between runs.
-              limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
-              day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
-              start = (day * limit) % len(selected)
-              selected = (selected[start:] + selected[:start])[:limit]
-          limit = int(os.environ.get("CASE_LIMIT") or 0)
-          if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
-              if limit > 0:
-                  selected = selected[:limit]
-
-          (sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
-          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
-              output.write(f"count={len(selected)}\n")
-          print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
-          PY
-
-      - name: Admit clean shared environment
-        id: environment
-        timeout-minutes: 5
-        env:
-          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
-        run: python3 scripts/functional_environment.py enter --suite heal
-
-      - name: Run validation cases
-        id: run_cases
-        timeout-minutes: 120
-        env:
-          AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
-          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
-        run: |
-          set -uo pipefail
-          mkdir -p "${SWEEP_ROOT}/results"
-          : > "${SWEEP_ROOT}/results.txt"
-          while read -r CASE_ID; do
-            [ -n "${CASE_ID}" ] || continue
-            echo "::group::case ${CASE_ID}"
-            rm -rf "${SWEEP_ROOT}/run"
-            if ./auto-testing/scripts/run_validation_case.sh \
-                 "${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
-                 "${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
-                 > "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
-              printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
-              echo "case ${CASE_ID}: PASS"
-            else
-              printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
-              echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
-            fi
-            if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
-              cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
-            fi
-            echo "::endgroup::"
-            # Fleet hygiene between cases: the case runners set up their own
-            # topology, this purge guarantees the next case starts clean even
-            # when the previous one died mid-run.
-            for node in ${RUSTFS_NODES}; do
-              # -n keeps ssh from consuming the case list this loop reads from stdin
-              ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
-                SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-                ${SUDO} systemctl stop rustfs 2>/dev/null || true
-                if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                  ${SUDO} dpkg -P rustfs
-                fi
-                for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-                ${SUDO} rm -f /etc/default/rustfs
-              ' || echo "inter-case cleanup failed on ${node} (continuing)"
-            done
-          done < "${SWEEP_ROOT}/sweep-cases.txt"
-          echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
-
-      - name: Generate sweep report
-        if: ${{ always() && steps.environment.outcome == 'success' }}
-        env:
-          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
-          PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
-          SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
-          AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
-        run: |
-          set -euo pipefail
-          python3 - <<'PY'
-          import json
-          import os
-          from pathlib import Path
-
-          root = Path(os.environ["SWEEP_ROOT"])
-          catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
-          module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
-          rows = []
-          for line in (root / "results.txt").read_text().splitlines():
-            if not line:
-                continue
-            case_id, status = line.split("\t")
-            rows.append((case_id, module_of.get(case_id, "?"), status))
-          passed = sum(1 for _, _, s in rows if s == "PASS")
-          failed = len(rows) - passed
-          run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
-                     f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
-                     f"{os.environ.get('GITHUB_RUN_ID', '')}")
-
-          report = [
-              "# RustFS validation sweep report", "",
-              f"- Run: {run_url}",
-              f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
-              f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
-              f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
-              f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
-              f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
-              "Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
-          ]
-          # Group the executed cases by catalog module (the major category:
-          # pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
-          # data services, identity, s3 compatibility). Per-module subtotals
-          # deliberately use "- Subtotal:" so the dashboard overview regexes
-          # (which anchor on "- Total:") keep reading the grand totals below.
-          by_module = {}
-          module_order = []
-          for case_id, module, status in rows:
-              if module not in by_module:
-                  by_module[module] = []
-                  module_order.append(module)
-              by_module[module].append((case_id, status))
-          report += ["## Case Summary", ""]
-          for module in module_order:
-              cases = by_module[module]
-              mod_pass = sum(1 for _, st in cases if st == "PASS")
-              mod_fail = len(cases) - mod_pass
-              report += [f"### {module} ({len(cases)})", "",
-                         "| Case | Status |", "| --- | --- |"]
-              report += [f"| {case_id} | {status} |" for case_id, status in cases]
-              report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
-          report += [
-              "## Totals", "",
-              f"- Total: {len(rows)}",
-              f"- PASS: {passed}",
-              f"- FAIL: {failed}",
-              "- UNSUPPORTED: 0", "",
-              "Failing cases keep this workflow red only as a report; the sweep artifact",
-              "retains every case log and per-case report for diagnosis.", "",
-          ]
-          (root / "report.md").write_text("\n".join(report))
-          (root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
-          PY
-          cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
-
-      - name: Upload functional report to dashboard
-        if: ${{ always() && steps.environment.outcome == 'success' }}
-        timeout-minutes: 2
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
-          SUITE: validation
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
-            exit 0
-          fi
-          DATE="$(date -u +%Y-%m-%d)"
-          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
-          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          fi
-          echo "dashboard updated: ${REPORT_PATH}"
-
-      - name: Retain sweep evidence
-        if: ${{ always() }}
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
-          path: |
-            ${{ runner.temp }}/validation-sweep/report.md
-            ${{ runner.temp }}/validation-sweep/results.txt
-            ${{ runner.temp }}/validation-sweep/sweep-cases.txt
-            ${{ runner.temp }}/validation-sweep/candidate.json
-            ${{ runner.temp }}/validation-sweep/results/
-            ${{ runner.temp }}/validation-sweep/run/
-          # warn (not error): an early infra failure legitimately leaves no
-          # per-case evidence, and the failing step is already red.
-          if-no-files-found: warn
-          retention-days: 30
-
-      - name: Cleanup environment (after)
-        id: environment_cleanup
-        timeout-minutes: 5
-        if: ${{ always() && steps.environment.outcome == 'success' }}
-        run: |
-          set -uo pipefail
-          cleanup_status=0
-          for node in ${RUSTFS_NODES}; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -f /etc/default/rustfs
-            ' || cleanup_status=$?
-          done
-          exit "${cleanup_status}"
-
-      - name: Release environment only after successful cleanup
-        if: ${{ always() && steps.environment.outcome == 'success' }}
-        timeout-minutes: 5
-        env:
-          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
-          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
-        run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
-
-      - name: Notify on failure
-        if: failure()
-        run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
-
-      - name: Enforce sweep result
-        if: ${{ always() && steps.environment.outcome == 'success' }}
-        env:
-          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
-        run: |
-          set -euo pipefail
-          if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
-            echo "validation sweep has failing cases:"
-            grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
-            exit 1
-          fi
-
-  alert-on-failure:
-    name: Alert on scheduled failure
-    # A scheduled sweep failure usually means a canonical validation case
-    # regressed or the fleet environment broke; surface it as a tracking
-    # issue instead of leaving it in the Actions tab.
-    needs: [sweep]
-    if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
-        with:
-          persist-credentials: false
-      - name: Open or update failure-tracking issue
-        uses: ./.github/actions/schedule-failure-issue
-        with:
-          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

ARCHITECTURE.md

@@ -155,7 +155,7 @@ default build (lifecycle:
    - ⚠️ PARTIALLY VIOLATED: serving-side `s3s` references remain in ecstore
      (bucket metadata/replication/lifecycle DTOs and error mapping). The
      count is ratcheted shrink-only by `scripts/check_s3s_footprint.sh`
-     (`S3S_ECSTORE_FILES_BASELINE`; the `object_lock` module was converted to
+     (the `ecstore_files` line of `.config/s3s-footprint-baseline.txt`; the `object_lock` module was converted to
      storage-level types as the first ratchet step). Target state: the
      baseline reaches zero and ecstore's `Cargo.toml` drops `s3s`.
 

CHANGELOG.md

@@ -26,7 +26,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
 - A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
 - Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
-- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
 - Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
 
 ### Security

Cargo.toml

@@ -48,7 +48,7 @@ members = [
     "crates/replication", # Replication contracts and wire formats
     "crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
     "crates/s3-client", # S3 client for engine-side consumption of remote S3 endpoints (tiering, transition targets)
-    "crates/s3-types", # S3 event type definitions
+    "crates/s3-types", # S3 event and error type definitions
     "crates/s3-ops", # S3 operation definitions and mapping
     "crates/s3select-api", # S3 Select API interface
     "crates/s3select-query", # S3 Select query engine
@@ -143,11 +143,18 @@ rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.2" }
 rustfs-utils = { path = "crates/utils", version = "1.0.2" }
 rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
 
+# The gateway's S3 types: the DTOs and persisted bucket-configuration codecs the
+# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
+# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
+# the s3s-facing edge of the rustfs binary.
+rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.32.0", default-features = false }
+rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.2.0", default-features = false }
+
 # Async Runtime and Networking
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
 mysql_async = { default-features = false, version = "0.37.1" }
-async-compression = { version = "0.4.52" }
+async-compression = { version = "0.4.51" }
 async-trait = "0.1.92"
 async-nats = { version = "0.51.0", default-features = false }
 axum = "0.8.9"
@@ -168,16 +175,16 @@ http-body-util = "0.1.5"
 memchr = "2.8.3"
 minlz = "1.2.3"
 reqwest = "0.13.5"
-rustfs-kafka-async = { version = "1.4.1" }
+rustfs-kafka-async = { version = "1.4.0" }
 socket2 = { version = "0.6.5" }
 tokio = { version = "1.54.1" }
 tokio-rustls = { default-features = false, version = "0.26.6" }
 tokio-stream = { version = "0.1.19" }
 tokio-test = "0.4.6"
 tokio-util = { version = "0.7.20" }
-tonic = { version = "0.14.9" }
-tonic-prost = { version = "0.14.9" }
-tonic-prost-build = { version = "0.14.9" }
+tonic = { version = "0.14.8" }
+tonic-prost = { version = "0.14.8" }
+tonic-prost-build = { version = "0.14.8" }
 tower = { version = "0.5.3" }
 tower-http = { version = "0.7.1" }
 h3 = "0.0.8"
@@ -191,7 +198,6 @@ quinn = { version = "0.11.12", default-features = false, features = [
 # Serialization and Data Formats
 apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
 bytes = { version = "1.12.1" }
-csv-core = "0.1.13"
 bytesize = "2.7.0"
 byteorder = "1.5.0"
 flatbuffers = "25.12.19"
@@ -213,13 +219,12 @@ serde_urlencoded = "0.7.1"
 # releases.
 aes-gcm = { version = "0.11.1" }
 argon2 = { version = "0.6.0" }
-aws-lc-rs = { version = "1.19.0", default-features = false }
 blake2 = "0.11.0"
 chacha20poly1305 = { version = "0.11.0" }
 crc-fast = "1.10.0"
 hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "11.1.0" }
-openidconnect = { default-features = false, version = "4.1.0" }
+openidconnect = { default-features = false, version = "4.0" }
 pbkdf2 = "0.13.0"
 p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
 rsa = { version = "=0.10.0-rc.18" }
@@ -236,7 +241,7 @@ proptest = "1"
 # Time and Date
 chrono = { version = "0.4.45" }
 humantime = "2.4.0"
-jiff = { version = "0.2.39" }
+jiff = { version = "0.2.38" }
 time = { version = "0.3.55" }
 
 # Database
@@ -255,9 +260,9 @@ atoi = "3.1.1"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.13.0" }
 aws-credential-types = { version = "1.3.1" }
-aws-sdk-kms = { default-features = false, version = "1.129.0" }
-aws-sdk-s3 = { default-features = false, version = "1.157.0" }
-aws-sdk-sts = { default-features = false, version = "1.124.0" }
+aws-sdk-kms = { default-features = false, version = "1.128.0" }
+aws-sdk-s3 = { default-features = false, version = "1.156.0" }
+aws-sdk-sts = { default-features = false, version = "1.123.0" }
 aws-smithy-async = { version = "1.3.1" }
 aws-smithy-http-client = { default-features = false, version = "1.5.2" }
 aws-smithy-runtime-api = { version = "1.19.2" }
@@ -321,13 +326,17 @@ rustfs-erasure-codec = { version = "9.0.0" }
 reed-solomon-simd = "3.1.0"
 regex = { version = "1.13.1" }
 rumqttc = { package = "rumqttc-next", version = "0.34.0" }
-redis = { version = "1.8.1" }
+redis = { version = "1.8.0" }
 rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.6" }
 rust-embed = { version = "8.13.0" }
 rustc-hash = { version = "2.1.3" }
 s3s = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
 s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" }
+# The same s3s pin under the name the legacy edge of the gateway migration uses
+# (crates/s3-types, feature `compat-s3s`). Keep `rev` identical to `s3s` above.
+# DELETE BY T4.2 (rustfs/backlog#2784).
+legacy-s3s = { package = "s3s", git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
 serial_test = "4.0.1"
 shadow-rs = { default-features = false, version = "2.0.0" }
 siphasher = "1.0.4"
@@ -336,10 +345,11 @@ compact_str = "0.10.0"
 snap = "1.1.2"
 starshard = { version = "2.4.0" }
 strum = { version = "0.28.0" }
-sysinfo = "0.40.1"
+sync_wrapper = "1.0.2"
+sysinfo = "0.40.0"
 temp-env = "0.3.6"
 tempfile = "3.27.0"
-test-case = "3.4.1"
+test-case = "3.4.0"
 thiserror = "2.0.21"
 tracing = { version = "0.1.44" }
 tracing-appender = "0.2.5"
@@ -358,13 +368,13 @@ winapi-util = "0.1.11"
 windows = { version = "0.62.2" }
 windows-sys = "0.61.2"
 xxhash-rust = { version = "0.8.19" }
-zip = "9.3.0"
+zip = "9.1.3"
 zstd = "0.14.3"
 
 # Observability and Metrics
 metrics = "0.24.6"
 metrics-util = "0.20"
-dial9 = { version = "0.5.7", default-features = false }
+dial9 = { version = "0.5.6", default-features = false }
 opentelemetry = { version = "0.33.1" }
 opentelemetry-appender-tracing = { version = "0.33.1" }
 opentelemetry-otlp = { version = "0.33.1" }
@@ -379,16 +389,16 @@ libunftp = { version = "0.24.0" }
 unftp-core = "0.1.0"
 suppaftp = { version = "12.2.1" }
 rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
-russh = { version = "0.65.1" }
+russh = { version = "0.65.0" }
 russh-sftp = "3.0.1"
 
 # WebDAV
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-rustfs-mimalloc = { version = "0.7.3" }
+rustfs-mimalloc = { version = "0.7.2" }
 # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
-hotpath = { version = "0.30.3", default-features = false }
+hotpath = { version = "0.30.1", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.49" }
 
@@ -429,3 +439,7 @@ strip = "none"
 # Pin upstream fixes until compatible crates.io releases are available.
 [patch.crates-io]
 zstd = { path = "crates/zstd-compat" }
+# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
+# Upstream: openidconnect-rs#244 and oauth2-rs#352.
+openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
+oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }

crates/concurrency/src/adaptive.rs

@@ -10,10 +10,6 @@ use std::time::Duration;
 use tokio::sync::{Notify, Semaphore};
 use tokio::time::Instant;
 
-pub mod permit_set;
-
-const MAX_PERMIT_REFERENCES: usize = usize::MAX / 2;
-
 /// A resource owner publishes a capacity; running units retain their permits
 /// after a reduction. Lease slots are reserved once, with no per-unit Arc
 /// allocation. Tokio wait queues may lazily initialize platform mutex storage
@@ -40,7 +36,6 @@ pub struct AdaptiveAdmission {
     maximum: u32,
     slots: Box<[PermitSlots]>,
     next_chunk: AtomicUsize,
-    completed: AtomicU64,
 }
 
 struct PermitSlots {
@@ -66,12 +61,10 @@ pub struct AdaptivePermit {
 
 impl Clone for AdaptivePermit {
     fn clone(&self) -> Self {
-        // Subset carriers can retain slot references through a shared owner,
-        // so the slot count needs its own bound in addition to Arc's guard.
+        // Every slot reference owns a strong owner reference. Clone the Arc
+        // first, so its overflow protection also bounds the slot refcount.
         let owner = Arc::clone(&self.owner);
-        if owner.slots[self.slot / 64].references[self.slot % 64].fetch_add(1, Ordering::Relaxed) >= MAX_PERMIT_REFERENCES {
-            std::process::abort();
-        }
+        owner.slots[self.slot / 64].references[self.slot % 64].fetch_add(1, Ordering::Relaxed);
         Self { owner, slot: self.slot }
     }
 }
@@ -121,22 +114,11 @@ impl NodeAdmission {
 
     /// Publish feedback with this owner's actual completed-work counter.
     pub fn update(&self, controller: &mut FeedbackController, observation: Option<FeedbackObservation>) -> FeedbackDecision {
-        self.update_with_probe(controller, observation, true)
-    }
-
-    /// Coordinate growth with other resource controllers in the same window.
-    /// Protective reductions never depend on permission to probe.
-    pub fn update_with_probe(
-        &self,
-        controller: &mut FeedbackController,
-        observation: Option<FeedbackObservation>,
-        allow_probe: bool,
-    ) -> FeedbackDecision {
         let observation = observation.map(|mut observation| {
             observation.completed_units = self.completed();
             observation
         });
-        controller.update_with_probe(&self.capacity, observation, allow_probe)
+        controller.update(&self.capacity, observation)
     }
 
     /// Current leased scheduling cap and held async units.
@@ -267,31 +249,16 @@ impl FeedbackController {
     /// Consume one sample and publish a three-second lease to the same gate.
     /// If this function stops being called, admission expires independently.
     pub fn update(&mut self, gate: &AdaptiveAdmission, observation: Option<FeedbackObservation>) -> FeedbackDecision {
-        self.update_with_probe(gate, observation, true)
-    }
-
-    /// Allow at most the sampling owner's selected controller to probe upward.
-    pub fn update_with_probe(
-        &mut self,
-        gate: &AdaptiveAdmission,
-        observation: Option<FeedbackObservation>,
-        allow_probe: bool,
-    ) -> FeedbackDecision {
         let observation = observation.map(|mut observation| {
             observation.max_concurrency = observation.max_concurrency.min(gate.maximum);
             observation
         });
-        let decision = self.decide_with_probe(Instant::now(), observation, allow_probe);
+        let decision = self.decide(Instant::now(), observation);
         gate.set_sampled_limit(decision.limit, Duration::from_secs(3));
         decision
     }
 
-    fn decide_with_probe(
-        &mut self,
-        now: Instant,
-        observation: Option<FeedbackObservation>,
-        allow_probe: bool,
-    ) -> FeedbackDecision {
+    fn decide(&mut self, now: Instant, observation: Option<FeedbackObservation>) -> FeedbackDecision {
         let Some(observation) = observation else {
             self.previous = None;
             self.stable_since = None;
@@ -336,8 +303,7 @@ impl FeedbackController {
             FeedbackReason::NewBaseline
         } else {
             let since = *self.stable_since.get_or_insert(now);
-            if allow_probe
-                && advanced
+            if advanced
                 && now.saturating_duration_since(since) >= Duration::from_secs(8)
                 && now.saturating_duration_since(self.last_change) >= Duration::from_secs(8)
                 && self.limit < observation.max_concurrency
@@ -359,13 +325,6 @@ impl FeedbackController {
 }
 
 impl AdaptiveAdmission {
-    /// Successful async operations using this physical owner. Native clones
-    /// retain capacity but never count another completion.
-    pub fn completed(&self) -> Option<u64> {
-        let value = self.completed.load(Ordering::Acquire);
-        (value != u64::MAX).then_some(value)
-    }
-
     /// Construct a bounded owner, with zero initial capacity allowed for unknown
     /// input. The immutable maximum reserves at most 65,535 lease slots once;
     /// acquiring or cloning a permit does not allocate a per-unit Arc.
@@ -392,7 +351,6 @@ impl AdaptiveAdmission {
             maximum: maximum_u32,
             slots,
             next_chunk: AtomicUsize::new(0),
-            completed: AtomicU64::new(0),
         })
     }
 
@@ -508,21 +466,6 @@ impl AdaptiveAdmission {
         self.slots[slot / 64].free.fetch_or(1_u64 << (slot % 64), Ordering::Release);
     }
 
-    fn release_reference(&self, slot: usize) {
-        self.release_reference_with(slot, || {});
-    }
-
-    fn release_reference_with(&self, slot: usize, before_release: impl FnOnce()) {
-        if self.slots[slot / 64].references[slot % 64].fetch_sub(1, Ordering::AcqRel) != 1 {
-            return;
-        }
-        before_release();
-        // Publish the free slot before exposing another admission capacity.
-        self.release_slot(slot);
-        self.state.fetch_sub(1, Ordering::AcqRel);
-        self.changed.notify_one();
-    }
-
     /// Wait for capacity. The future owns no permit until it returns; dropping
     /// a pending future leaves capacity untouched. Wrap in the caller's existing
     /// cancellation/deadline select before acquiring any storage commit locks.
@@ -550,7 +493,13 @@ impl AdaptiveAdmission {
 
 impl Drop for AdaptivePermit {
     fn drop(&mut self) {
-        self.owner.release_reference(self.slot);
+        if self.owner.slots[self.slot / 64].references[self.slot % 64].fetch_sub(1, Ordering::AcqRel) != 1 {
+            return;
+        }
+        // Publish the free slot before exposing another admission capacity.
+        self.owner.release_slot(self.slot);
+        self.owner.state.fetch_sub(1, Ordering::AcqRel);
+        self.owner.changed.notify_one();
     }
 }
 
@@ -912,24 +861,6 @@ mod tests {
         }
     }
 
-    #[tokio::test(start_paused = true)]
-    async fn shared_probe_permission_blocks_growth_but_never_protective_cuts() {
-        let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bound"), 0);
-        let mut controller = FeedbackController::new();
-        controller.update_with_probe(&gate, Some(healthy(0)), false);
-        for completed in 1..=20 {
-            tokio::time::advance(Duration::from_secs(1)).await;
-            assert_eq!(controller.update_with_probe(&gate, Some(healthy(completed)), false).limit, 1);
-        }
-        tokio::time::advance(Duration::from_secs(1)).await;
-        assert_eq!(controller.update_with_probe(&gate, Some(healthy(21)), true).reason, FeedbackReason::Probe);
-        assert_eq!(gate.snapshot().limit, 2);
-        let mut busy = healthy(22);
-        busy.pressure = ResourcePressure::Busy;
-        assert_eq!(controller.update_with_probe(&gate, Some(busy), false).limit, 1);
-        assert_eq!(controller.update_with_probe(&gate, None, false).limit, 0);
-    }
-
     #[tokio::test(start_paused = true)]
     async fn controller_grows_one_unit_per_cooldown_and_cuts_pressure_immediately() {
         let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);

crates/config/src/constants/env.rs

@@ -30,16 +30,6 @@ pub const COMMENT_KEY: &str = "comment";
 pub const ENV_AUDIT_ENABLE: &str = "RUSTFS_AUDIT_ENABLE";
 /// Global switch for enabling the notify module.
 pub const ENV_NOTIFY_ENABLE: &str = "RUSTFS_NOTIFY_ENABLE";
-/// Maximum audit entries per second for authorization denials of authenticated callers.
-/// `0` disables denial auditing for that class of caller.
-pub const ENV_AUDIT_DENIAL_RATE: &str = "RUSTFS_AUDIT_DENIAL_RATE";
-/// Maximum audit entries per second for authorization denials of anonymous callers.
-/// `0` disables denial auditing for that class of caller.
-pub const ENV_AUDIT_DENIAL_ANONYMOUS_RATE: &str = "RUSTFS_AUDIT_DENIAL_ANONYMOUS_RATE";
-/// Default denial audit rate for authenticated callers (entries per second).
-pub const DEFAULT_AUDIT_DENIAL_RATE: u64 = 50;
-/// Default denial audit rate for anonymous callers (entries per second).
-pub const DEFAULT_AUDIT_DENIAL_ANONYMOUS_RATE: u64 = 5;
 /// Default global audit switch (disabled by default).
 pub const DEFAULT_AUDIT_ENABLE: bool = false;
 /// Default global notify switch (disabled by default).
@@ -338,7 +328,5 @@ mod tests {
     fn test_global_audit_notify_switch_constants() {
         assert_eq!(ENV_AUDIT_ENABLE, "RUSTFS_AUDIT_ENABLE");
         assert_eq!(ENV_NOTIFY_ENABLE, "RUSTFS_NOTIFY_ENABLE");
-        assert_eq!(ENV_AUDIT_DENIAL_RATE, "RUSTFS_AUDIT_DENIAL_RATE");
-        assert_eq!(ENV_AUDIT_DENIAL_ANONYMOUS_RATE, "RUSTFS_AUDIT_DENIAL_ANONYMOUS_RATE");
     }
 }

crates/config/src/constants/proxy.rs

@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
 /// Default validation mode is "hop_by_hop".
 pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
 
-/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
+/// Environment variable to enable RFC 7239 "Forwarded" header support.
 pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
-/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
-pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
+/// RFC 7239 support is enabled by default.
+pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
 
 /// Environment variable for the maximum allowed proxy hops.
 pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";

crates/config/src/constants/targets.rs

@@ -146,9 +146,3 @@ pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
 
 /// Queue-store compression is enabled by default to reduce disk footprint.
 pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
-
-/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
-pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
-
-/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
-pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;

crates/data-usage/src/data_usage.rs

@@ -24,14 +24,6 @@ use std::{
     time::{Duration, SystemTime},
 };
 
-/// Copy an existing cache-model value without including surrounding work.
-#[inline]
-pub fn clone_cache_value<T: ToOwned + ?Sized>(source: &T) -> T::Owned {
-    #[cfg(feature = "hotpath-alloc")]
-    let _copy_scope = hotpath::MeasurementGuardSync::new("cache_model.owned_copy", false, false);
-    source.to_owned()
-}
-
 /// Maximum amount a persisted `last_update` may lead the local wall clock before the
 /// persisted timestamp is treated as untrustworthy.
 ///
@@ -350,7 +342,7 @@ impl UnknownTierStats {
                     self.diagnostics_dropped = checked_saturating_add(1, self.diagnostics_dropped, &mut self.counter_overflowed);
                     continue;
                 }
-                self.diagnostics.push(clone_cache_value(diagnostic));
+                self.diagnostics.push(diagnostic.clone());
             }
         }
         if !self.diagnostics.is_empty() {
@@ -375,7 +367,7 @@ impl UnknownTierStats {
         if !self.fits_add(other) {
             return None;
         }
-        let mut merged = clone_cache_value(self);
+        let mut merged = self.clone();
         merged.merge(other);
         Some(merged)
     }
@@ -415,7 +407,7 @@ impl AllTierStats {
             if st.is_empty() {
                 continue;
             }
-            let entry = self.tiers.entry(clone_cache_value(tier)).or_default();
+            let entry = self.tiers.entry(tier.clone()).or_default();
             *entry = entry.add(st);
         }
     }
@@ -445,7 +437,7 @@ impl AllTierStats {
             .tiers
             .keys()
             .filter(|tier| tier.as_str() != "STANDARD" && tier.as_str() != "REDUCED_REDUNDANCY" && !known.contains(tier.as_str()))
-            .map(clone_cache_value)
+            .cloned()
             .collect();
         for tier in retired_tiers {
             if let Some(stats) = self.tiers.remove(&tier) {
@@ -1245,10 +1237,10 @@ impl DataUsageEntry {
     }
 
     pub fn add_child(&mut self, hash: &DataUsageHash) {
-        if self.children.contains(&clone_cache_value(&hash.0)) {
+        if self.children.contains(&hash.key()) {
             return;
         }
-        self.children.insert(clone_cache_value(&hash.0));
+        self.children.insert(hash.key());
     }
 
     pub fn merge(&mut self, other: &DataUsageEntry) {
@@ -1265,7 +1257,7 @@ impl DataUsageEntry {
             s_rep.replica_size += o_rep.replica_size;
             s_rep.replica_count += o_rep.replica_count;
             for (arn, stat) in o_rep.targets.iter() {
-                let st = s_rep.targets.entry(clone_cache_value(arn)).or_default();
+                let st = s_rep.targets.entry(arn.clone()).or_default();
                 st.pending_size += stat.pending_size;
                 st.replicated_size += stat.replicated_size;
                 st.failed_size += stat.failed_size;
@@ -1459,7 +1451,7 @@ pub fn prefix_usage_in_cache(
     } else {
         format!("{bucket}/{prefix}")
     };
-    let entry = clone_cache_value(cache.get(&clone_cache_value(&hash_path(&root).0))?);
+    let entry = cache.get(&hash_path(&root).key())?.clone();
 
     let usage = PrefixUsageSummary::from_entry(&flatten_entry(cache, &entry, 0)?);
 
@@ -1602,38 +1594,38 @@ pub struct DataUsageCache {
 impl DataUsageCache {
     pub fn replace(&mut self, path: &str, parent: &str, e: DataUsageEntry) {
         let hash = hash_path(path);
-        self.cache.insert(clone_cache_value(&hash.0), e);
+        self.cache.insert(hash.key(), e);
         if !parent.is_empty() {
             let phash = hash_path(parent);
             let p = {
-                let p = self.cache.entry(clone_cache_value(&phash.0)).or_default();
+                let p = self.cache.entry(phash.key()).or_default();
                 p.add_child(&hash);
-                clone_cache_value(p)
+                p.clone()
             };
-            self.cache.insert(clone_cache_value(&phash.0), p);
+            self.cache.insert(phash.key(), p);
         }
     }
 
     pub fn replace_hashed(&mut self, hash: &DataUsageHash, parent: &Option<DataUsageHash>, e: &DataUsageEntry) {
-        self.cache.insert(clone_cache_value(&hash.0), clone_cache_value(e));
+        self.cache.insert(hash.key(), e.clone());
         if let Some(parent) = parent {
-            self.cache.entry(clone_cache_value(&parent.0)).or_default().add_child(hash);
+            self.cache.entry(parent.key()).or_default().add_child(hash);
         }
     }
 
     pub fn find(&self, path: &str) -> Option<DataUsageEntry> {
-        self.cache.get(&clone_cache_value(&hash_path(path).0)).map(clone_cache_value)
+        self.cache.get(&hash_path(path).key()).cloned()
     }
 
     pub fn find_children_copy(&mut self, h: DataUsageHash) -> DataUsageHashMap {
-        clone_cache_value(&self.cache.entry(clone_cache_value(&h.0)).or_default().children)
+        self.cache.entry(h.string()).or_default().children.clone()
     }
 
     pub fn flatten(&self, root: &DataUsageEntry) -> DataUsageEntry {
-        let mut root = clone_cache_value(root);
-        for id in clone_cache_value(&root.children).iter() {
+        let mut root = root.clone();
+        for id in root.children.clone().iter() {
             if let Some(e) = self.cache.get(id) {
-                let mut e = clone_cache_value(e);
+                let mut e = e.clone();
                 if !e.children.is_empty() {
                     e = self.flatten(&e);
                 }
@@ -1645,16 +1637,16 @@ impl DataUsageCache {
     }
 
     pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option<DataUsageHash>) {
-        if let Some(e) = src.cache.get(&clone_cache_value(&hash.0)) {
-            self.cache.insert(clone_cache_value(&hash.0), clone_cache_value(e));
+        if let Some(e) = src.cache.get(&hash.string()) {
+            self.cache.insert(hash.key(), e.clone());
             for ch in e.children.iter() {
-                if *ch == clone_cache_value(&hash.0) {
+                if *ch == hash.key() {
                     return;
                 }
-                self.copy_with_children(src, &DataUsageHash(clone_cache_value(ch)), &Some(clone_cache_value(hash)));
+                self.copy_with_children(src, &DataUsageHash(ch.to_string()), &Some(hash.clone()));
             }
             if let Some(parent) = parent {
-                let p = self.cache.entry(clone_cache_value(&parent.0)).or_default();
+                let p = self.cache.entry(parent.key()).or_default();
                 p.add_child(hash);
             }
         }
@@ -1662,14 +1654,14 @@ impl DataUsageCache {
 
     pub fn delete_recursive(&mut self, hash: &DataUsageHash) {
         let mut need_remove = Vec::new();
-        if let Some(v) = self.cache.get(&clone_cache_value(&hash.0)) {
+        if let Some(v) = self.cache.get(&hash.string()) {
             for child in v.children.iter() {
-                need_remove.push(clone_cache_value(child));
+                need_remove.push(child.clone());
             }
         }
-        self.cache.remove(&clone_cache_value(&hash.0));
+        self.cache.remove(&hash.string());
         need_remove.iter().for_each(|child| {
-            self.delete_recursive(&DataUsageHash(clone_cache_value(child)));
+            self.delete_recursive(&DataUsageHash(child.to_string()));
         });
     }
 
@@ -1690,7 +1682,7 @@ impl DataUsageCache {
     }
 
     pub fn search_parent(&self, hash: &DataUsageHash) -> Option<DataUsageHash> {
-        let want = clone_cache_value(&hash.0);
+        let want = hash.key();
         if let Some(last_index) = want.rfind('/')
             && let Some(v) = self.find(&want[0..last_index])
             && v.children.contains(&want)
@@ -1701,7 +1693,7 @@ impl DataUsageCache {
 
         for (k, v) in self.cache.iter() {
             if v.children.contains(&want) {
-                let found = DataUsageHash(clone_cache_value(k));
+                let found = DataUsageHash(k.clone());
                 return Some(found);
             }
         }
@@ -1709,7 +1701,7 @@ impl DataUsageCache {
     }
 
     pub fn is_compacted(&self, hash: &DataUsageHash) -> bool {
-        match self.cache.get(&clone_cache_value(&hash.0)) {
+        match self.cache.get(&hash.key()) {
             Some(due) => due.compacted,
             None => false,
         }
@@ -1734,7 +1726,7 @@ impl DataUsageCache {
         if self.cache.len() < limit {
             return;
         }
-        let top = clone_cache_value(&hash_path(&self.info.name).0);
+        let top = hash_path(&self.info.name).key();
         let top_e = match self.find(&top) {
             Some(e) => e,
             None => return,
@@ -1761,7 +1753,7 @@ impl DataUsageCache {
     }
 
     pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) {
-        let e = match self.cache.get(&clone_cache_value(&path.0)) {
+        let e = match self.cache.get(&path.key()) {
             Some(e) => e,
             None => return,
         };
@@ -1771,13 +1763,13 @@ impl DataUsageCache {
         }
 
         if e.children.len() > limit && compact_self {
-            let mut flat = self.size_recursive(&clone_cache_value(&path.0)).unwrap_or_default();
+            let mut flat = self.size_recursive(&path.key()).unwrap_or_default();
             flat.compacted = true;
             self.delete_recursive(path);
             self.replace_hashed(path, &None, &flat);
             return;
         }
-        let total = self.total_children_rec(&clone_cache_value(&path.0));
+        let total = self.total_children_rec(&path.key());
         if total < limit {
             return;
         }
@@ -1790,12 +1782,12 @@ impl DataUsageCache {
         let mut candidate_index = 0;
         while remove > 0 && candidate_index < candidates.len() {
             let e = &candidates[candidate_index];
-            let candidate = clone_cache_value(&e.path);
+            let candidate = e.path.clone();
             if candidate == *path && !compact_self {
                 break;
             }
-            let removing = self.total_children_rec(&clone_cache_value(&candidate.0));
-            let mut flat = match self.size_recursive(&clone_cache_value(&candidate.0)) {
+            let removing = self.total_children_rec(&candidate.key());
+            let mut flat = match self.size_recursive(&candidate.key()) {
                 Some(flat) => flat,
                 None => {
                     candidate_index += 1;
@@ -1832,7 +1824,7 @@ impl DataUsageCache {
             if o.root().is_none() {
                 return;
             }
-            *self = clone_cache_value(o);
+            *self = o.clone();
             return;
         };
 
@@ -1845,8 +1837,7 @@ impl DataUsageCache {
         }
 
         existing_root.merge(&other_root);
-        self.cache
-            .insert(clone_cache_value(&hash_path(&self.info.name).0), existing_root);
+        self.cache.insert(hash_path(&self.info.name).key(), existing_root);
 
         let root_hash = self.root_hash();
         for key in other_root.children.iter() {
@@ -1857,7 +1848,7 @@ impl DataUsageCache {
             if let Some(existing) = self.cache.get_mut(key) {
                 existing.merge(&flat);
             } else {
-                self.replace_hashed(&DataUsageHash(clone_cache_value(key)), &Some(clone_cache_value(&root_hash)), &flat);
+                self.replace_hashed(&DataUsageHash(key.clone()), &Some(root_hash.clone()), &flat);
             }
         }
     }
@@ -1901,7 +1892,7 @@ impl DataUsageCache {
 
                 for (arn, stat) in rs.targets.iter() {
                     bui.replication_info.insert(
-                        clone_cache_value(arn),
+                        arn.clone(),
                         BucketTargetUsageInfo {
                             replication_pending_size: stat.pending_size,
                             replicated_size: stat.replicated_size,
@@ -1914,7 +1905,7 @@ impl DataUsageCache {
                     );
                 }
             }
-            buckets_usage.insert(clone_cache_value(bucket_name), bui);
+            buckets_usage.insert(bucket_name.clone(), bui);
         }
 
         DataUsageInfo {
@@ -1946,20 +1937,20 @@ struct Inner {
 }
 
 fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, candidates: &mut Vec<Inner>) -> usize {
-    let e = match data_usage_cache.cache.get(&clone_cache_value(&path.0)) {
+    let e = match data_usage_cache.cache.get(&path.key()) {
         Some(e) => e,
         None => return 0,
     };
     let mut objects = e.objects;
     for ch in e.children.iter() {
-        objects += add(data_usage_cache, &DataUsageHash(clone_cache_value(ch)), candidates);
+        objects += add(data_usage_cache, &DataUsageHash(ch.clone()), candidates);
     }
     // Collect internal nodes (with children) as compaction candidates.
     // Leaf nodes have no children to remove, so compacting them is a no-op.
     if !e.children.is_empty() {
         candidates.push(Inner {
             objects,
-            path: clone_cache_value(path),
+            path: path.clone(),
         });
     }
     objects
@@ -1967,7 +1958,7 @@ fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, candidates: &mut
 
 fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet<String>) {
     for k in entry.children.iter() {
-        found.insert(clone_cache_value(k));
+        found.insert(k.to_string());
         if let Some(ch) = duc.cache.get(k) {
             mark(duc, ch, found);
         }
@@ -2417,11 +2408,11 @@ impl SizeSummary {
         }
 
         for entry in &other.size_reconciliation {
-            self.record_size_reconciliation(clone_cache_value(entry));
+            self.record_size_reconciliation(entry.clone());
         }
         self.size_reconciliation_truncated |= other.size_reconciliation_truncated;
         for scope in &other.reconciliation_scopes {
-            self.record_reconciliation_scope_with(&scope.bucket, &scope.object, || clone_cache_value(scope));
+            self.record_reconciliation_scope(&scope.bucket, &scope.object);
         }
     }
 
@@ -2446,13 +2437,6 @@ impl SizeSummary {
     /// Mark one object scope as refreshed. Duplicate scopes are suppressed so
     /// merging summaries remains bounded and deterministic.
     pub fn record_reconciliation_scope(&mut self, bucket: &str, object: &str) {
-        self.record_reconciliation_scope_with(bucket, object, || SizeReconciliationScope {
-            bucket: bucket.to_string(),
-            object: object.to_string(),
-        });
-    }
-
-    fn record_reconciliation_scope_with(&mut self, bucket: &str, object: &str, build: impl FnOnce() -> SizeReconciliationScope) {
         if !self
             .reconciliation_scopes
             .iter()
@@ -2462,7 +2446,10 @@ impl SizeSummary {
                 self.size_reconciliation_truncated = true;
                 return;
             }
-            self.reconciliation_scopes.push(build());
+            self.reconciliation_scopes.push(SizeReconciliationScope {
+                bucket: bucket.to_string(),
+                object: object.to_string(),
+            });
         }
     }
 }
@@ -3417,42 +3404,6 @@ mod tests {
         assert_eq!(summary1.versions, 15);
     }
 
-    #[test]
-    fn test_reconciliation_scope_builds_only_after_admission() {
-        let mut summary = SizeSummary::new();
-        let builds = std::cell::Cell::new(0);
-        let build = || {
-            builds.set(builds.get() + 1);
-            SizeReconciliationScope {
-                bucket: "bucket".to_string(),
-                object: "object".to_string(),
-            }
-        };
-
-        summary.record_reconciliation_scope_with("bucket", "object", build);
-        assert_eq!(builds.get(), 1, "an admitted scope is copied once");
-        assert_eq!(summary.reconciliation_scopes.len(), 1);
-        assert_eq!(summary.reconciliation_scopes[0].bucket, "bucket");
-        assert_eq!(summary.reconciliation_scopes[0].object, "object");
-
-        summary.record_reconciliation_scope_with("bucket", "object", build);
-        assert_eq!(builds.get(), 1, "duplicate scopes must not be copied");
-        assert_eq!(summary.reconciliation_scopes.len(), 1);
-        assert!(!summary.size_reconciliation_truncated);
-
-        for index in 1..1024 {
-            summary.record_reconciliation_scope("bucket", &index.to_string());
-        }
-        summary.record_reconciliation_scope_with("bucket", "object", build);
-        assert_eq!(builds.get(), 1, "a duplicate at capacity must not be copied");
-        assert!(!summary.size_reconciliation_truncated);
-
-        summary.record_reconciliation_scope_with("bucket", "overflow", build);
-        assert_eq!(builds.get(), 1, "a rejected scope must not be copied");
-        assert_eq!(summary.reconciliation_scopes.len(), 1024);
-        assert!(summary.size_reconciliation_truncated);
-    }
-
     #[test]
     fn test_size_histogram_compat_rollup_sums_all_sub_buckets() {
         let mut hist = SizeHistogram::default();

crates/e2e_test/Cargo.toml

@@ -25,7 +25,6 @@ workspace = true
 
 [features]
 default = []
-scanner-peer-mixed-version = []
 hotpath = [
     "hotpath/hotpath",
     "hotpath/tokio",

crates/e2e_test/README.md

@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
 | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
 | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
-| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
+| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
 
@@ -64,25 +64,6 @@ Each build/run holds an exclusive `rustfs.e2e.lock` marker beside the binary; co
 
 The protocols suite has its own fixed-port and single-worker contract in [`src/protocols/README.md`](src/protocols/README.md). Use its command under [Troubleshooting](#troubleshooting).
 
-### Scanner mixed-version gate
-
-The Linux N/O scanner gate needs a separately qualified `33482bb` server, which supports scoped ACKs. Set `RUSTFS_SCANNER_COMPAT_OLD_BINARY` to that executable and `RUSTFS_SCANNER_COMPAT_OLD_SHA256` to its independently verified digest. The release binary used by ordinary upgrade CI is a different baseline. Missing or mismatched inputs fail the test.
-
-Enable the harness-only `scanner-peer-mixed-version` feature explicitly; keep the server wrapper features identical between build and run:
-
-```bash
-python3 scripts/e2e_binary.py build --features e2e-test-hooks
-python3 scripts/e2e_binary.py run --features e2e-test-hooks -- \
-  cargo nextest run --profile e2e-distributed -p e2e_test \
-  --features scanner-peer-mixed-version --ignore-default-filter \
-  -E 'test(=distributed::scanner_peer_bucket_test::scanner_peer_bucket_new_coordinator_certifies_old_peers_and_scopes_next_cycle)' \
-  --test-threads 1 --retries 0 --no-tests=fail
-```
-
-The separate `scanner_peer_bucket_new_coordinator_certifies_old_worker_peers_and_scopes_next_cycle` case uses the same feature and qualified old binary inputs. Select its exact name in the command above. It keeps the original N process enabled and starts the three official O replacements with `RUSTFS_SCANNER_ENABLED=false`: their autonomous scanner loops are disabled, while real S3 dirty tracking, authenticated snapshots, publication leases, scoped ACKs, and N-dispatched RemoteScanner workers remain active. It records each node's actual scanner status and retains the same identity, durability, root, ACK, data-conservation, and physical-walk requirements. This fixed-role case does not validate autonomous O scanning or all-enabled election behavior. The original all-enabled N/O test remains separate and has not passed acceptance; a fixed-role result cannot satisfy that gate.
-
-For this fixed-role case, the `e2e-test-hooks` server also records the N client's complete bounded worker inventory. Requests are observed before opening the normal stream and paired only after authenticated source/plan validation and terminal EOF. Closed manifests and final health receipts must bind both cycles to their actual candidate/root; the next cycle needs exactly four sets of hot-only execution and at least one positive-progress worker mapped to a measured official O process. A zero-progress Current response, missing/extra request, unhealthy recorder, or missing physical coverage fails the gate. The recorder does not add O hooks or change any protocol or timeout. Final health reads check immutable closed-cycle evidence, not current N liveness.
-
 ### `#[ignore]` semantics
 
 Ignored tests are excluded from the default `cargo nextest run` pass because
@@ -194,9 +175,9 @@ the same profile for membership and execution with one nightly worker.
 | `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
 | ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
-| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
+| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
-| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
+| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
 | Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |

crates/e2e_test/src/anonymous_access_test.rs

@@ -17,7 +17,8 @@
 //! when PublicAccessBlock configuration is missing or explicitly set.
 
 use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
-use aws_sdk_s3::types::PublicAccessBlockConfiguration;
+use crate::sts_query_compat_test::{OPA_AUTH_TOKEN, OpaMock};
+use aws_sdk_s3::types::{PublicAccessBlockConfiguration, Tag, Tagging};
 use tracing::info;
 
 async fn setup_public_bucket(
@@ -69,6 +70,98 @@ async fn anonymous_get_object(
     local_http_client().get(&url).send().await
 }
 
+#[tokio::test]
+async fn test_plain_http_matches_secure_transport_false_bucket_policy() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    assert!(env.url.starts_with("http://"), "fixture must exercise plaintext HTTP: {}", env.url);
+
+    let bucket = "secure-transport-policy";
+    let key = "plain-http.txt";
+    let admin = env.create_s3_client();
+    admin.create_bucket().bucket(bucket).send().await?;
+    admin
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(aws_sdk_s3::primitives::ByteStream::from_static(b"plain http"))
+        .send()
+        .await?;
+    admin
+        .put_bucket_policy()
+        .bucket(bucket)
+        .policy(
+            serde_json::json!({
+                "Version": "2012-10-17",
+                "Statement": [{
+                    "Effect": "Allow",
+                    "Principal": "*",
+                    "Action": "s3:GetObject",
+                    "Resource": format!("arn:aws:s3:::{bucket}/*"),
+                    "Condition": {"Bool": {"aws:SecureTransport": "false"}}
+                }]
+            })
+            .to_string(),
+        )
+        .send()
+        .await?;
+
+    let response = anonymous_get_object(&env, bucket, key).await?;
+    assert_eq!(response.status().as_u16(), 200);
+    assert_eq!(response.bytes().await?.as_ref(), b"plain http");
+    Ok(())
+}
+
+/// Root credentials have no attached IAM policy, so these requests depend on
+/// the owner flag reaching policy evaluation unchanged.
+#[tokio::test]
+async fn test_root_requests_keep_policy_owner_flag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+
+    let opa = OpaMock::start().await?;
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server_with_env(
+        vec![],
+        &[
+            ("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
+            ("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
+        ],
+    )
+    .await?;
+
+    let bucket = "root-owner-flag";
+    let key = "owner.txt";
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(aws_sdk_s3::primitives::ByteStream::from_static(b"owner access"))
+        .send()
+        .await?;
+
+    // Duplicate stored tags are ambiguous. An owner request must not enter the
+    // OPA tag-loading path, which rejects this metadata before evaluation.
+    client
+        .put_bucket_tagging()
+        .bucket(bucket)
+        .tagging(
+            Tagging::builder()
+                .tag_set(Tag::builder().key("department").value("finance").build()?)
+                .tag_set(Tag::builder().key("department").value("engineering").build()?)
+                .build()?,
+        )
+        .send()
+        .await?;
+    let object = client.get_object().bucket(bucket).key(key).send().await?;
+
+    assert_eq!(object.body.collect().await?.into_bytes().as_ref(), b"owner access");
+    Ok(())
+}
+
 /// Issue #2036: Anonymous GetObject should succeed when bucket policy allows it
 /// and no PublicAccessBlock configuration exists (ConfigNotFound).
 #[tokio::test]

crates/e2e_test/src/delete_marker_migration_semantics_test.rs

@@ -16,193 +16,8 @@
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
     use aws_sdk_s3::Client;
-    use aws_sdk_s3::config::http::HttpResponse;
-    use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
     use aws_sdk_s3::primitives::ByteStream;
-    use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
-
-    fn assert_not_found<E: ProvideErrorMetadata + std::fmt::Debug>(error: &SdkError<E, HttpResponse>) {
-        assert_eq!(error.raw_response().map(|r| r.status().as_u16()), Some(404), "{error:?}");
-        assert_eq!(
-            error.as_service_error().and_then(ProvideErrorMetadata::code),
-            Some("NoSuchKey"),
-            "{error:?}"
-        );
-    }
-
-    #[tokio::test]
-    async fn conditional_writes_cannot_replace_delete_marker_with_if_match() {
-        init_logging();
-        let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
-        env.start_rustfs_server(vec![]).await.expect("start RustFS");
-        let client = env.create_s3_client();
-        let bucket = "conditional-delete-marker-proof";
-        create_versioned_bucket(&client, bucket).await;
-        client
-            .put_object()
-            .bucket(bucket)
-            .key("source")
-            .body(ByteStream::from_static(b"source"))
-            .send()
-            .await
-            .expect("write copy source");
-
-        for operation in ["put", "copy", "multipart"] {
-            let old = client
-                .put_object()
-                .bucket(bucket)
-                .key(operation)
-                .body(ByteStream::from_static(b"old"))
-                .send()
-                .await
-                .expect("write historical version");
-            let old_etag = old.e_tag().expect("historical ETag");
-            let marker = client
-                .delete_object()
-                .bucket(bucket)
-                .key(operation)
-                .send()
-                .await
-                .expect("create delete marker");
-            let marker_id = marker.version_id().expect("delete marker version");
-
-            let (upload_id, completed) = if operation == "multipart" {
-                let upload = client
-                    .create_multipart_upload()
-                    .bucket(bucket)
-                    .key(operation)
-                    .send()
-                    .await
-                    .expect("initiate multipart upload");
-                let upload_id = upload.upload_id().expect("upload ID").to_owned();
-                let part = client
-                    .upload_part()
-                    .bucket(bucket)
-                    .key(operation)
-                    .upload_id(&upload_id)
-                    .part_number(1)
-                    .body(ByteStream::from_static(b"new"))
-                    .send()
-                    .await
-                    .expect("upload part");
-                let completed = CompletedMultipartUpload::builder()
-                    .parts(
-                        CompletedPart::builder()
-                            .part_number(1)
-                            .set_e_tag(part.e_tag().map(str::to_owned))
-                            .build(),
-                    )
-                    .build();
-                (Some(upload_id), Some(completed))
-            } else {
-                (None, None)
-            };
-
-            for etag in [old_etag, "\"00000000000000000000000000000000\""] {
-                match operation {
-                    "put" => {
-                        let err = client
-                            .put_object()
-                            .bucket(bucket)
-                            .key(operation)
-                            .if_match(etag)
-                            .body(ByteStream::from_static(b"new"))
-                            .send()
-                            .await
-                            .expect_err("If-Match must reject marker");
-                        assert_not_found(&err);
-                    }
-                    "copy" => {
-                        let err = client
-                            .copy_object()
-                            .bucket(bucket)
-                            .key(operation)
-                            .copy_source(format!("{bucket}/source"))
-                            .if_match(etag)
-                            .send()
-                            .await
-                            .expect_err("If-Match copy must reject marker");
-                        assert_not_found(&err);
-                    }
-                    _ => {
-                        let err = client
-                            .complete_multipart_upload()
-                            .bucket(bucket)
-                            .key(operation)
-                            .upload_id(upload_id.as_deref().expect("upload ID"))
-                            .multipart_upload(completed.clone().expect("completed parts"))
-                            .if_match(etag)
-                            .send()
-                            .await
-                            .expect_err("If-Match completion must reject marker");
-                        assert_not_found(&err);
-                    }
-                }
-                let versions = client
-                    .list_object_versions()
-                    .bucket(bucket)
-                    .prefix(operation)
-                    .send()
-                    .await
-                    .expect("list versions after rejected write");
-                assert_eq!(versions.versions().len(), 1, "{operation}: failed write created a version");
-                assert_eq!(versions.delete_markers().len(), 1);
-                assert_eq!(versions.delete_markers()[0].version_id(), Some(marker_id));
-                assert_eq!(versions.delete_markers()[0].is_latest(), Some(true));
-            }
-
-            match operation {
-                "put" => {
-                    client
-                        .put_object()
-                        .bucket(bucket)
-                        .key(operation)
-                        .if_none_match("*")
-                        .body(ByteStream::from_static(b"new"))
-                        .send()
-                        .await
-                        .expect("If-None-Match PUT over marker");
-                }
-                "copy" => {
-                    client
-                        .copy_object()
-                        .bucket(bucket)
-                        .key(operation)
-                        .copy_source(format!("{bucket}/source"))
-                        .if_none_match("*")
-                        .send()
-                        .await
-                        .expect("If-None-Match copy over marker");
-                }
-                _ => {
-                    client
-                        .complete_multipart_upload()
-                        .bucket(bucket)
-                        .key(operation)
-                        .upload_id(upload_id.as_deref().expect("upload ID"))
-                        .multipart_upload(completed.expect("completed parts"))
-                        .if_none_match("*")
-                        .send()
-                        .await
-                        .expect("If-None-Match completion over marker");
-                }
-            }
-            let current = client
-                .get_object()
-                .bucket(bucket)
-                .key(operation)
-                .send()
-                .await
-                .expect("read successful conditional write")
-                .body
-                .collect()
-                .await
-                .expect("read object body")
-                .into_bytes();
-            let expected: &[u8] = if operation == "copy" { b"source" } else { b"new" };
-            assert_eq!(current.as_ref(), expected, "{operation}: unexpected current object");
-        }
-    }
+    use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
 
     async fn create_versioned_bucket(client: &Client, bucket: &str) {
         client

crates/e2e_test/src/distributed/concurrency_stability_test.rs

@@ -12,249 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
+use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
 use crate::common::init_logging;
-use aws_sdk_s3::{
-    Client,
-    error::{ProvideErrorMetadata, SdkError},
-    primitives::ByteStream,
-};
-use std::any::TypeId;
 use std::collections::BTreeSet;
-use std::fmt::Debug;
-use std::future::Future;
 use std::sync::Arc;
-use std::time::Duration;
 use tokio::sync::Barrier;
-use tokio::time::Instant;
-
-const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
-
-async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
-where
-    E: Debug + ProvideErrorMetadata + 'static,
-    F: FnMut() -> Fut,
-    Fut: Future<Output = Result<T, SdkError<E>>>,
-{
-    let started = Instant::now();
-    let mut attempts = 0;
-    // The workload uses unique unversioned keys. Repeat only explicitly
-    // advertised lock backpressure; every step still needs its real result.
-    let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
-        loop {
-            attempts += 1;
-            let result = call().await;
-            let retry = result.as_ref().err().is_some_and(|error| {
-                let Some(response) = error.raw_response() else {
-                    return false;
-                };
-                let Some(service_error) = error.as_service_error() else {
-                    return false;
-                };
-                // HEAD has no XML error body, so its SDK service error can
-                // lack a code even when the server supplies retry headers.
-                let namespace_code = service_error.code() == Some("ServiceUnavailable")
-                    || (service_error.code().is_none()
-                        && TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
-                response.status().as_u16() == 503
-                    && namespace_code
-                    && response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
-                    && response.headers().get("retry-after") == Some("1")
-            });
-            if !retry {
-                return result;
-            }
-            tokio::time::sleep(Duration::from_secs(1)).await;
-        }
-    })
-    .await;
-    if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
-        eprintln!(
-            "mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
-            started.elapsed()
-        );
-    }
-    outcome.map_err(|_| {
-        format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
-    })
-}
-
-async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
-    let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
-    let actual = output.body.collect().await?.into_bytes();
-    if actual.as_ref() != expected {
-        return Err(format!(
-            "object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
-            expected.len(),
-            sha256_hex(expected),
-            actual.len(),
-            sha256_hex(&actual)
-        )
-        .into());
-    }
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use aws_smithy_http_client::test_util::infallible_client_fn;
-    use aws_smithy_types::body::SdkBody;
-    use std::sync::atomic::{AtomicUsize, Ordering};
-
-    type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
-
-    fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
-        let requests = Arc::new(AtomicUsize::new(0));
-        let seen = Arc::clone(&requests);
-        let transport = infallible_client_fn(move |_| {
-            let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
-            let body = if code.is_empty() {
-                body.to_string()
-            } else {
-                format!("<Error><Code>{code}</Code></Error>")
-            };
-            let mut response = http::Response::builder()
-                .status(status)
-                .header("content-type", "application/xml")
-                .header("content-length", body.len());
-            if let Some(cause) = cause {
-                response = response.header("x-rustfs-error-cause", cause);
-            }
-            if let Some(retry_after) = retry_after {
-                response = response.header("retry-after", retry_after);
-            }
-            response.body(SdkBody::from(body)).expect("scripted S3 response")
-        });
-        let config = crate::common::build_test_s3_config(
-            "http://contention.example.invalid",
-            "synthetic-access",
-            "synthetic-secret",
-            None,
-            "contention-test",
-        );
-        let client = Client::from_conf(config.to_builder().http_client(transport).build());
-        (s3_probe_client(&client), requests)
-    }
-
-    fn contention() -> Reply {
-        (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
-        let (client, requests) = client(|attempt| {
-            if attempt < 3 {
-                contention()
-            } else {
-                (200, "", None, None, "")
-            }
-        });
-        let standard = Client::from_conf(
-            client
-                .config()
-                .to_builder()
-                .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
-                .build(),
-        );
-        standard
-            .put_object()
-            .bucket("bucket")
-            .key("key")
-            .body(ByteStream::from_static(b"body"))
-            .send()
-            .await
-            .expect_err("three advertised lock responses exhaust the standard SDK policy");
-        assert_eq!(requests.load(Ordering::SeqCst), 3);
-        requests.store(0, Ordering::SeqCst);
-        let started = Instant::now();
-        after_lock_contention("PUT recovery", || {
-            client
-                .put_object()
-                .bucket("bucket")
-                .key("key")
-                .body(ByteStream::from_static(b"body"))
-                .send()
-        })
-        .await??;
-        assert_eq!(requests.load(Ordering::SeqCst), 4);
-        assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
-        Ok(())
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
-        let (client, requests) = client(|attempt| {
-            if attempt == 0 {
-                (503, "", Some("namespace_lock_timeout"), Some("1"), "")
-            } else {
-                (200, "", None, None, "")
-            }
-        });
-        after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
-        assert_eq!(requests.load(Ordering::SeqCst), 2);
-        Ok(())
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
-        for reply in [
-            (500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
-            (403, "AccessDenied", None, None, ""),
-            (404, "NoSuchKey", None, None, ""),
-            (503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
-            (503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
-            (503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
-            (503, "", Some("namespace_lock_timeout"), Some("1"), ""),
-            (503, "ServiceUnavailable", None, Some("1"), ""),
-            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
-            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
-        ] {
-            let (client, requests) = client(move |_| reply);
-            let result = after_lock_contention("PUT fatal response", || {
-                client
-                    .put_object()
-                    .bucket("bucket")
-                    .key("key")
-                    .body(ByteStream::from_static(b"body"))
-                    .send()
-            })
-            .await?;
-            assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
-            assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
-        }
-        Ok(())
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn persistent_lock_contention_exhausts_the_request_budget() {
-        let (client, requests) = client(|_| contention());
-        let started = Instant::now();
-        let result = after_lock_contention("PUT unavailable", || {
-            client
-                .put_object()
-                .bucket("bucket")
-                .key("key")
-                .body(ByteStream::from_static(b"body"))
-                .send()
-        })
-        .await;
-        assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
-        assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
-        assert!(requests.load(Ordering::SeqCst) > 1);
-        assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn acknowledged_wrong_bytes_fail_without_retry() {
-        let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
-        let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
-            .await
-            .expect_err("a successful but incorrect read is a consistency failure");
-        assert!(error.to_string().contains("bytes mismatch"));
-        assert_eq!(requests.load(Ordering::SeqCst), 1);
-    }
-}
 
 #[tokio::test]
 async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
@@ -262,9 +24,7 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     let dist = DistCluster::start(DistLayout::FourByFour).await?;
     let bucket = unique_bucket("conc");
     dist.create_bucket(&bucket).await?;
-    // This consistency probe owns its admission policy; unrelated responses
-    // must not be hidden by the SDK's general-purpose retries.
-    let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
+    let clients = Arc::new(dist.clients()?);
     let worker_count = 24;
     let rounds = 4;
     let barrier = Arc::new(Barrier::new(worker_count));
@@ -284,46 +44,31 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
                 let key = format!("source/worker-{idx:02}-round-{round}.bin");
                 let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
                 let body = payload_for(&key, 64 * 1024);
-                let context = format!("worker={idx} round={round}");
-                after_lock_contention(&format!("{context} PUT source"), || {
-                    writer
-                        .put_object()
-                        .bucket(&bucket)
-                        .key(&key)
-                        .body(ByteStream::from(body.clone()))
-                        .send()
-                })
-                .await??;
+                put_object(writer, &bucket, &key, body.clone()).await?;
 
-                let head = after_lock_contention(&format!("{context} HEAD source"), || {
-                    reader.head_object().bucket(&bucket).key(&key).send()
-                })
-                .await??;
+                let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
                 if head.content_length() != Some(body.len() as i64) {
                     return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
                 }
-                assert_workload_bytes(reader, &bucket, &key, &body).await?;
+                assert_object_bytes(reader, &bucket, &key, &body).await?;
 
-                after_lock_contention(&format!("{context} COPY retained"), || {
-                    copier
-                        .copy_object()
-                        .bucket(&bucket)
-                        .key(&copy_key)
-                        .copy_source(format!("{bucket}/{key}"))
-                        .send()
-                })
-                .await??;
-                assert_workload_bytes(writer, &bucket, &copy_key, &body).await?;
+                copier
+                    .copy_object()
+                    .bucket(&bucket)
+                    .key(&copy_key)
+                    .copy_source(format!("{bucket}/{key}"))
+                    .send()
+                    .await?;
+                assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
 
-                after_lock_contention(&format!("{context} DELETE source"), || {
-                    writer.delete_object().bucket(&bucket).key(&key).send()
-                })
-                .await??;
-                let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
-                    reader.head_object().bucket(&bucket).key(&key).send()
-                })
-                .await?
-                .expect_err("deleted source key must not remain visible");
+                writer.delete_object().bucket(&bucket).key(&key).send().await?;
+                let missing = reader
+                    .head_object()
+                    .bucket(&bucket)
+                    .key(&key)
+                    .send()
+                    .await
+                    .expect_err("deleted source key must not remain visible");
                 if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
                     return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
                 }
@@ -337,22 +82,14 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     for handle in handles {
         inventory.extend(handle.await??);
     }
-    assert_eq!(
-        inventory.len(),
-        worker_count * rounds,
-        "every worker round must retain its acknowledged copy"
-    );
 
     let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
     for (node_idx, client) in clients.iter().enumerate() {
-        let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
-            client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
-        })
-        .await??;
+        let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
         let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
         assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
         for (key, body) in &inventory {
-            assert_workload_bytes(client, &bucket, key, body)
+            assert_object_bytes(client, &bucket, key, body)
                 .await
                 .map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
         }

crates/e2e_test/src/distributed/mod.rs

@@ -39,8 +39,6 @@ mod s3_during_data_movement_test;
 mod scanner_heal_topology_fixture;
 mod scanner_heal_topology_test;
 mod scanner_heal_topology_tier_test;
-mod scanner_peer_bucket_fixture;
-mod scanner_peer_bucket_test;
 mod site_replication_test;
 mod table_catalog_test;
 mod upgrade_test;

crates/e2e_test/src/distributed/multipart_fence_acceptance_test.rs

@@ -18,7 +18,7 @@ use super::harness::{
 };
 use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
 use crate::rpc_fault_proxy::{RpcFaultProxy, RpcPause};
-use crate::storage_api::multipart_fence_acceptance::{LOCK_DIAGNOSTICS_FILTER, RUSTFS_META_MULTIPART_BUCKET};
+use crate::storage_api::multipart_fence_acceptance::RUSTFS_META_MULTIPART_BUCKET;
 use aws_sdk_s3::{
     Client,
     config::retry::RetryConfig,
@@ -38,16 +38,14 @@ const DEADLINE: Duration = Duration::from_secs(45);
 async fn fixture(
     drives_per_node: usize,
     storage_class: &str,
-    lock_diagnostics: bool,
 ) -> TestResult<(RustFSTestClusterEnvironment, Vec<RpcFaultProxy>, Vec<Client>, String)> {
-    fixture_from_binary(drives_per_node, storage_class, None, lock_diagnostics).await
+    fixture_from_binary(drives_per_node, storage_class, None).await
 }
 
 async fn fixture_from_binary(
     drives_per_node: usize,
     storage_class: &str,
     binary: Option<&std::path::Path>,
-    lock_diagnostics: bool,
 ) -> TestResult<(RustFSTestClusterEnvironment, Vec<RpcFaultProxy>, Vec<Client>, String)> {
     let mut cluster =
         RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, drives_per_node)).await?;
@@ -92,10 +90,6 @@ async fn fixture_from_binary(
     ] {
         cluster.set_env(name, value);
     }
-    if lock_diagnostics {
-        cluster.set_env("RUST_LOG", LOCK_DIAGNOSTICS_FILTER);
-        cluster.set_env("RUSTFS_OBJECT_LOCK_DIAG_ENABLE", "true");
-    }
     let mut proxies = Vec::new();
     if binary.is_none() {
         for node in 0..4 {
@@ -369,120 +363,26 @@ fn upload_path_scope(bucket: &str, key: &str, upload_id: &str) -> TestResult<Str
     ))
 }
 
-async fn with_paused_peers<T, E>(
+async fn with_paused_peers<T: Send + 'static>(
     proxies: &[RpcFaultProxy],
     methods: &[&'static str],
     limits: &[usize],
     scope: &str,
-    request: impl std::future::Future<Output = Result<T, SdkError<E>>> + Send + 'static,
-) -> TestResult<Result<T, SdkError<E>>>
-where
-    T: Send + 'static,
-    E: std::error::Error + ProvideErrorMetadata + Send + Sync + 'static,
-{
-    let deadline = tokio::time::Instant::now() + DEADLINE;
+    request: impl std::future::Future<Output = T> + Send + 'static,
+) -> TestResult<T> {
     let pauses: Vec<_> = proxies[1..=limits.len()]
         .iter()
         .zip(limits)
         .map(|(proxy, limit)| proxy.pause(methods, *limit, Some(scope)))
         .collect::<Result<_, _>>()?;
-    let mut task = tokio::spawn(request);
-    let mut joined = None;
-    let mut deadline_expired = false;
-    let mut barrier_result = {
-        let faults = wait_for_faults(&pauses, methods);
-        tokio::pin!(faults);
-        match tokio::time::timeout_at(deadline, async {
-            tokio::select! {
-                result = &mut faults => result,
-                result = &mut task => {
-                    joined = Some(result);
-                    // A ready request is not evidence that every selected fault
-                    // was reached. Poll the original barrier once for that proof.
-                    match futures::poll!(faults.as_mut()) {
-                        std::task::Poll::Ready(result) => result,
-                        std::task::Poll::Pending => Err("SDK request completed before the selected RPC fault barrier".into()),
-                    }
-                }
-            }
-        })
-        .await
-        {
-            Ok(result) => result,
-            Err(error) => {
-                deadline_expired = true;
-                Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into())
-            }
-        }
-    };
-    // Tokio's timeout polls an already-ready inner future before its timer.
-    // A late observation cannot certify the original absolute deadline.
-    deadline_expired |= tokio::time::Instant::now() >= deadline;
-    if deadline_expired {
-        barrier_result = Err(std::io::Error::new(
-            std::io::ErrorKind::TimedOut,
-            "selected RPC fault proof observed after the absolute MPU deadline",
-        )
-        .into());
-    }
-    if barrier_result.is_err() {
-        for pause in &pauses {
-            pause.discard_pending();
-        }
-    }
-    let mut client_task_aborted = false;
-    let request_result = match joined {
-        Some(result) => result,
-        None => match tokio::time::timeout_at(deadline, &mut task).await {
-            Ok(result) => result,
-            Err(error) => {
-                deadline_expired = true;
-                for pause in &pauses {
-                    pause.discard_pending();
-                }
-                barrier_result = Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into());
-                // This owns only the test's SDK request, not server/native work.
-                // Retire the client task without granting another operation budget.
-                client_task_aborted = true;
-                task.abort();
-                task.await
-            }
-        },
-    };
-    deadline_expired |= tokio::time::Instant::now() >= deadline;
-    if deadline_expired {
-        barrier_result = Err(std::io::Error::new(
-            std::io::ErrorKind::TimedOut,
-            "SDK request result observed after the absolute MPU deadline",
-        )
-        .into());
-    }
-    let hits: Vec<_> = pauses.iter().map(RpcPause::hits).collect();
+    let task = tokio::spawn(request);
+    wait_for_faults(&pauses, methods).await?;
+    let result = tokio::time::timeout(DEADLINE, task).await??;
     for pause in &pauses {
         pause.discard_pending();
     }
     drop(pauses);
-    if barrier_result.is_err() {
-        let sdk_error = request_result.as_ref().ok().and_then(|result| result.as_ref().err());
-        let response = sdk_error.and_then(SdkError::raw_response);
-        eprintln!(
-            "MPU fault barrier failed: methods={methods:?}, hits={hits:?}, client_task_aborted={client_task_aborted}, deadline_expired={deadline_expired}, sdk_code={:?}, cause={:?}, request_id={:?}",
-            sdk_error
-                .and_then(|error| error.as_service_error())
-                .and_then(|error| error.meta().code()),
-            response.and_then(|response| response.headers().get("x-rustfs-error-cause")),
-            response.and_then(|response| response.headers().get("x-amz-request-id")),
-        );
-    }
-    match (barrier_result, request_result) {
-        // Preserve the actual SDK source instead of replacing an early service
-        // failure with an unrelated fault-wait timeout. The outer Err stays red.
-        (Err(_), Ok(Err(error))) => Err(Box::new(error)),
-        (Err(_), Err(error)) if !client_task_aborted => Err(Box::new(error)),
-        (Err(error), _) => Err(error),
-        (Ok(()), Ok(result)) => Ok(result),
-        (Ok(()), Err(error)) => Err(Box::new(error)),
-    }
+    Ok(result)
 }
 
 fn quorum_outcome<T: std::fmt::Debug, E: std::fmt::Debug + ProvideErrorMetadata>(
@@ -1418,7 +1318,7 @@ async fn concurrent_capped_parts_respect_limit_on_distinct_nodes(clients: &[Clie
 
 #[tokio::test]
 async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() -> TestResult {
-    let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", true).await?;
+    let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
     list_parts_disk_loss_after_metadata_read(&cluster, &proxies, &clients, &bucket).await?;
     concurrent_capped_parts_respect_limit_on_distinct_nodes(&clients, &bucket).await?;
     phase_quorum_matrix(&cluster, &proxies, &clients, &bucket, false, 1, 3).await
@@ -1426,13 +1326,13 @@ async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() ->
 
 #[tokio::test]
 async fn lock_delay_preserves_admission_quorum_and_reports_timeout() -> TestResult {
-    let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
+    let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
     phase_quorum_matrix(&cluster, &proxies, &clients, &bucket, true, 1, 3).await
 }
 
 #[tokio::test]
 async fn sixteen_shard_multipart_metadata_requires_twelve_votes() -> TestResult {
-    let (cluster, proxies, clients, bucket) = fixture(4, "EC:4", false).await?;
+    let (cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
     let storage = cluster_admin_ok(&cluster, http::Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
     let storage: serde_json::Value = serde_json::from_str(&storage)?;
     let storage: rustfs_madmin::StorageInfo = serde_json::from_value(storage["info"].clone())?;
@@ -1446,7 +1346,7 @@ async fn sixteen_shard_multipart_metadata_requires_twelve_votes() -> TestResult
 #[tokio::test]
 async fn mixed_version_failed_complete_requires_current_coordinator_for_abort() -> TestResult {
     let previous = super::upgrade_test::source_binary()?;
-    let (mut cluster, mut proxies, clients, bucket) = fixture_from_binary(4, "EC:4", Some(&previous), false).await?;
+    let (mut cluster, mut proxies, clients, bucket) = fixture_from_binary(4, "EC:4", Some(&previous)).await?;
     assert!(
         proxies.is_empty(),
         "the genuine previous fleet boots and seeds through its normal direct transport"
@@ -1649,7 +1549,7 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
         capability_sha256: Option<[u8; 32]>,
         abort: Option<AbortRef>,
     }
-    let (mut cluster, proxies, clients, bucket) = fixture(4, "EC:4", false).await?;
+    let (mut cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
     let key = "partial-abort";
     let committed = payload_for("committed-before-abort", 4096);
     clients[0]
@@ -1865,7 +1765,7 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
 #[tokio::test]
 async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation() -> TestResult {
     for size in [0, 8 * 1024 * 1024] {
-        let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
+        let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
         let key = "late-complete";
         let upload = clients[0].create_multipart_upload().bucket(&bucket).key(key).send().await?;
         let upload_id = upload.upload_id().ok_or("missing upload ID")?;
@@ -1992,7 +1892,7 @@ async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation(
 async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
     for (method, settles) in [("PreparePartTransaction", false), ("SettlePartTransaction", true)] {
         for recreate_bucket in [false, true] {
-            let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
+            let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
             let key = "late-part-transaction";
             let previous = payload_for("committed-before-late-part", 4096);
             clients[0]
@@ -2175,7 +2075,7 @@ async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
 
 #[tokio::test]
 async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestResult {
-    let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
+    let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
     let pauses: Vec<_> = proxies[1..]
         .iter()
         .map(|proxy| proxy.pause(&["WriteMetadata"], usize::MAX, Some(&bucket)))
@@ -2198,7 +2098,7 @@ async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestR
 async fn part_resend_owner_serializes_complete_and_abort_after_client_cancellation() -> TestResult {
     use md5::{Digest as _, Md5};
 
-    let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
+    let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
     for cancel_waiter in [false, true] {
         for complete in [false, true] {
             let key = format!("resend-{cancel_waiter}-{complete}");
@@ -2686,7 +2586,7 @@ async fn resume_at_rpc<T: Send + 'static>(
 
 #[tokio::test]
 async fn each_multipart_phase_resumes_after_metadata_and_lock_delays() -> TestResult {
-    let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
+    let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
     for locks in [false, true] {
         let key = format!("phase-{locks}");
         let client = clients[0].clone();

crates/e2e_test/src/dual_stack_differential/compare.rs

@@ -0,0 +1,512 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! What one stack answered, and how two answers are compared.
+//!
+//! An [`Observation`] keeps only what a client acts on: the HTTP status, the S3 error code, the
+//! allowlisted response headers and a few typed fields decoded from the response. The error
+//! message is kept for diagnostics and never compared: message wording is judged by the wire
+//! corpus, not here. The comparator is pure so it can be tested without a server.
+
+use serde::Serialize;
+use std::collections::{BTreeMap, BTreeSet};
+use std::path::{Path, PathBuf};
+
+/// Response headers compared exactly (after [`normalized_header_value`]).
+const HEADER_ALLOWLIST: [&str; 7] = [
+    "content-type",
+    "x-amz-version-id",
+    "etag",
+    "x-amz-delete-marker",
+    "content-range",
+    "accept-ranges",
+    "x-amz-request-charged",
+];
+
+/// Every response header with this prefix is compared as well.
+const HEADER_ALLOWLIST_PREFIX: &str = "x-amz-server-side-encryption";
+
+/// Placeholder for a field one side did not send.
+const ABSENT: &str = "<absent>";
+
+/// The lowercase header name if the comparator looks at this header.
+pub(crate) fn compared_header(name: &str) -> Option<String> {
+    let name = name.to_ascii_lowercase();
+    (HEADER_ALLOWLIST.contains(&name.as_str()) || name.starts_with(HEADER_ALLOWLIST_PREFIX)).then_some(name)
+}
+
+/// A version ID is minted per process, so its value never matches across two servers; only
+/// whether it is the literal `null` (an unversioned write) carries meaning.
+fn normalized_header_value(name: &str, value: &str) -> String {
+    if name == "x-amz-version-id" && value != "null" {
+        "<version-id>".to_owned()
+    } else {
+        value.to_owned()
+    }
+}
+
+/// One request's outcome as a client sees it.
+#[derive(Clone, Debug, Serialize)]
+pub(crate) struct Observation {
+    /// The S3 operation, named as in the `s3s::S3` trait (`GetObject`).
+    pub op: &'static str,
+    /// Which request of the script this was (`normal`, `negative`, or a setup label).
+    pub case: &'static str,
+    /// HTTP status; 0 when no response arrived.
+    pub status: u16,
+    /// The S3 error code, or the SDK's error kind when there was no service error.
+    pub error_code: Option<String>,
+    /// Allowlisted headers only, lowercase names, normalized values.
+    pub headers: BTreeMap<String, String>,
+    /// Typed values decoded from the response body, never free text.
+    pub fields: BTreeMap<String, String>,
+    /// Diagnostic only; never compared.
+    #[serde(skip)]
+    pub message: Option<String>,
+}
+
+impl Observation {
+    /// Builds an observation, keeping only the allowlisted headers.
+    pub(crate) fn new<'h>(
+        op: &'static str,
+        case: &'static str,
+        status: u16,
+        error_code: Option<String>,
+        headers: impl IntoIterator<Item = (&'h str, &'h str)>,
+    ) -> Self {
+        let headers = headers
+            .into_iter()
+            .filter_map(|(name, value)| compared_header(name).map(|name| (name, value)))
+            .map(|(name, value)| {
+                let value = normalized_header_value(&name, value);
+                (name, value)
+            })
+            .collect();
+        Self {
+            op,
+            case,
+            status,
+            error_code,
+            headers,
+            fields: BTreeMap::new(),
+            message: None,
+        }
+    }
+
+    pub(crate) fn with_field(mut self, name: &str, value: impl Into<String>) -> Self {
+        self.fields.insert(name.to_owned(), value.into());
+        self
+    }
+
+    pub(crate) fn is_success(&self) -> bool {
+        (200..300).contains(&self.status)
+    }
+}
+
+/// One field on which the two stacks answered differently.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct Difference {
+    pub op: String,
+    pub case: String,
+    pub field: String,
+    pub a: String,
+    pub b: String,
+}
+
+/// Every difference between two transcripts of the same script, matched by `(op, case)`.
+pub(crate) fn compare(a: &[Observation], b: &[Observation]) -> Vec<Difference> {
+    let mut differences = Vec::new();
+    let (index_a, duplicates_a) = index(a);
+    let (index_b, duplicates_b) = index(b);
+    for (op, case) in duplicates_a.union(&duplicates_b) {
+        let count = |side: &[Observation]| {
+            side.iter()
+                .filter(|observation| (observation.op, observation.case) == (*op, *case))
+                .count()
+                .to_string()
+        };
+        differences.push(difference(op, case, "duplicate_case", count(a), count(b)));
+    }
+    let keys: BTreeSet<_> = index_a.keys().chain(index_b.keys()).copied().collect();
+    for key @ (op, case) in keys {
+        match (index_a.get(&key), index_b.get(&key)) {
+            (Some(a), Some(b)) => compare_one(a, b, &mut differences),
+            (a, b) => {
+                let presence = |side: Option<&&Observation>| if side.is_some() { "present" } else { ABSENT }.to_owned();
+                differences.push(difference(op, case, "observation", presence(a), presence(b)));
+            }
+        }
+    }
+    differences
+}
+
+type Key = (&'static str, &'static str);
+
+fn index(side: &[Observation]) -> (BTreeMap<Key, &Observation>, BTreeSet<Key>) {
+    let mut index = BTreeMap::new();
+    let mut duplicates = BTreeSet::new();
+    for observation in side {
+        let key = (observation.op, observation.case);
+        if index.insert(key, observation).is_some() {
+            duplicates.insert(key);
+        }
+    }
+    (index, duplicates)
+}
+
+fn difference(op: &str, case: &str, field: &str, a: String, b: String) -> Difference {
+    Difference {
+        op: op.to_owned(),
+        case: case.to_owned(),
+        field: field.to_owned(),
+        a,
+        b,
+    }
+}
+
+fn compare_one(a: &Observation, b: &Observation, differences: &mut Vec<Difference>) {
+    let mut field = |name: String, value_a: Option<&str>, value_b: Option<&str>| {
+        if value_a != value_b {
+            let show = |value: Option<&str>| value.unwrap_or(ABSENT).to_owned();
+            differences.push(difference(a.op, a.case, &name, show(value_a), show(value_b)));
+        }
+    };
+    field(
+        "status".to_owned(),
+        Some(a.status.to_string().as_str()),
+        Some(b.status.to_string().as_str()),
+    );
+    field("error_code".to_owned(), a.error_code.as_deref(), b.error_code.as_deref());
+    let headers: BTreeSet<_> = a.headers.keys().chain(b.headers.keys()).collect();
+    for name in headers {
+        field(
+            format!("header.{name}"),
+            a.headers.get(name).map(String::as_str),
+            b.headers.get(name).map(String::as_str),
+        );
+    }
+    let fields: BTreeSet<_> = a.fields.keys().chain(b.fields.keys()).collect();
+    for name in fields {
+        field(
+            format!("body.{name}"),
+            a.fields.get(name).map(String::as_str),
+            b.fields.get(name).map(String::as_str),
+        );
+    }
+}
+
+/// Operations observed on both sides.
+pub(crate) fn compared_ops(a: &[Observation], b: &[Observation]) -> BTreeSet<&'static str> {
+    let ops_b: BTreeSet<_> = b.iter().map(|observation| observation.op).collect();
+    a.iter()
+        .map(|observation| observation.op)
+        .filter(|op| ops_b.contains(op))
+        .collect()
+}
+
+/// `(op, case)` pairs observed on both sides.
+pub(crate) fn compared_cases(a: &[Observation], b: &[Observation]) -> usize {
+    let cases_b: BTreeSet<_> = b.iter().map(|observation| (observation.op, observation.case)).collect();
+    a.iter()
+        .map(|observation| (observation.op, observation.case))
+        .collect::<BTreeSet<_>>()
+        .intersection(&cases_b)
+        .count()
+}
+
+/// The JSON report written for every run.
+#[derive(Debug, Serialize)]
+pub(crate) struct Report {
+    pub schema: u32,
+    pub run: String,
+    pub stack_a: String,
+    pub stack_b: String,
+    pub region_a: Option<String>,
+    pub region_b: Option<String>,
+    pub compared_ops: usize,
+    pub compared_cases: usize,
+    pub differences: Vec<Difference>,
+}
+
+impl Report {
+    /// Writes the report as `<directory>/<run>.json` and returns its path.
+    pub(crate) fn write_to(&self, directory: &Path) -> std::io::Result<PathBuf> {
+        std::fs::create_dir_all(directory)?;
+        let path = directory.join(format!("{}.json", self.run));
+        let json = serde_json::to_vec_pretty(self).map_err(std::io::Error::other)?;
+        std::fs::write(&path, json)?;
+        Ok(path)
+    }
+}
+
+/// One line per difference, for a failing assertion.
+pub(crate) fn render(differences: &[Difference]) -> String {
+    differences
+        .iter()
+        .map(|d| format!("({}, {}, {}, a={:?}, b={:?})", d.op, d.case, d.field, d.a, d.b))
+        .collect::<Vec<_>>()
+        .join("\n")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn ok(op: &'static str) -> Observation {
+        Observation::new(op, "normal", 200, None, [("content-type", "application/xml")])
+    }
+
+    fn fields(differences: &[Difference]) -> Vec<&str> {
+        differences.iter().map(|d| d.field.as_str()).collect()
+    }
+
+    #[test]
+    fn identical_transcripts_have_no_difference() {
+        let a = vec![
+            ok("HeadBucket"),
+            ok("GetBucketLocation").with_field("location_constraint", "us-east-1"),
+        ];
+        let b = a.clone();
+        assert_eq!(compare(&a, &b), Vec::new());
+        assert_eq!(compared_ops(&a, &b).len(), 2);
+    }
+
+    #[test]
+    fn a_status_difference_is_reported() {
+        let a = vec![ok("PutBucketAcl")];
+        let b = vec![Observation::new(
+            "PutBucketAcl",
+            "normal",
+            403,
+            None,
+            [("content-type", "application/xml")],
+        )];
+        assert_eq!(
+            compare(&a, &b),
+            vec![Difference {
+                op: "PutBucketAcl".into(),
+                case: "normal".into(),
+                field: "status".into(),
+                a: "200".into(),
+                b: "403".into(),
+            }]
+        );
+    }
+
+    #[test]
+    fn an_error_code_difference_is_reported() {
+        let a = vec![Observation::new("GetObject", "negative", 404, Some("NoSuchKey".into()), [])];
+        let b = vec![Observation::new(
+            "GetObject",
+            "negative",
+            404,
+            Some("NoSuchBucket".into()),
+            [],
+        )];
+        let differences = compare(&a, &b);
+        assert_eq!(fields(&differences), ["error_code"]);
+        assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("NoSuchKey", "NoSuchBucket"));
+    }
+
+    #[test]
+    fn an_error_code_on_one_side_only_is_reported() {
+        let a = vec![Observation::new("HeadBucket", "negative", 404, Some("NotFound".into()), [])];
+        let b = vec![Observation::new("HeadBucket", "negative", 404, None, [])];
+        let differences = compare(&a, &b);
+        assert_eq!(fields(&differences), ["error_code"]);
+        assert_eq!(differences[0].b, ABSENT);
+    }
+
+    #[test]
+    fn an_allowlisted_header_difference_is_reported() {
+        let a = vec![Observation::new("HeadObject", "normal", 200, None, [("ETag", "\"aa\"")])];
+        let b = vec![Observation::new("HeadObject", "normal", 200, None, [("etag", "\"bb\"")])];
+        assert_eq!(fields(&compare(&a, &b)), ["header.etag"]);
+    }
+
+    #[test]
+    fn a_header_sent_by_one_side_only_is_reported() {
+        let a = vec![Observation::new(
+            "DeleteObject",
+            "normal",
+            204,
+            None,
+            [("x-amz-delete-marker", "true")],
+        )];
+        let b = vec![Observation::new("DeleteObject", "normal", 204, None, [])];
+        let differences = compare(&a, &b);
+        assert_eq!(fields(&differences), ["header.x-amz-delete-marker"]);
+        assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("true", ABSENT));
+    }
+
+    #[test]
+    fn every_server_side_encryption_header_is_compared() {
+        let header = "x-amz-server-side-encryption-customer-key-md5";
+        let a = vec![Observation::new("PutObject", "sse-c", 200, None, [(header, "one")])];
+        let b = vec![Observation::new("PutObject", "sse-c", 200, None, [(header, "two")])];
+        assert_eq!(fields(&compare(&a, &b)), [format!("header.{header}")]);
+    }
+
+    #[test]
+    fn headers_outside_the_allowlist_are_not_compared() {
+        let a = vec![Observation::new(
+            "ListBuckets",
+            "normal",
+            200,
+            None,
+            [
+                ("date", "Mon"),
+                ("x-amz-request-id", "1"),
+                ("server", "a"),
+                ("content-length", "10"),
+            ],
+        )];
+        let b = vec![Observation::new(
+            "ListBuckets",
+            "normal",
+            200,
+            None,
+            [
+                ("date", "Tue"),
+                ("x-amz-request-id", "2"),
+                ("server", "b"),
+                ("content-length", "11"),
+            ],
+        )];
+        assert_eq!(compare(&a, &b), Vec::new());
+        assert!(a[0].headers.is_empty(), "non-allowlisted headers were kept: {:?}", a[0].headers);
+    }
+
+    #[test]
+    fn distinct_version_ids_are_not_a_difference() {
+        let a = vec![Observation::new(
+            "PutObject",
+            "normal",
+            200,
+            None,
+            [("x-amz-version-id", "0b1c")],
+        )];
+        let b = vec![Observation::new(
+            "PutObject",
+            "normal",
+            200,
+            None,
+            [("x-amz-version-id", "9f8e")],
+        )];
+        assert_eq!(compare(&a, &b), Vec::new());
+    }
+
+    #[test]
+    fn a_null_version_id_against_a_real_one_is_reported() {
+        let a = vec![Observation::new(
+            "PutObject",
+            "normal",
+            200,
+            None,
+            [("x-amz-version-id", "null")],
+        )];
+        let b = vec![Observation::new(
+            "PutObject",
+            "normal",
+            200,
+            None,
+            [("x-amz-version-id", "9f8e")],
+        )];
+        assert_eq!(fields(&compare(&a, &b)), ["header.x-amz-version-id"]);
+    }
+
+    #[test]
+    fn a_body_field_difference_is_reported() {
+        let a = vec![ok("GetBucketLocation").with_field("location_constraint", "us-east-1")];
+        let b = vec![ok("GetBucketLocation").with_field("location_constraint", "eu-west-1")];
+        let differences = compare(&a, &b);
+        assert_eq!(fields(&differences), ["body.location_constraint"]);
+        assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("us-east-1", "eu-west-1"));
+    }
+
+    #[test]
+    fn a_body_field_on_one_side_only_is_reported() {
+        let a = vec![ok("ListObjectsV2").with_field("keys", "[]")];
+        let b = vec![ok("ListObjectsV2")];
+        assert_eq!(fields(&compare(&a, &b)), ["body.keys"]);
+    }
+
+    #[test]
+    fn error_messages_are_never_compared() {
+        let mut a = Observation::new("GetObject", "negative", 404, Some("NoSuchKey".into()), []);
+        let mut b = a.clone();
+        a.message = Some("The specified key does not exist.".into());
+        b.message = Some("Object not found".into());
+        assert_eq!(compare(&[a], &[b]), Vec::new());
+    }
+
+    #[test]
+    fn a_case_observed_on_one_side_only_is_reported() {
+        let a = vec![ok("HeadBucket"), ok("ListBuckets")];
+        let b = vec![ok("HeadBucket")];
+        let differences = compare(&a, &b);
+        assert_eq!(fields(&differences), ["observation"]);
+        assert_eq!(differences[0].op, "ListBuckets");
+        assert_eq!(compared_ops(&a, &b).into_iter().collect::<Vec<_>>(), ["HeadBucket"]);
+    }
+
+    #[test]
+    fn a_case_recorded_twice_is_reported() {
+        let a = vec![ok("HeadBucket"), ok("HeadBucket")];
+        let b = vec![ok("HeadBucket")];
+        assert_eq!(fields(&compare(&a, &b)), ["duplicate_case"]);
+    }
+
+    #[test]
+    fn compared_ops_counts_operations_not_cases() {
+        let negative = Observation::new("HeadBucket", "negative", 404, Some("NotFound".into()), []);
+        let a = vec![ok("HeadBucket"), negative, ok("ListBuckets")];
+        let b = a.clone();
+        assert_eq!(compared_ops(&a, &b).len(), 2);
+    }
+
+    #[test]
+    fn the_report_lists_each_difference_as_op_field_a_b() {
+        let directory = std::env::temp_dir().join(format!("dual-stack-report-{}", uuid::Uuid::new_v4()));
+        let report = Report {
+            schema: 1,
+            run: "unit".into(),
+            stack_a: "legacy".into(),
+            stack_b: "legacy".into(),
+            region_a: None,
+            region_b: Some("eu-west-1".into()),
+            compared_ops: 1,
+            compared_cases: 1,
+            differences: compare(
+                &[ok("GetBucketLocation").with_field("location_constraint", "us-east-1")],
+                &[ok("GetBucketLocation").with_field("location_constraint", "eu-west-1")],
+            ),
+        };
+        let path = report.write_to(&directory).expect("write the report");
+        let json: serde_json::Value = serde_json::from_slice(&std::fs::read(&path).expect("read the report")).expect("parse");
+        let _ = std::fs::remove_dir_all(&directory);
+        assert_eq!(path.file_name().and_then(|name| name.to_str()), Some("unit.json"));
+        assert_eq!(json["compared_ops"], 1);
+        assert_eq!(
+            json["differences"],
+            serde_json::json!([{
+                "op": "GetBucketLocation",
+                "case": "normal",
+                "field": "body.location_constraint",
+                "a": "us-east-1",
+                "b": "eu-west-1",
+            }])
+        );
+    }
+}

crates/e2e_test/src/dual_stack_differential/mod.rs

@@ -0,0 +1,199 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Dual-stack differential: one `rustfs` binary, two processes, one SDK script (rustfs/backlog#2740).
+//!
+//! Stack A always runs `RUSTFS_S3_STACK=legacy`. Stack B runs the value of `RUSTFS_E2E_STACK_B`
+//! (default `legacy` until the gateway stack is assembled), and `RUSTFS_E2E_STACK_B_REGION`, when
+//! set, starts B in another region so that a difference can be injected by hand. Both servers
+//! answer the script in `scenarios`, which covers every operation of `impl S3 for FS`; `compare`
+//! reports each `(op, field, a, b)` on which the answers differ, and every run writes its report
+//! to `<target>/dual-stack/<run>.json`. Two processes, not two embedded servers: bucket metadata
+//! is process-global, so two stacks in one process would read each other's buckets.
+
+mod compare;
+mod scenarios;
+
+use crate::common::{RustFSTestEnvironment, init_logging, workspace_root};
+use compare::{Difference, Observation, Report, compare, compared_cases, compared_ops, render};
+use std::path::PathBuf;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+type BoxError = Box<dyn std::error::Error + Send + Sync>;
+
+const STACK_A: &str = "legacy";
+const ENV_STACK_B: &str = "RUSTFS_E2E_STACK_B";
+const ENV_STACK_B_REGION: &str = "RUSTFS_E2E_STACK_B_REGION";
+/// Becomes `gateway` once rustfs/backlog#2734 T2.5 assembles the gateway stack.
+const DEFAULT_STACK_B: &str = "legacy";
+/// The region the negative control gives stack A explicitly, so an inherited `RUSTFS_REGION`
+/// cannot erase the injected difference. Not `us-east-1`: that region answers GetBucketLocation
+/// with an empty LocationConstraint and re-creating an owned bucket with 200, so it would not
+/// report its own name and would add CreateBucket differences to the control.
+const CONTROL_REGION_A: &str = "us-west-2";
+const CONTROL_REGION_B: &str = "eu-west-1";
+/// `normal` cases the legacy stack must answer with a 2xx. The comparison alone cannot tell a
+/// script that exercises real behavior from one in which both stacks fail the same way. Five of
+/// the 73 fail on legacy by design of this script: PutBucketReplication (no registered remote
+/// target), GetBucketReplication (so no configuration), PutBucketNotificationConfiguration (no
+/// notification target configured), RestoreObject (the object is not archived) and
+/// GetObjectTorrent (legacy answers `NoSuchKey`).
+const MIN_NORMAL_SUCCESSES: usize = 68;
+
+struct PairConfig {
+    stack_b: String,
+    region_a: Option<String>,
+    region_b: Option<String>,
+}
+
+impl PairConfig {
+    fn from_env() -> Self {
+        Self {
+            stack_b: std::env::var(ENV_STACK_B).unwrap_or_else(|_| DEFAULT_STACK_B.to_owned()),
+            region_a: None,
+            region_b: std::env::var(ENV_STACK_B_REGION).ok(),
+        }
+    }
+}
+
+/// Starts one server on `stack`, overriding its region only when asked.
+async fn start_stack(stack: &str, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
+    let mut env = RustFSTestEnvironment::new().await?;
+    let mut vars = vec![("RUSTFS_S3_STACK", stack)];
+    if let Some(region) = region {
+        vars.push(("RUSTFS_REGION", region));
+    }
+    env.start_rustfs_server_without_cleanup_with_env(&vars).await?;
+    Ok(env)
+}
+
+/// Starts stack A and stack B side by side, each with its own port and data directory.
+async fn spawn_pair(config: &PairConfig) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment), BoxError> {
+    tokio::try_join!(
+        start_stack(STACK_A, config.region_a.as_deref()),
+        start_stack(&config.stack_b, config.region_b.as_deref())
+    )
+}
+
+fn report_directory() -> PathBuf {
+    let target = std::env::var_os("CARGO_TARGET_DIR").map_or_else(|| PathBuf::from("target"), PathBuf::from);
+    let target = if target.is_absolute() {
+        target
+    } else {
+        workspace_root().join(target)
+    };
+    target.join("dual-stack")
+}
+
+struct Run {
+    report: Report,
+    path: PathBuf,
+    a: Vec<Observation>,
+}
+
+/// Runs the script against both stacks concurrently and writes the report.
+async fn run_differential(label: &str, config: &PairConfig) -> Run {
+    let (server_a, server_b) = spawn_pair(config).await.expect("start stack A and stack B");
+    let (a, b) = tokio::join!(Box::pin(scenarios::run(&server_a)), Box::pin(scenarios::run(&server_b)));
+    let millis = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map_or(0, |elapsed| elapsed.as_millis());
+    let report = Report {
+        schema: 1,
+        run: format!("{label}-{millis}-{}", std::process::id()),
+        stack_a: STACK_A.to_owned(),
+        stack_b: config.stack_b.clone(),
+        region_a: config.region_a.clone(),
+        region_b: config.region_b.clone(),
+        compared_ops: compared_ops(&a, &b).len(),
+        compared_cases: compared_cases(&a, &b),
+        differences: compare(&a, &b),
+    };
+    let path = report.write_to(&report_directory()).expect("write the dual-stack report");
+    Run { report, path, a }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn both_stacks_answer_every_operation_identically() {
+        init_logging();
+        let config = PairConfig::from_env();
+        let run = run_differential("self-comparison", &config).await;
+        let report = &run.report;
+
+        let mut observed: Vec<_> = run.a.iter().map(|observation| observation.op).collect();
+        observed.sort_unstable();
+        observed.dedup();
+        assert_eq!(
+            observed,
+            scenarios::OPERATIONS,
+            "the script must drive exactly the operations of impl S3 for FS"
+        );
+        assert_eq!(report.compared_ops, scenarios::OPERATIONS.len(), "report: {}", run.path.display());
+        assert!(
+            report.differences.is_empty(),
+            "stack {} and stack {} differ on {} of {} cases (report: {}):\n{}",
+            report.stack_a,
+            report.stack_b,
+            report.differences.len(),
+            report.compared_cases,
+            run.path.display(),
+            render(&report.differences)
+        );
+
+        let not_4xx: Vec<_> = run
+            .a
+            .iter()
+            .filter(|observation| observation.case == "negative" && !(400..500).contains(&observation.status))
+            .collect();
+        assert!(not_4xx.is_empty(), "negative cases must fail with a 4xx on {STACK_A}: {not_4xx:#?}");
+        let successes = run
+            .a
+            .iter()
+            .filter(|observation| observation.case == "normal" && observation.is_success())
+            .count();
+        assert!(
+            successes >= MIN_NORMAL_SUCCESSES,
+            "only {successes} normal cases succeeded on {STACK_A}: {:#?}",
+            run.a
+        );
+    }
+
+    #[tokio::test]
+    async fn differential_detects_an_injected_difference() {
+        init_logging();
+        let config = PairConfig {
+            stack_b: STACK_A.to_owned(),
+            region_a: Some(CONTROL_REGION_A.to_owned()),
+            region_b: Some(CONTROL_REGION_B.to_owned()),
+        };
+        let run = run_differential("injected-region", &config).await;
+        let expected = Difference {
+            op: "GetBucketLocation".to_owned(),
+            case: "normal".to_owned(),
+            field: "body.location_constraint".to_owned(),
+            a: CONTROL_REGION_A.to_owned(),
+            b: CONTROL_REGION_B.to_owned(),
+        };
+        assert!(
+            run.report.differences.contains(&expected),
+            "B started in {CONTROL_REGION_B} but GetBucketLocation was not reported (report: {}):\n{}",
+            run.path.display(),
+            render(&run.report.differences)
+        );
+    }
+}

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -1700,15 +1700,6 @@ mod tests {
         );
 
         let mut cluster = RustFSTestClusterEnvironment::with_topology(topology.cluster_topology()).await?;
-        let bucket = "heal-restart-during-rebuild";
-        let mrf_fallback_scope = if !background_enabled {
-            let path = PathBuf::from(&cluster.temp_dir).join("skip-outage-mrf-fallback");
-            std::fs::write(&path, format!("{bucket}/cluster/written-while-node-down-"))?;
-            cluster.set_env("RUSTFS_E2E_SKIP_MRF_FALLBACK_PATH", path.to_string_lossy().into_owned());
-            Some(path)
-        } else {
-            None
-        };
         cluster.set_env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true");
         cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
         if let Some(storage_class) = evidence_case.and_then(|case| case.storage_class_standard) {
@@ -1961,21 +1952,6 @@ mod tests {
             }
         };
 
-        if let Some(path) = &mrf_fallback_scope {
-            let observed = path.with_extension("observed");
-            let expected = format!("{bucket}/{outage_key}");
-            let deadline = Instant::now() + Duration::from_secs(30);
-            loop {
-                if tokio::fs::read_to_string(&observed).await.ok().as_deref() == Some(expected.as_str()) {
-                    break;
-                }
-                if Instant::now() >= deadline {
-                    return Err("outage repair did not reach the isolated best-effort fallback before target rejoin".into());
-                }
-                sleep(Duration::from_millis(50)).await;
-            }
-        }
-
         assert!(
             !outage_target_manifest_required
                 || target_set_peer_drives
@@ -2069,12 +2045,7 @@ mod tests {
             } else {
                 cluster_heal_is_idle(&recovered)
             };
-            let pool_ready = if ready && background_enabled && !outage_write_deferred_until_rejoin {
-                all_nodes_pool_metadata_readiness(&cluster, recovery_deadline).await.0
-            } else {
-                true
-            };
-            if ready && pool_ready {
+            if ready {
                 break;
             }
             if Instant::now() >= recovery_deadline {
@@ -2571,10 +2542,6 @@ mod tests {
             );
         }
 
-        if let Some(path) = &mrf_fallback_scope {
-            std::fs::remove_file(path)?;
-        }
-
         if let Some(cycle_end) = scanner_cycle_floor {
             wait_for_scanner_cycle_after(&cluster, cycle_end).await?;
         }

crates/e2e_test/src/lib.rs

@@ -31,6 +31,12 @@ pub mod chaos;
 #[cfg(test)]
 pub mod fake_s3_target;
 
+// Dual-stack differential (rustfs/backlog#2740): one binary, two processes on
+// different RUSTFS_S3_STACK values, the same SDK script for every S3 operation,
+// compared on status, error code, allowlisted headers and typed body fields.
+#[cfg(test)]
+mod dual_stack_differential;
+
 // On-demand migration (backlog#2147): shared two-server environment, admin
 // wrappers, and the harness self-test (backlog#2151). Behavior scenarios are
 // added by later ODM tasks.

crates/e2e_test/src/protocols/webdav_core.rs

@@ -32,7 +32,9 @@
 //! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-3p3x-734c-h5vx>
 
 use crate::common::rustfs_binary_path_with_features;
-use crate::common::{AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via};
+use crate::common::{
+    AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, signed_s3_request,
+};
 use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, admin_request};
 use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
 use anyhow::Result;
@@ -774,6 +776,81 @@ async fn test_webdav_core_operations_direct() -> Result<()> {
     test_webdav_core_operations().await
 }
 
+#[tokio::test]
+async fn test_webdav_content_type_directory_marker() -> Result<()> {
+    let env = ProtocolTestEnvironment::new().map_err(|e| anyhow::anyhow!("{}", e))?;
+    let admin_base_url = format!("http://{}", S3_TEST_ADDRESS);
+    let binary_path = rustfs_binary_path_with_features(Some("webdav"));
+    let mut server_process = Command::new(&binary_path)
+        .arg("--address")
+        .arg(S3_TEST_ADDRESS)
+        .env("RUSTFS_CONSOLE_ENABLE", "false")
+        .env("RUSTFS_WEBDAV_ENABLE", "true")
+        .env("RUSTFS_WEBDAV_ADDRESS", WEBDAV_ADDRESS)
+        .env("RUSTFS_WEBDAV_TLS_ENABLED", "false")
+        .arg(&env.temp_dir)
+        .spawn()?;
+
+    let result = async {
+        ProtocolTestEnvironment::wait_for_port_ready(WEBDAV_PORT, 30)
+            .await
+            .map_err(|e| anyhow::anyhow!("{}", e))?;
+
+        let client = create_client();
+        let bucket_name = "webdav-content-type-bucket";
+        let directory_name = "content-type-directory";
+        let webdav_base_url = format!("http://{}", WEBDAV_ADDRESS);
+        let mkcol_bucket = client
+            .request(
+                reqwest::Method::from_bytes(b"MKCOL").expect("MKCOL is a valid HTTP method"),
+                format!("{}/{}", webdav_base_url, bucket_name),
+            )
+            .header("Authorization", basic_auth_header())
+            .send()
+            .await?;
+        anyhow::ensure!(
+            mkcol_bucket.status().is_success() || mkcol_bucket.status().as_u16() == 201,
+            "MKCOL bucket should succeed, got: {}",
+            mkcol_bucket.status()
+        );
+
+        let mkcol_directory = client
+            .request(
+                reqwest::Method::from_bytes(b"MKCOL").expect("MKCOL is a valid HTTP method"),
+                format!("{}/{}/{}", webdav_base_url, bucket_name, directory_name),
+            )
+            .header("Authorization", basic_auth_header())
+            .send()
+            .await?;
+        anyhow::ensure!(
+            mkcol_directory.status().is_success() || mkcol_directory.status().as_u16() == 201,
+            "MKCOL directory should succeed, got: {}",
+            mkcol_directory.status()
+        );
+
+        let head_url = format!("{}/{}/{}/", admin_base_url, bucket_name, directory_name);
+        let head = signed_s3_request(http::Method::HEAD, &head_url, None, None, DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY)
+            .await
+            .map_err(|error| anyhow::anyhow!("{error}"))?;
+        anyhow::ensure!(head.status().is_success(), "HEAD directory marker should succeed, got: {}", head.status());
+        let content_type = head
+            .headers()
+            .get("content-type")
+            .and_then(|value| value.to_str().ok())
+            .unwrap_or_default();
+        anyhow::ensure!(
+            content_type == "application/x-directory",
+            "WebDAV directory marker must preserve its content type, got: {content_type}"
+        );
+        Ok(())
+    }
+    .await;
+
+    let _ = server_process.kill().await;
+    let _ = server_process.wait().await;
+    result
+}
+
 /// Extract the numeric value of the first occurrence of a DAV property in a
 /// PROPFIND multistatus body. Namespace-prefix agnostic; a self-closing
 /// (unreported) property yields None.

crates/e2e_test/src/replication_extension_test.rs

@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
-    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
-    ServerSideEncryption, VersioningConfiguration,
+    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
+    VersioningConfiguration,
 };
 use base64_simd::STANDARD as BASE64_STANDARD;
 use bytes::Bytes;
@@ -3729,8 +3729,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
     init_logging();
 
     let mut source_env = RustFSTestEnvironment::new().await?;
-    let source_log = format!("{}/source.log", source_env.temp_dir);
-    source_env.capture_log_path = Some(source_log.clone());
     let mut source_env_vars = replication_fast_env();
     source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
     source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
@@ -3793,66 +3791,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         "target did not preserve the source delete-marker version ID"
     );
 
-    let wait_for_source_writeback = || async {
-        let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
-        timeout(Duration::from_secs(30), async {
-            loop {
-                let response = signed_request(
-                    http::Method::GET,
-                    &metrics_url,
-                    &source_env.access_key,
-                    &source_env.secret_key,
-                    None,
-                    None,
-                )
-                .await?;
-                let metrics: serde_json::Value = response.error_for_status()?.json().await?;
-                let queued = metrics["queued"]["curr"]["count"]
-                    .as_f64()
-                    .ok_or("missing replication queue count")?;
-                if metrics["provider_available"] == true && queued == 0.0 {
-                    return Ok::<(), Box<dyn Error + Send + Sync>>(());
-                }
-                sleep(Duration::from_millis(50)).await;
-            }
-        })
-        .await?
-    };
-    wait_for_source_writeback().await?;
-
-    let purged = source_client
-        .delete_objects()
+    source_client
+        .delete_object()
         .bucket(source_bucket)
-        .delete(
-            Delete::builder()
-                .objects(
-                    ObjectIdentifier::builder()
-                        .key(object_key)
-                        .version_id(&purged_version_id)
-                        .build()?,
-                )
-                .objects(
-                    ObjectIdentifier::builder()
-                        .key(object_key)
-                        .version_id(delete_marker_version_id)
-                        .build()?,
-                )
-                .build()?,
-        )
+        .key(object_key)
+        .version_id(&purged_version_id)
         .send()
         .await?;
-    assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
-    assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
     assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
     let target_state = list_replication_state(&target_client, target_bucket).await?;
     assert!(
         target_state.iter().all(|entry| entry.version_id != purged_version_id),
         "target retained the explicitly purged object version"
     );
-    assert!(
-        target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
-        "target retained the explicitly purged delete marker"
-    );
     assert!(
         target_state
             .iter()
@@ -3868,19 +3819,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         .await?;
     assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
 
-    // Wait for the source workers to finish their final writeback, which runs
-    // after the remote purge has already become visible (rustfs#9121).
-    wait_for_source_writeback().await?;
-    let logs = fs::read_to_string(source_log).await?;
-    let writeback_failures: Vec<_> = logs
-        .lines()
-        .filter(|line| line.contains("apply_replication_delete_state"))
-        .collect();
-    assert!(
-        writeback_failures.is_empty(),
-        "a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
-    );
-
     Ok(())
 }
 

crates/e2e_test/src/rpc_fault_proxy.rs

@@ -28,8 +28,7 @@ use hyper_util::{
 use prost::Message as _;
 use rustfs_protos::proto_gen::node_service::{
     BatchGenerallyLockRequest, DeleteRequest, GenerallyLockRequest, PreparePartTransactionResponse, ReadVersionRequest,
-    RecoverMultipartSourceRequest, RenameDataResponse, ScannerScopedDirtyUsageAckRequest, SettlePartTransactionRequest,
-    SettlePartTransactionResponse,
+    RecoverMultipartSourceRequest, RenameDataResponse, SettlePartTransactionRequest, SettlePartTransactionResponse,
 };
 use std::{
     collections::HashSet,
@@ -50,9 +49,6 @@ use tokio::{
 type ProxyBody = BoxBody<Bytes, io::Error>;
 type ProxyClient = Client<HttpConnector, ProxyBody>;
 
-mod scoped_ack;
-pub(crate) use scoped_ack::{ScopedAckFault, ScopedAckQueue, ScopedAckScope};
-
 #[derive(Clone, Copy, PartialEq, Eq)]
 enum Disposition {
     Hold,
@@ -125,8 +121,7 @@ impl RpcPause {
         self.state.blocked.load(Ordering::SeqCst)
     }
 
-    /// Selected RPCs with successful HTTP/gRPC response headers, excluding
-    /// discarded packets. This does not prove a successful gRPC trailer or ACK apply.
+    /// Selected RPCs with successful HTTP/gRPC headers, excluding discarded packets.
     pub(crate) fn forwarded(&self) -> usize {
         self.state.forwarded.load(Ordering::SeqCst)
     }
@@ -238,7 +233,6 @@ pub(crate) struct RpcFaultProxy {
     address: SocketAddr,
     slot: PauseSlot,
     lock_slot: PauseSlot,
-    scoped_ack: scoped_ack::QueueSlot,
     task: JoinHandle<()>,
     late_requests: LateRequests,
 }
@@ -251,8 +245,6 @@ impl RpcFaultProxy {
         let active = slot.clone();
         let lock_slot: PauseSlot = Arc::new(Mutex::new(None));
         let active_lock = lock_slot.clone();
-        let scoped_ack = scoped_ack::empty_slot();
-        let active_ack = scoped_ack.clone();
         let late_requests: LateRequests = Arc::new(Mutex::new(Some(Vec::new())));
         let late = late_requests.clone();
         let task = tokio::spawn(async move {
@@ -267,7 +259,6 @@ impl RpcFaultProxy {
                         let h2 = h2.clone();
                         let active = active.clone();
                         let active_lock = active_lock.clone();
-                        let active_ack = active_ack.clone();
                         let late = late.clone();
                         connections.spawn(async move {
                             let service = service_fn(move |request: Request<Incoming>| {
@@ -281,7 +272,6 @@ impl RpcFaultProxy {
                                     None
                                 };
                                 let pause = owner_pause.or_else(|| active.lock().expect("RPC pause slot").clone());
-                                let ack = active_ack.lock().expect("scoped ACK queue slot").clone();
                                 let retain = pause.as_ref().is_some_and(|pause| {
                                     pause.retain_after_disconnect
                                         && request.uri().path().strip_prefix("/node_service.NodeService/")
@@ -290,11 +280,11 @@ impl RpcFaultProxy {
                                 let late = late.clone();
                                 async move {
                                     if !retain {
-                                        return forward_request(request, client, target, pause, ack).await;
+                                        return forward_request(request, client, target, pause).await;
                                     }
                                     // This opt-in fault owns the packet after client cancellation.
                                     // Proxy teardown still aborts every retained request.
-                                    let task = tokio::spawn(forward_request(request, client, target, pause, ack));
+                                    let task = tokio::spawn(forward_request(request, client, target, pause));
                                     {
                                         let mut requests = late.lock().expect("late RPC ownership");
                                         if let Some(requests) = requests.as_mut() {
@@ -319,7 +309,6 @@ impl RpcFaultProxy {
             address,
             slot,
             lock_slot,
-            scoped_ack,
             task,
             late_requests,
         })
@@ -329,11 +318,6 @@ impl RpcFaultProxy {
         self.address
     }
 
-    /// Retain only genuine scoped ACKs for one receiver/session and bucket set.
-    pub(crate) fn scoped_ack_queue(&self, scope: ScopedAckScope, capacity: usize) -> io::Result<ScopedAckQueue> {
-        ScopedAckQueue::arm(self.scoped_ack.clone(), scope, capacity)
-    }
-
     /// Pause at most limit requests to the named unary methods; other streams
     /// on the same HTTP/2 connection continue forwarding.
     pub(crate) fn pause(&self, methods: &[&'static str], limit: usize, scope: Option<&str>) -> io::Result<RpcPause> {
@@ -485,9 +469,6 @@ fn unary_payload(body: &[u8]) -> io::Result<&[u8]> {
 }
 
 fn scope_matches(method: &str, body: &[u8], scope: &[u8]) -> io::Result<bool> {
-    if method == "ScannerScopedDirtyUsageAck" {
-        return scoped_ack_matches(body, Some(scope));
-    }
     if method == "RecoverMultipartSource" {
         let request = RecoverMultipartSourceRequest::decode(unary_payload(body)?).map_err(io::Error::other)?;
         if request.protocol_version != u32::from(MULTIPART_SOURCE_PROTOCOL_VERSION) || request.path.as_bytes() != scope {
@@ -544,9 +525,7 @@ fn scoped_lock_owners(method: &str, body: &[u8], scope: &[u8]) -> io::Result<Vec
     let args = match method {
         "Lock" => vec![GenerallyLockRequest::decode(payload).map_err(io::Error::other)?.args],
         "LockBatch" => BatchGenerallyLockRequest::decode(payload).map_err(io::Error::other)?.args,
-        _ => {
-            return Err(io::Error::new(io::ErrorKind::InvalidInput, "lock observer requires a lock method"));
-        }
+        _ => return Err(io::Error::new(io::ErrorKind::InvalidInput, "lock observer requires a lock method")),
     };
     args.into_iter()
         .filter(|args| args.as_bytes().windows(scope.len()).any(|window| window == scope))
@@ -561,39 +540,18 @@ fn scoped_lock_owners(method: &str, body: &[u8], scope: &[u8]) -> io::Result<Vec
         .collect()
 }
 
-fn scoped_ack_matches(body: &[u8], scope: Option<&[u8]>) -> io::Result<bool> {
-    let request = ScannerScopedDirtyUsageAckRequest::decode(unary_payload(body)?).map_err(io::Error::other)?;
-    Ok(!request.probe_only && scope.is_none_or(|scope| request.entries.iter().any(|entry| entry.bucket.as_bytes() == scope)))
-}
-
 async fn forward_request(
     request: Request<Incoming>,
     client: ProxyClient,
     target: SocketAddr,
     pause: Option<Arc<PauseState>>,
-    ack: Option<Arc<scoped_ack::QueueState>>,
 ) -> io::Result<hyper::Response<ProxyBody>> {
-    let request = request.map(|body| body.map_err(io::Error::other).boxed());
-    let request = if let Some(ack) = ack
-        && request.uri().path() == "/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"
-    {
-        match scoped_ack::intercept(request, client.clone(), target, ack).await? {
-            scoped_ack::Interception::Pass(request) => request,
-            scoped_ack::Interception::Handled(response) => return Ok(response),
-        }
-    } else {
-        request
-    };
-    let (parts, body) = request.into_parts();
-    let mut body = body;
+    let (mut parts, body) = request.into_parts();
+    let mut body = body.map_err(io::Error::other).boxed();
     let mut observed = None;
     let mut inspection = None;
     if let Some(pause) = pause {
-        let method = parts
-            .uri
-            .path()
-            .strip_prefix("/node_service.NodeService/")
-            .or_else(|| parts.uri.path().strip_prefix("/node_service.ScannerControlService/"));
+        let method = parts.uri.path().strip_prefix("/node_service.NodeService/");
         if let Some(method) = method.filter(|name| pause.methods.contains(name)) {
             let mut route = None;
             let mut lock_owners = Vec::new();
@@ -641,16 +599,6 @@ async fn forward_request(
                     .with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
                     .boxed();
                 matches
-            } else if method == "ScannerScopedDirtyUsageAck" {
-                let collected = Limited::new(body, 1024 * 1024).collect().await.map_err(io::Error::other)?;
-                let trailers = collected.trailers().cloned();
-                let bytes = collected.to_bytes();
-                let matches = scoped_ack_matches(&bytes, None)?;
-                body = Full::new(bytes)
-                    .map_err(|never| match never {})
-                    .with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
-                    .boxed();
-                matches
             } else {
                 true
             };
@@ -694,7 +642,24 @@ async fn forward_request(
             }
         }
     }
-    let response = forward_to_backend(Request::from_parts(parts, body), client, target).await?;
+    if !parts.headers.contains_key(http::header::HOST)
+        && let Some(authority) = parts.uri.authority()
+    {
+        parts.headers.insert(
+            http::header::HOST,
+            http::HeaderValue::from_str(authority.as_str()).map_err(io::Error::other)?,
+        );
+    }
+    parts.uri = http::Uri::builder()
+        .scheme("http")
+        .authority(target.to_string())
+        .path_and_query(parts.uri.path_and_query().map_or("/", |value| value.as_str()))
+        .build()
+        .map_err(io::Error::other)?;
+    let response = client
+        .request(Request::from_parts(parts, body))
+        .await
+        .map_err(io::Error::other)?;
     if let Some((pause, disk)) = inspection
         && response.status().is_success()
         && response.headers().get("grpc-status").is_some_and(|status| status == "12")
@@ -747,32 +712,6 @@ async fn forward_request(
     Ok(response.map(|body| body.map_err(io::Error::other).boxed()))
 }
 
-async fn forward_to_backend(
-    request: Request<ProxyBody>,
-    client: ProxyClient,
-    target: SocketAddr,
-) -> io::Result<hyper::Response<Incoming>> {
-    let (mut parts, body) = request.into_parts();
-    if !parts.headers.contains_key(http::header::HOST)
-        && let Some(authority) = parts.uri.authority()
-    {
-        parts.headers.insert(
-            http::header::HOST,
-            http::HeaderValue::from_str(authority.as_str()).map_err(io::Error::other)?,
-        );
-    }
-    parts.uri = http::Uri::builder()
-        .scheme("http")
-        .authority(target.to_string())
-        .path_and_query(parts.uri.path_and_query().map_or("/", |value| value.as_str()))
-        .build()
-        .map_err(io::Error::other)?;
-    client
-        .request(Request::from_parts(parts, body))
-        .await
-        .map_err(io::Error::other)
-}
-
 impl Drop for RpcFaultProxy {
     fn drop(&mut self) {
         if let Some(requests) = self.late_requests.lock().expect("late RPC ownership").take() {
@@ -792,7 +731,6 @@ mod tests {
 
     struct Backend {
         received: Arc<AtomicUsize>,
-        auth_headers: Arc<Mutex<Vec<(String, String)>>>,
         address: SocketAddr,
         task: JoinHandle<()>,
     }
@@ -816,8 +754,6 @@ mod tests {
         let address = listener.local_addr()?;
         let received = Arc::new(AtomicUsize::new(0));
         let received_task = received.clone();
-        let auth_headers = Arc::new(Mutex::new(Vec::new()));
-        let backend_auth = auth_headers.clone();
         let task = tokio::spawn(async move {
             let mut children = JoinSet::new();
             loop {
@@ -825,12 +761,10 @@ mod tests {
                     accepted = listener.accept() => {
                         let Ok((socket, _)) = accepted else { break };
                         let received = received_task.clone();
-                        let backend_auth = backend_auth.clone();
                         let reply = reply.clone();
                         children.spawn(async move {
                             let service = service_fn(move |request: Request<Incoming>| {
                                 let received = received.clone();
-                                let backend_auth = backend_auth.clone();
                                 let reply = reply.clone();
                                 async move {
                                 received.fetch_add(1, Ordering::SeqCst);
@@ -839,10 +773,6 @@ mod tests {
                                         http::HeaderValue::from_str(authority.as_str()).expect("valid HTTP authority")
                                     })
                                 });
-                                if let (Some(nonce), Some(signature)) = (request.headers().get("x-rustfs-rpc-nonce"), request.headers().get("x-rustfs-rpc-signature-v2")) {
-                                    backend_auth.lock().expect("backend auth header observations").push((
-                                        nonce.to_str().expect("test nonce").to_string(), signature.to_str().expect("test signature").to_string()));
-                                }
                                 let body = request.into_body().collect().await?.to_bytes();
                                 let body = reply.unwrap_or(body);
                                 let mut trailers = http::HeaderMap::new();
@@ -865,12 +795,7 @@ mod tests {
                 }
             }
         });
-        Ok(Backend {
-            received,
-            auth_headers,
-            address,
-            task,
-        })
+        Ok(Backend { received, address, task })
     }
 
     async fn call(client: &ProxyClient, address: SocketAddr, method: &str) -> io::Result<hyper::Response<Incoming>> {
@@ -884,14 +809,9 @@ mod tests {
         bytes: Bytes,
     ) -> io::Result<hyper::Response<Incoming>> {
         let body: ProxyBody = Full::new(bytes).map_err(|never| match never {}).boxed();
-        let service = if method == "ScannerScopedDirtyUsageAck" {
-            "ScannerControlService"
-        } else {
-            "NodeService"
-        };
         let request = Request::builder()
             .method("POST")
-            .uri(format!("http://{address}/node_service.{service}/{method}"))
+            .uri(format!("http://{address}/node_service.NodeService/{method}"))
             .header(http::header::TE, "trailers")
             .body(body)
             .map_err(io::Error::other)?;
@@ -906,353 +826,6 @@ mod tests {
         frame.into()
     }
 
-    const ACK_OWNER: &str = "11111111-1111-1111-1111-111111111111";
-    const ACK_INSTANCE: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
-
-    fn ack_scope() -> ScopedAckScope {
-        ScopedAckScope {
-            nonce: uuid::Uuid::new_v4().to_string(),
-            owner_id: ACK_OWNER.to_string(),
-            instance_id: ACK_INSTANCE.to_string(),
-            buckets: std::collections::BTreeSet::from(["cold-bucket".to_string()]),
-        }
-    }
-
-    fn ack_request(bucket: &str, generation: u64) -> ScannerScopedDirtyUsageAckRequest {
-        ScannerScopedDirtyUsageAckRequest {
-            challenge: vec![7; 16].into(),
-            protocol_version: 1,
-            owner_id: ACK_OWNER.to_string(),
-            instance_id: ACK_INSTANCE.to_string(),
-            scope: 1,
-            probe_only: false,
-            entries: vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry {
-                bucket: bucket.to_string(),
-                bucket_incarnation: vec![4; 16].into(),
-                generation,
-            }],
-        }
-    }
-
-    fn ack_frame(request: &ScannerScopedDirtyUsageAckRequest) -> Bytes {
-        let payload = request.encode_to_vec();
-        let mut frame = vec![0];
-        frame.extend_from_slice(&u32::try_from(payload.len()).expect("bounded test frame").to_be_bytes());
-        frame.extend_from_slice(&payload);
-        Bytes::from(frame)
-    }
-
-    fn ack_hash(bytes: &[u8]) -> String {
-        use sha2::Digest as _;
-        hex_simd::encode_to_string(&sha2::Sha256::digest(bytes)[..], hex_simd::AsciiCase::Lower)
-    }
-
-    // Transport-only fixtures: the distributed tests supply production signatures.
-    async fn call_ack_frame(
-        client: &ProxyClient,
-        address: SocketAddr,
-        frame: Bytes,
-        nonce: &str,
-    ) -> io::Result<hyper::Response<Incoming>> {
-        let request = Request::builder()
-            .method("POST")
-            .uri(format!("http://{address}/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"))
-            .header(http::header::TE, "trailers")
-            .header("x-rustfs-rpc-nonce", nonce)
-            .header("x-rustfs-rpc-signature-v2", format!("opaque-signature-{nonce}"))
-            .body(Full::new(frame).map_err(|never| match never {}).boxed())
-            .map_err(io::Error::other)?;
-        client.request(request).await.map_err(io::Error::other)
-    }
-
-    #[tokio::test]
-    async fn scoped_ack_queue_loss_preserves_backend_boundary() {
-        tokio::time::timeout(Duration::from_secs(10), async {
-            for fault in [ScopedAckFault::RequestLoss, ScopedAckFault::ResponseLoss] {
-                let backend = backend().await.expect("backend");
-                let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
-                    .await
-                    .expect("proxy");
-                let queue = proxy.scoped_ack_queue(ack_scope(), 1).expect("ACK queue");
-                let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
-                let frame = ack_frame(&ack_request("cold-bucket", 7));
-                let address = proxy.local_addr();
-                let body = frame.clone();
-                let caller = tokio::spawn(async move { call_ack_frame(&client, address, body, "loss").await });
-                queue.wait_for_packets(1).await.expect("captured ACK");
-                assert_eq!(backend.received.load(Ordering::SeqCst), 0, "retention must precede backend dispatch");
-                queue.settle(0, fault).expect("settle actual retained packet");
-                assert!(
-                    caller.await.expect("caller task").is_err(),
-                    "the caller must observe the injected transport failure"
-                );
-                let receipt = queue.wait_for_completion(0).await.expect("complete fault receipt");
-                let response_loss = fault == ScopedAckFault::ResponseLoss;
-                assert_eq!(receipt["forwarded"], response_loss);
-                assert_eq!(receipt["backend_complete"], response_loss);
-                assert_eq!(backend.received.load(Ordering::SeqCst), usize::from(response_loss));
-                if response_loss {
-                    assert_eq!(receipt["outcome"], "response_lost");
-                    assert_eq!(receipt["response"]["grpc_status"], "0", "status exists only in real backend trailers");
-                    assert_eq!(receipt["response"]["bytes"], frame.len());
-                    assert_eq!(receipt["response"]["sha256"], ack_hash(&frame));
-                    assert!(
-                        receipt["response"]["trailers_sha256"]
-                            .as_str()
-                            .is_some_and(|value| value.len() == 64)
-                    );
-                } else {
-                    assert_eq!(receipt["outcome"], "request_lost");
-                }
-            }
-        })
-        .await
-        .expect("bounded ACK loss checks");
-    }
-
-    #[tokio::test]
-    async fn scoped_ack_queue_keeps_cancelled_packets_and_releases_out_of_order() {
-        tokio::time::timeout(Duration::from_secs(10), async {
-            let backend = backend().await.expect("backend");
-            let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
-                .await
-                .expect("proxy");
-            let queue = proxy.scoped_ack_queue(ack_scope(), 2).expect("two-packet ACK queue");
-            let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
-            let first = ack_frame(&ack_request("cold-bucket", 7));
-            let second = ack_frame(&ack_request("cold-bucket", 9));
-            let address = proxy.local_addr();
-            let old_client = client.clone();
-            let old_body = first.clone();
-            let old_caller = tokio::spawn(async move { call_ack_frame(&old_client, address, old_body, "first").await });
-            queue.wait_for_packets(1).await.expect("first retained packet");
-            old_caller.abort();
-            assert!(old_caller.await.expect_err("cancel original caller").is_cancelled());
-            let new_body = second.clone();
-            let new_caller = tokio::spawn(async move { call_ack_frame(&client, address, new_body, "second").await });
-            queue.wait_for_packets(2).await.expect("second retained packet");
-            assert_eq!(backend.received.load(Ordering::SeqCst), 0);
-            queue.settle(1, ScopedAckFault::Forward).expect("release newer packet first");
-            let response = new_caller
-                .await
-                .expect("new caller task")
-                .expect("new response")
-                .into_body()
-                .collect()
-                .await
-                .expect("complete body");
-            assert_eq!(response.trailers().expect("response trailers")["x-proof"], "unchanged-proof");
-            assert_eq!(response.to_bytes(), second);
-            queue.wait_for_completion(1).await.expect("new packet complete");
-            queue
-                .settle(0, ScopedAckFault::Forward)
-                .expect("release old packet after its caller disappeared");
-            let old = queue.wait_for_completion(0).await.expect("detached old packet complete");
-            assert_eq!(old["backend_complete"], true);
-            assert_eq!(old["request_sha256"], ack_hash(&first));
-            assert_eq!(old["response"]["sha256"], ack_hash(&first));
-            assert_eq!(backend.received.load(Ordering::SeqCst), 2);
-            assert_eq!(
-                *backend.auth_headers.lock().expect("backend header observations"),
-                vec![
-                    ("second".to_string(), "opaque-signature-second".to_string()),
-                    ("first".to_string(), "opaque-signature-first".to_string())
-                ]
-            );
-            assert!(
-                queue.settle(0, ScopedAckFault::Forward).is_err(),
-                "one original packet may only be forwarded once"
-            );
-        })
-        .await
-        .expect("bounded detached retention check");
-    }
-
-    #[tokio::test]
-    async fn scoped_ack_queue_bounds_capacity_and_excludes_other_scopes() {
-        tokio::time::timeout(Duration::from_secs(10), async {
-            let backend = backend().await.expect("backend");
-            let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
-                .await
-                .expect("proxy");
-            assert!(proxy.scoped_ack_queue(ack_scope(), 0).is_err());
-            assert!(proxy.scoped_ack_queue(ack_scope(), 5).is_err());
-            let mut invalid = ack_scope();
-            invalid.nonce.clear();
-            assert!(proxy.scoped_ack_queue(invalid, 1).is_err());
-            let mut scope = ack_scope();
-            scope.buckets.insert("cold-bucket-z".to_string());
-            let queue = proxy.scoped_ack_queue(scope, 1).expect("bounded ACK queue");
-            let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
-            let mut probe = ack_request("cold-bucket", 7);
-            probe.probe_only = true;
-            let mut other_peer = ack_request("cold-bucket", 7);
-            other_peer.owner_id = uuid::Uuid::new_v4().to_string();
-            let mut other_instance = ack_request("cold-bucket", 7);
-            other_instance.instance_id = "b".repeat(32);
-            let mut invalid_challenge = ack_request("cold-bucket", 7);
-            invalid_challenge.challenge = vec![7; 15].into();
-            let mut noncanonical_entries = ack_request("cold-bucket-z", 7);
-            noncanonical_entries.entries.extend(ack_request("cold-bucket", 7).entries);
-            let unselected = [
-                probe,
-                other_peer,
-                other_instance,
-                ack_request("cold-bucket-other", 7),
-                ack_request("cold-bucket", u64::MAX),
-                invalid_challenge,
-                noncanonical_entries,
-            ];
-            let unselected_count = unselected.len();
-            for (index, request) in unselected.into_iter().enumerate() {
-                let body = ack_frame(&request);
-                let response = call_ack_frame(&client, proxy.local_addr(), body.clone(), "unselected")
-                    .await
-                    .expect("unselected ACK progresses");
-                assert_eq!(response.into_body().collect().await.expect("unselected body").to_bytes(), body);
-                assert!(queue.receipts().is_empty(), "unselected or protocol-invalid ACK must not consume a slot");
-                assert_eq!(backend.received.load(Ordering::SeqCst), index + 1);
-            }
-            assert!(queue.receipts().is_empty());
-            let address = proxy.local_addr();
-            let held_client = client.clone();
-            let valid_frame = ack_frame(&ack_request("cold-bucket", 7));
-            let held_body = valid_frame.clone();
-            let held = tokio::spawn(async move { call_ack_frame(&held_client, address, held_body, "held").await });
-            queue.wait_for_packets(1).await.expect("one matching packet");
-            assert!(!held.is_finished(), "the subsequent valid ACK must be retained");
-            assert_eq!(queue.receipts()[0]["request_sha256"], ack_hash(&valid_frame));
-            assert_eq!(backend.received.load(Ordering::SeqCst), unselected_count);
-            let extra = call_ack_frame(&client, address, ack_frame(&ack_request("cold-bucket", 8)), "beyond-capacity")
-                .await
-                .expect("capacity overflow passes without retention");
-            extra.into_body().collect().await.expect("overflow response");
-            assert_eq!(queue.receipts().len(), 1);
-            assert_eq!(backend.received.load(Ordering::SeqCst), unselected_count + 1);
-            drop(queue);
-            assert!(
-                held.await.expect("held caller").is_err(),
-                "dropping the session must cancel its retained packet"
-            );
-            assert_eq!(
-                backend.received.load(Ordering::SeqCst),
-                unselected_count + 1,
-                "session drop must never forward the held packet"
-            );
-        })
-        .await
-        .expect("bounded scope and lifecycle checks");
-    }
-
-    #[tokio::test]
-    async fn scoped_ack_queue_expiry_never_forwards_retained_packet() {
-        let (backend, _proxy, queue, caller) = tokio::time::timeout(Duration::from_secs(10), async {
-            let backend = backend().await.expect("backend");
-            let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
-                .await
-                .expect("proxy");
-            let queue = proxy.scoped_ack_queue(ack_scope(), 1).expect("ACK queue");
-            let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
-            let address = proxy.local_addr();
-            let caller = tokio::spawn(async move {
-                call_ack_frame(&client, address, ack_frame(&ack_request("cold-bucket", 7)), "expires").await
-            });
-            queue.wait_for_packets(1).await.expect("retained packet");
-            (backend, proxy, queue, caller)
-        })
-        .await
-        .expect("bounded queue setup");
-        tokio::time::pause();
-        tokio::time::advance(Duration::from_secs(120)).await;
-        tokio::time::resume();
-        assert!(
-            tokio::time::timeout(Duration::from_secs(10), caller)
-                .await
-                .expect("expiry completes caller")
-                .expect("caller task")
-                .is_err()
-        );
-        let receipt = queue.receipts().into_iter().next().expect("expired receipt");
-        assert_eq!(receipt["completed"], true);
-        assert_eq!(receipt["forwarded"], false);
-        assert_eq!(receipt["backend_complete"], false);
-        assert_eq!(backend.received.load(Ordering::SeqCst), 0);
-        assert_eq!(
-            queue
-                .settle(0, ScopedAckFault::Forward)
-                .expect_err("expired packet cannot be released")
-                .kind(),
-            io::ErrorKind::TimedOut
-        );
-    }
-
-    #[tokio::test]
-    async fn scanner_ack_fault_matches_exact_bucket_and_excludes_probes() {
-        use rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry;
-
-        tokio::time::timeout(Duration::from_secs(10), async {
-            let backend = backend().await.expect("backend listener");
-            let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
-                .await
-                .expect("proxy listener");
-            let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
-            let pause = proxy
-                .pause(&["ScannerScopedDirtyUsageAck"], 1, Some("cold-bucket"))
-                .expect("scoped ACK fault");
-            let frame = |bucket: &str, probe_only: bool| {
-                let payload = ScannerScopedDirtyUsageAckRequest {
-                    challenge: vec![3; 16].into(),
-                    protocol_version: 1,
-                    owner_id: "11111111-1111-1111-1111-111111111111".to_string(),
-                    instance_id: "a".repeat(32),
-                    scope: 1,
-                    probe_only,
-                    entries: vec![ScannerScopedDirtyUsageEntry {
-                        bucket: bucket.to_string(),
-                        bucket_incarnation: vec![4; 16].into(),
-                        generation: 17,
-                    }],
-                }
-                .encode_to_vec();
-                let mut frame = vec![0];
-                frame.extend_from_slice(&u32::try_from(payload.len()).expect("small ACK frame").to_be_bytes());
-                frame.extend_from_slice(&payload);
-                Bytes::from(frame)
-            };
-            for bytes in [frame("cold-bucket", true), frame("cold-bucket-other", false)] {
-                let response = call_body(&client, proxy.local_addr(), "ScannerScopedDirtyUsageAck", bytes.clone())
-                    .await
-                    .expect("probe and other bucket must progress");
-                assert_eq!(response.into_body().collect().await.expect("unselected response").to_bytes(), bytes);
-            }
-            assert_eq!(pause.hits(), 0);
-            let signed_bytes = frame("cold-bucket", false);
-            let task_bytes = signed_bytes.clone();
-            let address = proxy.local_addr();
-            let blocked =
-                tokio::spawn(async move { call_body(&client, address, "ScannerScopedDirtyUsageAck", task_bytes).await });
-            pause
-                .wait_for_hits(1)
-                .await
-                .expect("real ScannerControlService method must reach the fault");
-            assert!(!blocked.is_finished());
-            assert_eq!(backend.received.load(Ordering::SeqCst), 2, "held ACK must not apply at the backend");
-            pause.release();
-            let collected = blocked
-                .await
-                .expect("ACK task")
-                .expect("released ACK")
-                .into_body()
-                .collect()
-                .await
-                .expect("ACK body and trailers");
-            assert_eq!(collected.trailers().expect("ACK trailers")["x-proof"], "unchanged-proof");
-            assert_eq!(collected.to_bytes(), signed_bytes, "proxy must preserve the complete signed ACK frame");
-        })
-        .await
-        .expect("scanner ACK fault deadline");
-    }
-
     fn native_abort_request(path: &str) -> RecoverMultipartSourceRequest {
         use crate::storage_api::multipart_source_rpc::{
             MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind, MultipartAbortRound, MultipartSourcePhase,
@@ -2146,4 +1719,3 @@ mod tests {
         );
     }
 }
-// event-drill: isolated harness-only change

crates/e2e_test/src/rpc_fault_proxy/scoped_ack.rs

@@ -1,404 +0,0 @@
-// Copyright 2026 RustFS Team
-// SPDX-License-Identifier: Apache-2.0
-
-//! Bounded retention of original scoped ACK frames, independent of caller cancellation.
-
-use super::{ProxyBody, ProxyClient, forward_to_backend, unary_payload};
-use bytes::Bytes;
-use http_body_util::{BodyExt as _, Full, Limited};
-use hyper::{Request, Response};
-use prost::Message as _;
-use rustfs_protos::proto_gen::node_service::{ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse};
-use rustfs_protos::scoped_dirty_usage::validate_scoped_dirty_usage_request;
-use serde_json::{Value, json};
-use sha2::{Digest as _, Sha256};
-use std::{
-    collections::BTreeSet,
-    io,
-    net::SocketAddr,
-    sync::{
-        Arc, Mutex,
-        atomic::{AtomicBool, Ordering},
-    },
-    time::Duration,
-};
-use tokio::{
-    sync::{Notify, oneshot, watch},
-    task::AbortHandle,
-    time::Instant,
-};
-
-const MAX_RETAINED: usize = 4;
-const MAX_BYTES: usize = 1024 * 1024;
-const RETENTION: Duration = Duration::from_secs(120);
-
-pub(crate) struct ScopedAckScope {
-    pub nonce: String,
-    pub owner_id: String,
-    pub instance_id: String,
-    pub buckets: BTreeSet<String>,
-}
-
-impl ScopedAckScope {
-    fn validate(&self) -> io::Result<()> {
-        let valid_uuid = |value: &str| uuid::Uuid::parse_str(value).is_ok_and(|value| !value.is_nil());
-        if !valid_uuid(&self.nonce)
-            || !valid_uuid(&self.owner_id)
-            || self.instance_id.is_empty()
-            || self.instance_id.len() > 128
-            || self.buckets.is_empty()
-            || self.buckets.len() > 16
-            || self.buckets.iter().any(|bucket| bucket.is_empty() || bucket.len() > 63)
-        {
-            return Err(io::Error::new(
-                io::ErrorKind::InvalidInput,
-                "scoped ACK queue needs a bounded session, receiver and bucket scope",
-            ));
-        }
-        Ok(())
-    }
-
-    fn matches(&self, request: &ScannerScopedDirtyUsageAckRequest) -> bool {
-        !request.probe_only
-            && validate_scoped_dirty_usage_request(request).is_ok()
-            && request.owner_id == self.owner_id
-            && request.instance_id == self.instance_id
-            && request.entries.iter().all(|entry| self.buckets.contains(&entry.bucket))
-    }
-}
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub(crate) enum ScopedAckFault {
-    Forward,
-    RequestLoss,
-    ResponseLoss,
-}
-
-struct Packet {
-    action: watch::Sender<Option<ScopedAckFault>>,
-    receipt: Mutex<Value>,
-    task: Mutex<Option<AbortHandle>>,
-}
-
-pub(super) struct QueueState {
-    scope: ScopedAckScope,
-    capacity: usize,
-    started: Instant,
-    deadline: Instant,
-    closed: AtomicBool,
-    // The packet inventory lock is released before any packet lock or await.
-    packets: Mutex<Vec<Arc<Packet>>>,
-    changed: Notify,
-}
-
-pub(super) type QueueSlot = Arc<Mutex<Option<Arc<QueueState>>>>;
-
-pub(super) fn empty_slot() -> QueueSlot {
-    Arc::new(Mutex::new(None))
-}
-
-pub(crate) struct ScopedAckQueue {
-    state: Arc<QueueState>,
-    slot: QueueSlot,
-}
-
-impl ScopedAckQueue {
-    pub(super) fn arm(slot: QueueSlot, scope: ScopedAckScope, capacity: usize) -> io::Result<Self> {
-        scope.validate()?;
-        if !(1..=MAX_RETAINED).contains(&capacity) {
-            return Err(io::Error::new(io::ErrorKind::InvalidInput, "scoped ACK capacity must be 1..=4"));
-        }
-        let mut active = slot.lock().expect("scoped ACK queue slot");
-        if active.is_some() {
-            return Err(io::Error::new(io::ErrorKind::AlreadyExists, "scoped ACK queue is already armed"));
-        }
-        let started = Instant::now();
-        let state = Arc::new(QueueState {
-            scope,
-            capacity,
-            started,
-            deadline: started + RETENTION,
-            closed: AtomicBool::new(false),
-            packets: Mutex::new(Vec::new()),
-            changed: Notify::new(),
-        });
-        *active = Some(state.clone());
-        drop(active);
-        Ok(Self { state, slot })
-    }
-
-    pub(crate) fn receipts(&self) -> Vec<Value> {
-        let packets = self.state.packets.lock().expect("scoped ACK packets").clone();
-        packets
-            .iter()
-            .map(|packet| packet.receipt.lock().expect("scoped ACK receipt").clone())
-            .collect()
-    }
-
-    pub(crate) async fn wait_for_packets(&self, count: usize) -> io::Result<()> {
-        self.wait_until(|| self.receipts().len() >= count).await
-    }
-
-    pub(crate) async fn wait_for_completion(&self, id: usize) -> io::Result<Value> {
-        self.wait_until(|| {
-            self.receipts()
-                .get(id)
-                .is_some_and(|receipt| receipt["completed"].as_bool() == Some(true))
-        })
-        .await?;
-        self.receipts()
-            .get(id)
-            .cloned()
-            .ok_or_else(|| io::Error::other("scoped ACK receipt disappeared"))
-    }
-
-    async fn wait_until(&self, ready: impl Fn() -> bool) -> io::Result<()> {
-        tokio::time::timeout_at(self.state.deadline, async {
-            loop {
-                let changed = self.state.changed.notified();
-                if ready() {
-                    return;
-                }
-                changed.await;
-            }
-        })
-        .await
-        .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "scoped ACK queue lifetime elapsed"))
-    }
-
-    pub(crate) fn settle(&self, id: usize, fault: ScopedAckFault) -> io::Result<()> {
-        if Instant::now() >= self.state.deadline {
-            return Err(io::Error::new(io::ErrorKind::TimedOut, "scoped ACK retention expired"));
-        }
-        let packet = self
-            .state
-            .packets
-            .lock()
-            .expect("scoped ACK packets")
-            .get(id)
-            .cloned()
-            .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "scoped ACK packet is not retained"))?;
-        if !packet.action.send_if_modified(|current| {
-            if current.is_none() {
-                *current = Some(fault);
-                true
-            } else {
-                false
-            }
-        }) {
-            return Err(io::Error::new(io::ErrorKind::AlreadyExists, "scoped ACK packet was already settled"));
-        }
-        Ok(())
-    }
-}
-
-impl Drop for ScopedAckQueue {
-    fn drop(&mut self) {
-        self.state.closed.store(true, Ordering::SeqCst);
-        let mut slot = self.slot.lock().expect("scoped ACK queue slot");
-        if slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, &self.state)) {
-            *slot = None;
-        }
-        drop(slot);
-        let packets = self.state.packets.lock().expect("scoped ACK packets").clone();
-        for packet in packets {
-            if let Some(task) = packet.task.lock().expect("scoped ACK task").take() {
-                task.abort();
-            }
-        }
-    }
-}
-
-pub(super) enum Interception {
-    Pass(Request<ProxyBody>),
-    Handled(Response<ProxyBody>),
-}
-
-fn restored_body(bytes: Bytes, trailers: Option<http::HeaderMap>) -> ProxyBody {
-    Full::new(bytes)
-        .map_err(|never| match never {})
-        .with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
-        .boxed()
-}
-
-fn hash(bytes: &[u8]) -> String {
-    hex(Sha256::digest(bytes).as_ref())
-}
-
-fn hex(bytes: &[u8]) -> String {
-    hex_simd::encode_to_string(bytes, hex_simd::AsciiCase::Lower)
-}
-
-fn trailer_receipt(trailers: Option<&http::HeaderMap>) -> Value {
-    trailers.map_or(Value::Null, |headers| {
-        json!(
-            headers
-                .iter()
-                .map(|(name, value)| { json!({"name": name.as_str(), "value_hex": hex(value.as_bytes())}) })
-                .collect::<Vec<_>>()
-        )
-    })
-}
-
-fn header_hash(headers: &http::HeaderMap) -> String {
-    let mut hasher = Sha256::new();
-    for (name, value) in headers {
-        hasher.update(name.as_str().as_bytes());
-        hasher.update([0]);
-        hasher.update(value.as_bytes());
-        hasher.update([0]);
-    }
-    hex(hasher.finalize().as_ref())
-}
-
-fn millis(state: &QueueState) -> u64 {
-    u64::try_from(state.started.elapsed().as_millis()).unwrap_or(u64::MAX)
-}
-
-pub(super) async fn intercept(
-    request: Request<ProxyBody>,
-    client: ProxyClient,
-    target: SocketAddr,
-    state: Arc<QueueState>,
-) -> io::Result<Interception> {
-    if Instant::now() >= state.deadline || state.closed.load(Ordering::SeqCst) {
-        return Ok(Interception::Pass(request));
-    }
-    let (parts, body) = request.into_parts();
-    let collected = tokio::time::timeout_at(state.deadline, Limited::new(body, MAX_BYTES).collect())
-        .await
-        .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "scoped ACK frame collection expired"))?
-        .map_err(io::Error::other)?;
-    let trailers = collected.trailers().cloned();
-    let bytes = collected.to_bytes();
-    let decoded = ScannerScopedDirtyUsageAckRequest::decode(unary_payload(&bytes)?).map_err(io::Error::other)?;
-    let request = Request::from_parts(parts, restored_body(bytes.clone(), trailers.clone()));
-    if Instant::now() >= state.deadline || !state.scope.matches(&decoded) {
-        return Ok(Interception::Pass(request));
-    }
-    let header = |name| request.headers().get(name).and_then(|value| value.to_str().ok());
-    let (action, mut selected) = watch::channel(None);
-    let packet = {
-        let mut packets = state.packets.lock().expect("scoped ACK packets");
-        if packets.len() >= state.capacity || state.closed.load(Ordering::SeqCst) {
-            return Ok(Interception::Pass(request));
-        }
-        let id = packets.len();
-        let packet = Arc::new(Packet {
-            action,
-            task: Mutex::new(None),
-            receipt: Mutex::new(json!({
-                "id": id, "session_nonce": state.scope.nonce, "receiver": target.to_string(),
-                "authority": request.uri().authority().map(|value| value.as_str()),
-                "captured_at_ms": millis(&state), "completed": false, "forwarded": false,
-                "backend_complete": false, "outcome": "held", "request_bytes": bytes.len(),
-            "request_sha256": hash(&bytes), "request_frame_hex": hex(&bytes), "request_headers_sha256": header_hash(request.headers()),
-                "request_trailers_sha256": trailers.as_ref().map(header_hash),
-            "request_trailers": trailer_receipt(trailers.as_ref()),
-                "auth_nonce": header("x-rustfs-rpc-nonce"), "replay_nonce": header("x-rustfs-rpc-replay-nonce"),
-                "signature_sha256": header("x-rustfs-rpc-signature-v2").map(|value| hash(value.as_bytes())),
-                "replay_signature_sha256": header("x-rustfs-rpc-signature-v3").map(|value| hash(value.as_bytes())),
-                "request": {"owner_id": decoded.owner_id, "instance_id": decoded.instance_id,
-                    "probe_only": decoded.probe_only, "entries": decoded.entries.iter().map(|entry| json!({
-                        "bucket": entry.bucket, "bucket_incarnation": uuid::Uuid::from_slice(&entry.bucket_incarnation).ok().map(|value| value.to_string()),
-                        "generation": entry.generation })).collect::<Vec<_>>() }
-            })),
-        });
-        packets.push(packet.clone());
-        packet
-    };
-    let (reply, response) = oneshot::channel();
-    let worker_packet = packet.clone();
-    let worker_state = state.clone();
-    // This owner survives an HTTP/2 stream reset from the original 5 s caller.
-    // Every request is settled once and is discarded at the session deadline.
-    let task = tokio::spawn(async move {
-        let result = tokio::time::timeout_at(worker_state.deadline, async {
-            let fault = loop {
-                if let Some(fault) = *selected.borrow_and_update() {
-                    break fault;
-                }
-                selected
-                    .changed()
-                    .await
-                    .map_err(|_| io::Error::other("scoped ACK queue closed"))?;
-            };
-            if fault == ScopedAckFault::RequestLoss {
-                worker_packet.receipt.lock().expect("scoped ACK receipt")["outcome"] = json!("request_lost");
-                return Err(io::Error::new(io::ErrorKind::ConnectionAborted, "injected scoped ACK request loss"));
-            }
-            if worker_state.closed.load(Ordering::SeqCst) || Instant::now() >= worker_state.deadline {
-                return Err(io::Error::new(
-                    io::ErrorKind::ConnectionAborted,
-                    "scoped ACK session ended or expired before forwarding",
-                ));
-            }
-            {
-                let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
-                receipt["forwarded"] = json!(true);
-                receipt["forwarded_at_ms"] = json!(millis(&worker_state));
-            }
-            let backend = forward_to_backend(request, client, target).await?;
-            let (parts, body) = backend.into_parts();
-            let collected = Limited::new(body, MAX_BYTES).collect().await.map_err(io::Error::other)?;
-            let trailers = collected.trailers().cloned();
-            let bytes = collected.to_bytes();
-            let grpc_status = trailers
-                .as_ref()
-                .and_then(|values| values.get("grpc-status"))
-                .or_else(|| parts.headers.get("grpc-status"))
-                .and_then(|value| value.to_str().ok());
-            let response = unary_payload(&bytes)
-                .ok()
-                .and_then(|payload| ScannerScopedDirtyUsageAckResponse::decode(payload).ok())
-                .map(|response| {
-                    json!({"owner_id": response.owner_id, "instance_id": response.instance_id,
-                    "supported": response.supported, "cleared": response.cleared,
-                    "response_proof_bytes": response.response_proof.len()})
-                });
-            {
-                let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
-                receipt["backend_complete"] = json!(true);
-                receipt["backend_completed_at_ms"] = json!(millis(&worker_state));
-                receipt["response"] = json!({"status": parts.status.as_u16(), "grpc_status": grpc_status,
-                    "bytes": bytes.len(), "sha256": hash(&bytes), "frame_hex": hex(&bytes), "headers_sha256": header_hash(&parts.headers),
-                    "trailers": trailer_receipt(trailers.as_ref()),
-                    "trailers_sha256": trailers.as_ref().map(header_hash), "ack": response});
-                receipt["outcome"] = json!(if fault == ScopedAckFault::ResponseLoss {
-                    "response_lost"
-                } else {
-                    "forwarded"
-                });
-            }
-            if fault == ScopedAckFault::ResponseLoss {
-                return Err(io::Error::new(
-                    io::ErrorKind::ConnectionAborted,
-                    "injected scoped ACK response loss after backend trailers",
-                ));
-            }
-            Ok(Response::from_parts(parts, restored_body(bytes, trailers)))
-        })
-        .await
-        .unwrap_or_else(|_| Err(io::Error::new(io::ErrorKind::TimedOut, "scoped ACK retention expired")));
-        {
-            let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
-            if result.is_err() && receipt["outcome"] == "held" {
-                receipt["outcome"] = json!("transport_failed_or_expired");
-            }
-            receipt["completed"] = json!(true);
-            receipt["completed_at_ms"] = json!(millis(&worker_state));
-        }
-        let _ = reply.send(result);
-        worker_state.changed.notify_one();
-    });
-    *packet.task.lock().expect("scoped ACK task") = Some(task.abort_handle());
-    if state.closed.load(Ordering::SeqCst) {
-        task.abort();
-    }
-    drop(task);
-    state.changed.notify_one();
-    response
-        .await
-        .map_err(|_| io::Error::new(io::ErrorKind::ConnectionAborted, "scoped ACK queue ended"))?
-        .map(Interception::Handled)
-}

crates/e2e_test/src/storage_api.rs

@@ -70,9 +70,6 @@ pub(crate) mod multipart_rpc_delay {
 #[cfg(test)]
 pub(crate) mod multipart_fence_acceptance {
     pub(crate) use super::RUSTFS_META_MULTIPART_BUCKET;
-
-    /// Targeted lock diagnostics for the metadata-delay quorum case.
-    pub(crate) const LOCK_DIAGNOSTICS_FILTER: &str = "warn,rustfs_lock::distributed_lock=debug,rustfs_ecstore::object_lock_diag=trace,rustfs_ecstore::cluster::rpc::remote_locker=info";
 }
 
 #[cfg(test)]
@@ -83,23 +80,3 @@ pub(crate) mod multipart_source_rpc {
         PublicationIdentity, SourceGeneration, SourceGeometry, SourceLineage,
     };
 }
-
-#[cfg(test)]
-pub(crate) mod scanner_peer_bucket {
-    use rustfs_ecstore::api::disk as ecstore_disk;
-    use rustfs_ecstore::api::erasure as ecstore_erasure;
-    use rustfs_ecstore::api::rpc as ecstore_rpc;
-
-    pub(crate) use ecstore_disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
-    pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
-    pub(crate) use ecstore_rpc::PeerRestClient;
-}
-
-/// Storage handles used to seed a pre-validation bucket record while the fixture server is stopped.
-#[cfg(test)]
-pub(crate) mod bucket_tag_fixture {
-    pub(crate) use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
-    pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools};
-    pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
-    pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks_with_instance_ctx};
-}

crates/e2e_test/src/sts_query_compat_test.rs

@@ -360,26 +360,12 @@ impl OpaMock {
 
     pub(crate) async fn expect_bucket_tags(&mut self, action: &str, bucket: &str, department: Option<&str>) -> TestResult {
         let payload = self.next_request().await?;
-        Self::assert_bucket_tags(&payload, action, bucket, department);
-        Ok(())
-    }
-
-    async fn expect_versioned_tag_permission(&mut self, bucket: &str, object: &str, version: &str) -> TestResult {
-        let payload = self.next_request().await?;
-        Self::assert_bucket_tags(&payload, "s3:GetObjectVersionTagging", bucket, Some("finance"));
-        assert_eq!(payload["input"]["resource"]["object"], object);
-        assert_eq!(payload["input"]["context"]["conditions"]["versionid"], serde_json::json!([version]));
-        Ok(())
-    }
-
-    fn assert_bucket_tags(payload: &Value, action: &str, bucket: &str, department: Option<&str>) {
         let input = &payload["input"];
-        assert_eq!(input["identity"]["account"], "opabuckettags");
-        assert_eq!(input["identity"]["is_owner"], false);
         assert_eq!(input["action"], action);
         assert_eq!(input["resource"]["bucket"], bucket);
         let expected = department.map(|value| serde_json::json!([value]));
         assert_eq!(input["context"]["conditions"].get("ExistingBucketTag/department"), expected.as_ref());
+        Ok(())
     }
 
     async fn wait_for_validation(&mut self) -> TestResult {
@@ -692,61 +678,22 @@ async fn test_list_buckets_opa_contract() -> TestResult {
     Ok(())
 }
 
-/// Stop and reap the directory's owning server before opening the fixture store.
-async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, bucket: &str) -> TestResult {
-    use crate::storage_api::bucket_tag_fixture::{
-        DisksLayout, ECStore, EndpointServerPools, InstanceContext, init_local_disks_with_instance_ctx, load_bucket_metadata,
-    };
-
-    let server = env
-        .process
-        .as_mut()
-        .ok_or("missing server process before offline fixture write")?;
-    server.kill()?;
-    server.wait()?;
-    env.process = None;
-    let layout = DisksLayout::from_volumes(std::slice::from_ref(&env.temp_dir))?;
-    let (pools, _) = EndpointServerPools::create_server_endpoints(&env.address, &layout).await?;
-    let context = std::sync::Arc::new(InstanceContext::new());
-    init_local_disks_with_instance_ctx(&context, pools.clone()).await?;
-    let store = ECStore::new_with_instance_ctx(env.address.parse()?, pools, Default::default(), context.clone()).await?;
-    let _cancel = context
-        .background_cancel_token()
-        .expect("store binds its shutdown token")
-        .drop_guard();
-    let mut metadata = load_bucket_metadata(store.clone(), bucket).await?;
-    let incarnation = metadata.bucket_incarnation_id;
-    let policy = metadata.policy_config_json.clone();
-    assert!(!incarnation.is_nil());
-    assert!(!policy.is_empty(), "the bucket-policy Allow must survive legacy seeding");
-    // Encode the duplicate tag set with the same bucket record and erasure writer used by the store.
-    let xml = b"<Tagging><TagSet><Tag><Key>department</Key><Value>finance</Value></Tag><Tag><Key>department</Key><Value>engineering</Value></Tag></TagSet></Tagging>";
-    metadata.tagging_config_xml = xml.to_vec();
-    metadata.save_with_store(store.clone()).await?;
-    let persisted = load_bucket_metadata(store, bucket).await?;
-    assert_eq!(persisted.bucket_incarnation_id, incarnation);
-    assert_eq!(persisted.policy_config_json, policy);
-    assert_eq!(persisted.tagging_config_xml.as_slice(), xml);
-    Ok(())
-}
-
 #[tokio::test]
 async fn test_bucket_tags_opa_contract() -> TestResult {
     use aws_sdk_s3::primitives::ByteStream;
     use aws_sdk_s3::types::{
-        BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, TaggingDirective, VersioningConfiguration,
+        BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
     };
 
     init_logging();
     let mut opa = OpaMock::start().await?;
     let mut env = RustFSTestEnvironment::new().await?;
-    let opa_url = opa.url.clone();
-    let server_env = [
-        ("RUSTFS_POLICY_PLUGIN_URL", opa_url.as_str()),
+    env.start_rustfs_server_without_cleanup_with_env(&[
+        ("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
         ("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
         ("NO_PROXY", "127.0.0.1,localhost"),
-    ];
-    env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
+    ])
+    .await?;
     let secret = uuid::Uuid::new_v4().to_string();
     create_user(&env, "opabuckettags", &secret).await?;
     let client = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, "opabuckettags", &secret, None, "e2e-bucket-tags"));
@@ -786,7 +733,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
         .await?;
     assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
     opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-    let head = client
+    client
         .head_object()
         .bucket(source)
         .key("report")
@@ -794,8 +741,6 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
         .send()
         .await?;
     opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-    assert_eq!(head.tag_count(), Some(1));
-    opa.expect_versioned_tag_permission(source, "report", version).await?;
     client.list_objects_v2().bucket(source).send().await?;
     opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?;
     client.list_object_versions().bucket(source).send().await?;
@@ -805,65 +750,28 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     admin.create_bucket().bucket(destination).send().await?;
     set_department(&admin, destination, "engineering").await?;
     let copy_source = format!("{source}/report?versionId={version}");
-    for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
-        let error = client
-            .copy_object()
-            .bucket(destination)
-            .key("copied")
-            .copy_source(&copy_source)
-            .set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
-            .set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
-            .send()
-            .await
-            .expect_err("source tags must not authorize a different destination");
-        assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
-        opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-        if directive == TaggingDirective::Copy {
-            opa.expect_versioned_tag_permission(source, "report", version).await?;
-        }
-        opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
-            .await?;
-        let missing = admin
-            .head_object()
-            .bucket(destination)
-            .key("copied")
-            .send()
-            .await
-            .expect_err("denied COPY and REPLACE must leave no destination object");
-        assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
-    }
+    let error = client
+        .copy_object()
+        .bucket(destination)
+        .key("copied")
+        .copy_source(&copy_source)
+        .send()
+        .await
+        .expect_err("source tags must not authorize a different destination");
+    assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
+    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+    opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
+        .await?;
     set_department(&admin, destination, "finance").await?;
-    for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
-        client
-            .copy_object()
-            .bucket(destination)
-            .key("copied")
-            .copy_source(&copy_source)
-            .set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
-            .set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
-            .send()
-            .await?;
-        opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-        // The access hook checks preliminary tags, then the copy use case checks
-        // the exact source reader's tags before publishing the destination.
-        for _ in 0..2 {
-            if directive == TaggingDirective::Copy {
-                opa.expect_versioned_tag_permission(source, "report", version).await?;
-            }
-            opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
-        }
-        let tags = admin.get_object_tagging().bucket(destination).key("copied").send().await?;
-        assert_eq!(tags.tag_set().len(), 1);
-        assert_eq!(tags.tag_set()[0].key(), "department");
-        assert_eq!(
-            tags.tag_set()[0].value(),
-            if directive == TaggingDirective::Copy {
-                "engineering"
-            } else {
-                "legal"
-            }
-        );
-    }
+    client
+        .copy_object()
+        .bucket(destination)
+        .key("copied")
+        .copy_source(&copy_source)
+        .send()
+        .await?;
+    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+    opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
 
     let upload = client
         .create_multipart_upload()
@@ -939,14 +847,6 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
     opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("engineering"))
         .await?;
-    let missing = admin
-        .head_object()
-        .bucket(destination)
-        .key("denied-source")
-        .send()
-        .await
-        .expect_err("a denied source must leave no destination object");
-    assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
     set_department(&client, source, "finance")
         .await
         .expect_err("requested tags must not authorize their own mutation");
@@ -985,7 +885,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
     opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering"))
         .await?;
-    let invalid_write = admin
+    admin
         .put_bucket_tagging()
         .bucket(destination)
         .tagging(
@@ -995,25 +895,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
                 .build()?,
         )
         .send()
-        .await
-        .expect_err("new duplicate tags must be rejected before publication");
-    assert_eq!(invalid_write.as_service_error().and_then(ProvideErrorMetadata::code), Some("InvalidTag"));
-    assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400));
-
-    set_department(&admin, source, "finance").await?;
-    seed_legacy_duplicate_bucket_tags(&mut env, destination).await?;
-    env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
-    // Prove this process can authorize the same caller and read a healthy bucket's persisted tags.
-    let read = client
-        .get_object()
-        .bucket(source)
-        .key("report")
-        .version_id(version)
-        .send()
         .await?;
-    assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
-    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-
     let error = client
         .get_object()
         .bucket(destination)

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -1112,9 +1112,8 @@ async fn wait_for_upgrade_writes(
             match response {
                 Ok(Ok(_)) => break,
                 Ok(Err(error)) => {
-                    let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
                     if error.raw_response().map(|response| response.status().as_u16()) != Some(503)
-                        || !matches!(code, Some("ServiceUnavailable" | "SlowDown" | "SlowDownRead"))
+                        || error.as_service_error().and_then(ProvideErrorMetadata::code) != Some("ServiceUnavailable")
                     {
                         return Err(format!("{phase}: node {node} {probe} failed: {error:?}").into());
                     }

crates/ecstore/Cargo.toml

@@ -31,7 +31,6 @@ workspace = true
 
 [features]
 default = []
-fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
 diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
 # Sampled wall/CPU diagnostics only; does not change the read backend.
 diagnostic-local-batch = ["rustix/time"]
@@ -140,6 +139,7 @@ rustfs-lifecycle.workspace = true
 rustfs-s3-client = { workspace = true }
 rustfs-s3-types = { workspace = true }
 rustfs-data-usage.workspace = true
+rustfs-gateway-types.workspace = true
 rustfs-object-capacity.workspace = true
 rustfs-object-data-cache = { workspace = true, features = ["runtime-memory"] }
 arc-swap.workspace = true
@@ -158,7 +158,6 @@ serde = { workspace = true, features = ["derive"] }
 time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
 bytesize.workspace = true
 serde_json = { workspace = true, features = ["raw_value"] }
-s3s = { workspace = true, features = ["minio"] }
 http.workspace = true
 opentelemetry.workspace = true
 http-body = { workspace = true }
@@ -226,11 +225,9 @@ metrics = { workspace = true }
 # The guard scripts/check_no_tokio_io_uring.sh allows an explicit
 # io-uring integration; only the tokio "io-uring" runtime feature is banned.
 [target.'cfg(target_os = "linux")'.dependencies]
-# Native completion owners keep physical I/O leases alive after cancellation.
-rustfs-uring = { git = "https://github.com/rustfs/uring.git", rev = "eb8fc160e71d804427de1e905f5c226d580f71bf" }
-
-[target.'cfg(target_os = "linux")'.dev-dependencies]
-rustfs-uring = { git = "https://github.com/rustfs/uring.git", rev = "eb8fc160e71d804427de1e905f5c226d580f71bf", features = ["fault-injection"] }
+# SharedReadBudget and the bounded read APIs are available in the 0.3.0
+# registry release; keep the application on the published contract.
+rustfs-uring = "0.3.0"
 
 [target.'cfg(windows)'.dependencies]
 winapi-util.workspace = true
@@ -249,6 +246,7 @@ windows-sys = { workspace = true, features = [
 windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
 
 [dev-dependencies]
+s3s = { workspace = true, features = ["minio"] }
 aws-smithy-async.workspace = true
 rustfs-s3-ops.workspace = true
 tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }

crates/ecstore/src/api/mod.rs

@@ -66,6 +66,8 @@ pub mod bucket {
         }
 
         pub mod bucket_lifecycle_ops {
+            pub use rustfs_lifecycle::{RestoreOutputLocation, RestoreRequest, RestoreS3Location, RestoreTag, RestoreTagging};
+
             pub use crate::bucket::lifecycle::bucket_lifecycle_ops::{
                 ExpiryState, LifecycleOps, ManualTransitionCancelCheck, ManualTransitionProgressSink,
                 ManualTransitionQueueSnapshot, ManualTransitionRunExecution, ManualTransitionRunOptions,
@@ -137,7 +139,7 @@ pub mod bucket {
             pub use crate::bucket::lifecycle::lifecycle::{
                 Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate,
                 ObjectOpts, RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time,
-                object_opts_from_object_info,
+                format_lifecycle_date, object_opts_from_object_info, parse_lifecycle_date,
             };
         }
 
@@ -192,7 +194,6 @@ pub mod bucket {
             table_catalog_path_hash, unreadable_config_refusal,
         };
         pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
-        pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, validate_tag_set_entries};
     }
 
     pub mod config_parse_mode {
@@ -236,7 +237,7 @@ pub mod bucket {
     }
 
     pub mod object_lock {
-        pub use crate::bucket::object_lock::{ObjectLockApi, ObjectLockStatusExt};
+        pub use crate::bucket::object_lock::ObjectLockApi;
 
         pub mod types {
             pub use crate::bucket::object_lock::types::{
@@ -331,8 +332,8 @@ pub mod bucket {
 
     pub mod utils {
         pub use crate::bucket::utils::{
-            check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
-            check_valid_bucket_name_strict, deserialize, has_bad_path_component, has_xml_invalid_control_characters,
+            BucketConfigXml, BucketConfigXmlError, check_bucket_and_object_names, check_list_objs_args,
+            check_object_name_for_length_and_slash, check_valid_bucket_name_strict, deserialize, has_bad_path_component,
             is_meta_bucketname, is_valid_object_prefix, serialize,
         };
     }
@@ -360,7 +361,7 @@ pub mod capacity {
 }
 
 pub mod object_api_utils {
-    pub use crate::object_api::object_api_utils::{ObjReaderFn, PutObjReader, get_raw_etag, new_getobjectreader, to_s3s_etag};
+    pub use crate::object_api::object_api_utils::{ObjReaderFn, PutObjReader, get_raw_etag, new_getobjectreader};
 }
 
 pub mod cluster {
@@ -448,19 +449,10 @@ pub mod data_usage {
 }
 
 pub mod disk {
-    pub use crate::cluster::rpc::background_pipe::{BackgroundPipeReader, BackgroundPipeWriter, background_pipe};
-    pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
-    pub use crate::disk::background_memory::BackgroundEncodeBuffer;
     pub use crate::disk::disk_store::get_object_disk_read_timeout;
-    #[cfg(feature = "test-util")]
-    pub use crate::disk::disk_store::wait_for_multipart_index_bootstrap;
     pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
     #[cfg(feature = "test-util")]
     pub use crate::disk::local::{local_durability_modes, set_local_rename_data_fail_after_metadata_commit};
-    pub use crate::disk::multipart_index::{
-        MULTIPART_INDEX_RPC_MAX_BYTES, MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES, MultipartIndexPage, MultipartIndexRpcReply,
-        MultipartIndexRpcRequest,
-    };
     pub use crate::disk::multipart_source::{
         MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind,
         MultipartAbortRound, MultipartSourceOperation, MultipartSourcePhase, MultipartSourceRequest, MultipartSourceResponse,
@@ -473,9 +465,7 @@ pub mod disk {
     pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
     pub use crate::disk::{
         BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
-        BoundedDeleteRecoveryDiscovery, BoundedDeleteRecoveryInspection, BoundedDeleteRecoverySettlement,
-        BoundedDeleteRecoverySettlementReply, CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration,
-        CurrentDeleteTransition, DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, DELETE_RECOVERY_MAX_INSPECTION_ENTRIES,
+        CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
         DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
         DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
         Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
@@ -487,10 +477,6 @@ pub mod disk {
         UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
         validate_batch_read_version_item_count,
     };
-    pub use crate::disk::{
-        BackgroundMemoryBudget, PhysicalDiskBinding, PhysicalDiskPlan, PhysicalDiskSample, PhysicalDiskSampler,
-        PhysicalResourcePlan, RemoteBackgroundIo, with_background_resource_unit, with_remote_background_io,
-    };
     pub use bytes::Bytes;
     pub use endpoint::Endpoint;
     pub use error::DiskError;
@@ -626,12 +612,11 @@ pub mod rebalance {
 pub mod rio {
     pub use crate::io_support::rio::{
         DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
-        boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
+        boxed_reader, compression_metadata_value, wrap_reader,
     };
 }
 
 pub mod rpc {
-    pub use crate::cluster::rpc::http_auth::with_rpc_background_request;
     pub use crate::cluster::rpc::{
         AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
         NetworkPeerProbeClient, NetworkPeerProbeError, NetworkPeerProbeMeasurement, NetworkPeerTarget, PEER_RESTDRY_RUN,
@@ -641,8 +626,8 @@ pub mod rpc {
         TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce,
         decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers,
         gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
-        node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, rpc_background_admission,
-        set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
+        node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
+        sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
         sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
         tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
         verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
@@ -663,9 +648,7 @@ pub mod set_disk {
 
     #[cfg(feature = "test-util")]
     pub mod test_util {
-        pub use crate::bucket::quota::reservation::{
-            fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test, retry_quota_reconciliation_for_test,
-        };
+        pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
         pub use crate::set_disk::ConditionalReadBenchmarkMetadataGuard;
         pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
 
@@ -694,9 +677,8 @@ pub mod storage {
     pub use crate::store::HealWalkVersion;
     pub use crate::store::{
         BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
-        all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc, init_local_disks,
-        init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
-        prewarm_local_disk_id_map_with_instance_ctx,
+        all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
+        prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
     };
     pub use crate::store::{HealObjectAbsenceProof, HealObjectStorageResult};
 }

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -16,8 +16,7 @@ use crate::bucket::metadata::BucketMetadata;
 use crate::bucket::metadata_sys::get_bucket_targets_config;
 use crate::bucket::metadata_sys::get_replication_config;
 use crate::bucket::remote_s3_client::{
-    PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, RemoteS3RetryPolicy,
-    UploadResponseDeadline, build_remote_s3_client,
+    PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
 };
 use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
 use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
@@ -120,7 +119,6 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
             ca_cert_pem: (!target.ca_cert_pem.trim().is_empty()).then(|| target.ca_cert_pem.clone()),
             connect_timeout: None,
             read_timeout: None,
-            operation_timeout: Some(TargetRequestTimeouts::from_env().request),
             // Replication has no retry budget of its own on the request path,
             // so it keeps the SDK's standard three attempts; stating it here
             // pins the behaviour to this line instead of an SDK default.
@@ -1193,7 +1191,6 @@ impl BucketTargetSys {
             health_check_duration: target.health_check_duration,
             replicate_sync: target.replication_sync,
             client: Arc::new(client),
-            timeouts: TargetRequestTimeouts::from_env(),
         })
     }
 
@@ -1925,83 +1922,6 @@ impl std::fmt::Display for S3ClientError {
     }
 }
 
-/// Replication request budgets, snapshotted when a target client is built.
-/// Streamed GET bodies are not subject to the upload-response budget.
-#[derive(Debug, Clone, Copy)]
-pub struct TargetRequestTimeouts {
-    request: Duration,
-    complete: Duration,
-    minimum_transfer_rate: u64,
-}
-
-impl Default for TargetRequestTimeouts {
-    fn default() -> Self {
-        Self {
-            request: Duration::from_secs(60),
-            complete: Duration::from_secs(15 * 60),
-            minimum_transfer_rate: 1024 * 1024,
-        }
-    }
-}
-
-impl TargetRequestTimeouts {
-    fn from_env() -> Self {
-        let defaults = Self::default();
-        let seconds = |name: &str, fallback: Duration| {
-            let value = rustfs_utils::get_env_u64(name, fallback.as_secs());
-            Duration::from_secs(if (1..=86400).contains(&value) {
-                value
-            } else {
-                fallback.as_secs()
-            })
-        };
-        let minimum_transfer_rate =
-            rustfs_utils::get_env_u64("RUSTFS_REPLICATION_MIN_TRANSFER_RATE_BYTES_PER_SEC", defaults.minimum_transfer_rate);
-        Self {
-            request: seconds("RUSTFS_REPLICATION_REQUEST_TIMEOUT_SECS", defaults.request),
-            complete: seconds("RUSTFS_REPLICATION_COMPLETE_TIMEOUT_SECS", defaults.complete),
-            minimum_transfer_rate: if minimum_transfer_rate == 0 {
-                defaults.minimum_transfer_rate
-            } else {
-                minimum_transfer_rate
-            },
-        }
-    }
-
-    fn upload_timeout(self, size: i64) -> Result<Duration, S3ClientError> {
-        let size = u64::try_from(size).map_err(|_| S3ClientError::new("replication upload size must be non-negative"))?;
-        // The SDK can make three attempts. Leave transmission headroom for
-        // all three; the separate response deadline still bounds a trickling
-        // target after each upload finishes.
-        let attempts = match REPLICATION_TARGET_RETRY_POLICY {
-            RemoteS3RetryPolicy::Disabled => 1,
-            RemoteS3RetryPolicy::Standard { max_attempts } => u64::from(max_attempts.max(1)),
-        };
-        let seconds = size
-            .div_ceil(self.minimum_transfer_rate)
-            .checked_mul(attempts)
-            .ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
-        let timeout = self
-            .request
-            .checked_add(Duration::from_secs(seconds))
-            .ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
-        if Instant::now().checked_add(timeout).is_none() {
-            return Err(S3ClientError::new("replication upload timeout exceeds supported duration"));
-        }
-        Ok(timeout)
-    }
-
-    fn upload_override(self, size: i64) -> Result<aws_sdk_s3::config::Builder, S3ClientError> {
-        Ok(aws_sdk_s3::config::Builder::new()
-            .timeout_config(
-                aws_smithy_types::timeout::TimeoutConfig::builder()
-                    .operation_timeout(self.upload_timeout(size)?)
-                    .build(),
-            )
-            .interceptor(UploadResponseDeadline(self.request)))
-    }
-}
-
 #[derive(Debug)]
 pub struct TargetClient {
     pub endpoint: String,
@@ -2015,7 +1935,6 @@ pub struct TargetClient {
     pub health_check_duration: Duration,
     pub replicate_sync: bool,
     pub client: Arc<S3Client>,
-    pub timeouts: TargetRequestTimeouts,
 }
 
 impl TargetClient {
@@ -2441,11 +2360,12 @@ impl TargetClient {
 
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             });
-        let mut config = self.timeouts.upload_override(size)?;
         if forwards_source_checksum {
-            config = config.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired);
+            operation = operation.config_override(
+                aws_sdk_s3::config::Builder::new()
+                    .request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
+            );
         }
-        operation = operation.config_override(config);
         match operation.send().await {
             Ok(output) => {
                 // Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
@@ -2581,7 +2501,6 @@ impl TargetClient {
                 }
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             })
-            .config_override(self.timeouts.upload_override(size)?)
             .send()
             .await
         {
@@ -2619,13 +2538,6 @@ impl TargetClient {
                 }
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             })
-            .config_override(
-                aws_sdk_s3::config::Builder::new().timeout_config(
-                    aws_smithy_types::timeout::TimeoutConfig::builder()
-                        .operation_timeout(self.timeouts.complete)
-                        .build(),
-                ),
-            )
             .send()
             .await
         {
@@ -3054,241 +2966,11 @@ mod tests {
                 health_check_duration: Duration::from_secs(5),
                 replicate_sync: false,
                 client: Arc::new(client),
-                timeouts: TargetRequestTimeouts::default(),
             },
             request_headers,
         )
     }
 
-    #[derive(Debug, Clone)]
-    struct TimedResponseConnector {
-        upload_delay: Duration,
-        response_delay: Duration,
-        prefix_bytes: usize,
-        response: bytes::Bytes,
-        frames: Arc<std::sync::atomic::AtomicUsize>,
-        dropped: Arc<std::sync::atomic::AtomicUsize>,
-    }
-
-    struct ResponseDrop(Arc<std::sync::atomic::AtomicUsize>);
-    impl Drop for ResponseDrop {
-        fn drop(&mut self) {
-            self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-        }
-    }
-
-    impl SmithyHttpConnector for TimedResponseConnector {
-        fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
-            let settings = self.clone();
-            HttpConnectorFuture::new(async move {
-                http_body_util::BodyExt::collect(request.into_body())
-                    .await
-                    .map_err(aws_smithy_runtime_api::client::result::ConnectorError::io)?;
-                tokio::time::sleep(settings.upload_delay).await;
-                let guard = ResponseDrop(Arc::clone(&settings.dropped));
-                let stream = futures::stream::unfold(
-                    (settings.prefix_bytes, Some(settings.response), true, guard),
-                    move |(remaining, mut body, first, guard)| {
-                        let frames = Arc::clone(&settings.frames);
-                        async move {
-                            if remaining == 0 && body.is_none() {
-                                return None;
-                            }
-                            if !first {
-                                tokio::time::sleep(settings.response_delay).await;
-                            }
-                            let bytes = if remaining > 0 {
-                                bytes::Bytes::from_static(b" ")
-                            } else {
-                                body.take()?
-                            };
-                            frames.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-                            Some((
-                                Ok::<_, std::io::Error>(http_body::Frame::data(bytes)),
-                                (remaining.saturating_sub(1), body, false, guard),
-                            ))
-                        }
-                    },
-                )
-                .fuse();
-                let body = SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream));
-                let mut response =
-                    HttpResponse::new(aws_smithy_runtime_api::http::StatusCode::try_from(200u16).expect("status"), body);
-                response.headers_mut().insert("etag", "\"test-etag\"");
-                Ok(response)
-            })
-        }
-    }
-
-    fn timed_response_target(
-        upload_delay: Duration,
-        response_delay: Duration,
-        prefix_bytes: usize,
-        response: &'static [u8],
-    ) -> (TargetClient, Arc<std::sync::atomic::AtomicUsize>, Arc<std::sync::atomic::AtomicUsize>) {
-        let frames = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        let dropped = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        let connector = SharedHttpConnector::new(TimedResponseConnector {
-            upload_delay,
-            response_delay,
-            prefix_bytes,
-            response: bytes::Bytes::from_static(response),
-            frames: Arc::clone(&frames),
-            dropped: Arc::clone(&dropped),
-        });
-        let http_client = http_client_fn(move |_settings, _components| connector.clone());
-        let (mut target, _) = header_recording_target_client(Vec::new());
-        target.timeouts = TargetRequestTimeouts {
-            request: Duration::from_secs(2),
-            complete: Duration::from_secs(10),
-            minimum_transfer_rate: 1024 * 1024,
-        };
-        target.client = Arc::new(S3Client::from_conf(
-            target
-                .client
-                .config()
-                .to_builder()
-                .http_client(http_client)
-                .retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
-                .timeout_config(
-                    aws_smithy_types::timeout::TimeoutConfig::builder()
-                        .operation_timeout(target.timeouts.request)
-                        .build(),
-                )
-                .build(),
-        ));
-        (target, frames, dropped)
-    }
-
-    #[test]
-    fn replication_upload_budgets_preserve_large_transfers_and_reject_invalid_sizes() {
-        let policy = TargetRequestTimeouts::default();
-        assert_eq!(policy.upload_timeout(0).expect("empty upload"), Duration::from_secs(60));
-        assert_eq!(
-            policy.upload_timeout(5 * 1024 * 1024 * 1024).expect("maximum single PUT"),
-            Duration::from_secs(15420)
-        );
-        assert!(policy.upload_timeout(-1).is_err());
-        let slow = TargetRequestTimeouts {
-            minimum_transfer_rate: 1,
-            ..policy
-        };
-        assert!(slow.upload_timeout(i64::MAX).is_err());
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn replication_upload_response_deadline_is_independent_of_size_and_checksum_override() {
-        for operation in ["put", "forwarded-checksum", "part"] {
-            let (target, frames, dropped) = timed_response_target(Duration::from_secs(4), Duration::from_millis(500), 100, b"");
-            let body = vec![b'x'; 2 * 1024 * 1024];
-            let size = i64::try_from(body.len()).expect("size");
-            let started = tokio::time::Instant::now();
-            let error = if operation == "part" {
-                target
-                    .put_object_part(
-                        "bucket",
-                        "key",
-                        "upload",
-                        1,
-                        size,
-                        ByteStream::from(body),
-                        &PutObjectPartOptions::default(),
-                    )
-                    .await
-                    .expect_err("response deadline")
-            } else {
-                let mut opts = PutObjectOptions::default();
-                if operation == "forwarded-checksum" {
-                    opts.user_metadata.insert(
-                        "x-amz-checksum-sha256".to_owned(),
-                        "OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_owned(),
-                    );
-                }
-                target
-                    .put_object("bucket", "key", size, ByteStream::from(body), &opts)
-                    .await
-                    .expect_err("response deadline")
-            };
-            assert!(
-                started.elapsed() >= Duration::from_secs(6),
-                "upload gets its full independent window: {error}"
-            );
-            assert!(
-                started.elapsed() < target.timeouts.upload_timeout(size).expect("budget"),
-                "response must end before the larger total deadline"
-            );
-            assert!(
-                frames.load(std::sync::atomic::Ordering::SeqCst) >= 2,
-                "trickle delivered progress before failure"
-            );
-            assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1, "transport body released");
-        }
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn replication_slow_upload_preserves_transfer_window_and_streaming_get_body() {
-        let (target, _, _) = timed_response_target(Duration::from_secs(4), Duration::from_millis(100), 5, b"");
-        let body = vec![b'x'; 2 * 1024 * 1024];
-        let size = i64::try_from(body.len()).expect("size");
-        target
-            .put_object("bucket", "key", size, ByteStream::from(body), &PutObjectOptions::default())
-            .await
-            .expect("slow upload, prompt response");
-        let (target, _, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), 5, b"payload");
-        let response = target
-            .get_object("bucket", "key", None, None, None, HeaderMap::new())
-            .await
-            .expect("GET headers");
-        let started = tokio::time::Instant::now();
-        let body = response
-            .body
-            .collect()
-            .await
-            .expect("streamed GET remains outside upload deadline")
-            .into_bytes();
-        assert!(started.elapsed() > target.timeouts.request);
-        assert_eq!(&body[..], b"     payload");
-        assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn replication_complete_processing_whitespace_has_a_separate_bounded_window() {
-        const RESULT: &[u8] = b"<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>\"etag\"</ETag></CompleteMultipartUploadResult>";
-        for prefix in [5, 100] {
-            let (target, frames, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), prefix, RESULT);
-            let started = tokio::time::Instant::now();
-            let result = target
-                .complete_multipart_upload(
-                    "bucket",
-                    "key",
-                    "upload",
-                    vec![CompletedPart::builder().part_number(1).e_tag("etag").build()],
-                    &PutObjectOptions::default(),
-                )
-                .await;
-            if prefix == 5 {
-                assert!(result.is_ok(), "legal processing whitespace: {result:?}");
-            } else {
-                assert!(result.is_err());
-                assert!(started.elapsed() <= Duration::from_secs(11));
-            }
-            assert!(started.elapsed() > target.timeouts.request);
-            assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
-            assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
-        }
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn replication_control_operation_deadline_covers_response_trickle() {
-        let (target, frames, dropped) =
-            timed_response_target(Duration::ZERO, Duration::from_millis(500), 100, b"<VersioningConfiguration/>");
-        let started = tokio::time::Instant::now();
-        assert!(target.get_bucket_versioning("bucket").await.is_err());
-        assert!(started.elapsed() <= Duration::from_secs(3));
-        assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
-        assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
-    }
-
     fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
         let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
         let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
@@ -3597,7 +3279,6 @@ mod tests {
                 health_check_duration: Duration::from_secs(5),
                 replicate_sync: false,
                 client: Arc::new(client),
-                timeouts: TargetRequestTimeouts::default(),
             },
             request_uris,
         )
@@ -3910,7 +3591,6 @@ mod tests {
             health_check_duration: Duration::from_secs(5),
             replicate_sync: false,
             client: Arc::new(client),
-            timeouts: TargetRequestTimeouts::default(),
         })
     }
 

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -88,12 +88,14 @@ use rustfs_filemeta::{
     FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatus, RestoreStatusOps, TRANSITION_COMPLETE, get_file_info,
     is_restored_object_on_disk,
 };
+use rustfs_gateway_types::persistence::PersistedObjectLockConfiguration as ObjectLockConfiguration;
+use rustfs_gateway_types::persistence::{PersistedLifecycleConfiguration, PersistedLifecycleFilter};
+use rustfs_lifecycle::{RestoreRequest, RestoreS3Location};
 use rustfs_lock::{NamespaceLockGuard, error::LockError};
 use rustfs_scanner_metrics::metrics::{
     IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics,
 };
 use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::parse_bool};
-use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, ObjectLockConfiguration, RestoreRequest, RestoreRequestType};
 use sha2::{Digest, Sha256};
 use std::any::Any;
 use std::collections::{BTreeMap, HashMap, HashSet};
@@ -300,7 +302,7 @@ impl LifecycleSys {
         Arc::new(Self)
     }
 
-    pub async fn get(&self, bucket: &str) -> Option<BucketLifecycleConfiguration> {
+    pub async fn get(&self, bucket: &str) -> Option<PersistedLifecycleConfiguration> {
         match metadata_boundary::get_lifecycle_config(bucket).await {
             Ok((lc, _)) => Some(lc),
             Err(Error::ConfigNotFound) => None,
@@ -3557,7 +3559,7 @@ async fn stale_upload_lifecycle_due(
         Err(_) => return None,
     };
 
-    let has_size_filter = |filter: &s3s::dto::LifecycleRuleFilter| {
+    let has_size_filter = |filter: &PersistedLifecycleFilter| {
         filter.object_size_greater_than.is_some()
             || filter.object_size_less_than.is_some()
             || filter
@@ -3566,7 +3568,7 @@ async fn stale_upload_lifecycle_due(
                 .is_some_and(|and| and.object_size_greater_than.is_some() || and.object_size_less_than.is_some())
     };
     let needs_size = lifecycle.rules.iter().any(|rule| {
-        rule.status.as_str() != ExpirationStatus::DISABLED
+        rule.status.as_str() != "Disabled"
             && rule.abort_incomplete_multipart_upload.is_some()
             && rule.filter.as_ref().is_some_and(has_size_filter)
     });
@@ -4000,7 +4002,7 @@ pub fn init_background_stale_multipart_upload_cleanup(api: Arc<ECStore>) {
     });
 }
 
-pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Result<(), std::io::Error> {
+pub async fn validate_transition_tier(lc: &PersistedLifecycleConfiguration) -> Result<(), std::io::Error> {
     for rule in &lc.rules {
         if let Some(transitions) = &rule.transitions {
             for transition in transitions {
@@ -4117,8 +4119,9 @@ pub(crate) async fn enqueue_immediate_expiry(api: Arc<ECStore>, oi: &ObjectInfo,
         .map(lifecycle::object_opts_from_object_info)
         .collect::<Vec<ObjectOpts>>();
     let lock_config = configs.object_lock;
+    let evaluator_lock_config = lock_config.clone();
     let events = match Evaluator::new(lifecycle)
-        .with_lock_retention(lock_config.clone())
+        .with_lock_retention(evaluator_lock_config)
         .eval(&object_opts)
         .await
     {
@@ -4598,19 +4601,26 @@ pub async fn enqueue_transition_for_existing_objects_scoped(
     }
 }
 
-fn lifecycle_rule_has_date_expiration(lc: &BucketLifecycleConfiguration, rule_id: &str) -> bool {
+fn lifecycle_rule_has_date_expiration(lc: &PersistedLifecycleConfiguration, rule_id: &str) -> bool {
     lc.rules.iter().any(|rule| {
-        rule.status == ExpirationStatus::from_static(ExpirationStatus::ENABLED)
+        rule.status == "Enabled"
             && rule.id.as_deref() == Some(rule_id)
             && rule.expiration.as_ref().is_some_and(|expiration| expiration.date.is_some())
     })
 }
 
-fn should_defer_date_expiry_for_recent_config_update(lc: &BucketLifecycleConfiguration, now: OffsetDateTime) -> bool {
-    lc.expiry_updated_at.as_ref().is_some_and(|updated_at| {
-        let updated_at = OffsetDateTime::from(updated_at.clone());
-        now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) < DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS
-    })
+fn should_defer_date_expiry_for_recent_config_update(lc: &PersistedLifecycleConfiguration, now: OffsetDateTime) -> bool {
+    lc.expiry_updated_at
+        .as_ref()
+        .is_some_and(|updated_at| match rustfs_lifecycle::parse_lifecycle_date(updated_at) {
+            Ok(updated_at) => {
+                now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) < DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS
+            }
+            Err(error) => {
+                warn!(error = %error, "Deferring date expiry with an unreadable update timestamp");
+                true
+            }
+        })
 }
 
 async fn apply_existing_object_expiry(
@@ -4630,7 +4640,7 @@ async fn apply_existing_object_expiry(
 struct ExistingObjectExpiryContext<'a> {
     api: Arc<ECStore>,
     bucket: &'a str,
-    lc: Arc<BucketLifecycleConfiguration>,
+    lc: Arc<PersistedLifecycleConfiguration>,
     lock_config: Option<Arc<ObjectLockConfiguration>>,
     bucket_incarnation_id: Uuid,
     src: &'a LcEventSrc,
@@ -4650,8 +4660,9 @@ async fn enqueue_expiry_for_existing_object_group(
         .iter()
         .map(lifecycle::object_opts_from_object_info)
         .collect::<Vec<ObjectOpts>>();
+    let evaluator_lock_config = context.lock_config.clone();
     let events = match Evaluator::new(context.lc.clone())
-        .with_lock_retention(context.lock_config.clone())
+        .with_lock_retention(evaluator_lock_config)
         .eval(&object_opts)
         .await
     {
@@ -4845,7 +4856,7 @@ fn manual_transition_version_marker(oi: &ObjectInfo) -> String {
 async fn enqueue_transition_with_lifecycle_report(
     api: Option<Arc<ECStore>>,
     oi: &ObjectInfo,
-    lc: &BucketLifecycleConfiguration,
+    lc: &PersistedLifecycleConfiguration,
     src: &LcEventSrc,
     options: &ManualTransitionRunOptions,
     report: &mut ManualTransitionRunReport,
@@ -4911,7 +4922,7 @@ async fn enqueue_transition_with_lifecycle_report(
     false
 }
 
-async fn enqueue_transition_with_lifecycle(oi: &ObjectInfo, lc: &BucketLifecycleConfiguration, src: &LcEventSrc) -> bool {
+async fn enqueue_transition_with_lifecycle(oi: &ObjectInfo, lc: &PersistedLifecycleConfiguration, src: &LcEventSrc) -> bool {
     let event = lc.eval(&oi.to_lifecycle_opts()).await;
     match event.action {
         IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
@@ -5425,12 +5436,8 @@ pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) ->
     })
 }
 
-fn select_restore_s3_location(rreq: &RestoreRequest) -> Result<Option<&s3s::dto::S3Location>, std::io::Error> {
-    if rreq
-        .type_
-        .as_ref()
-        .is_none_or(|type_| type_.as_str() != RestoreRequestType::SELECT)
-    {
+fn select_restore_s3_location(rreq: &RestoreRequest) -> Result<Option<&RestoreS3Location>, std::io::Error> {
+    if rreq.type_.as_ref().is_none_or(|type_| type_.as_str() != "SELECT") {
         return Ok(None);
     }
     let output_location = rreq
@@ -5472,35 +5479,25 @@ pub async fn put_restore_opts(
     // the OutputLocation metadata instead of a write to `OutputLocation.S3`
     // (backlog#1341). RestoreObject rejects SELECT at the API boundary; this
     // is the fail-closed backstop for any other caller.
-    if rreq
-        .type_
-        .as_ref()
-        .is_some_and(|type_| type_.as_str() == RestoreRequestType::SELECT)
-    {
+    if rreq.type_.as_ref().is_some_and(|type_| type_.as_str() == "SELECT") {
         return Err(std::io::Error::other("SELECT restore requests are not supported"));
     }
     for (k, v) in oi.user_defined.iter() {
         meta.insert(k.to_string(), v.clone());
     }
+    // The fleet capability gate guarantees physical copy-back support. Keep
+    // the completed transition identity for orphan recovery and remote cleanup.
     rustfs_utils::http::metadata_compat::remove_str(&mut meta, rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID);
-    rustfs_utils::http::metadata_compat::remove_str(&mut meta, rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_WORKER_LOCK);
     if !oi.user_tags.is_empty() {
         meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
     }
-    if let Some(expires) = oi.expires {
-        meta.insert(
-            "expires".to_string(),
-            expires
-                .format(&time::format_description::well_known::Rfc3339)
-                .map_err(std::io::Error::other)?,
-        );
-    }
-    let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
+    // Finalization publishes completion and its expiry under the object lock.
+    // The local copy-back commit must not make unfinished work eligible for expiry.
     meta.insert(
         metadata_keys::RESTORE.to_string(),
         RestoreStatus {
-            is_restore_in_progress: Some(false),
-            restore_expiry_date: Some(restore_expiry),
+            is_restore_in_progress: Some(true),
+            restore_expiry_date: None,
         }
         .to_string(),
     );
@@ -5514,6 +5511,8 @@ pub async fn put_restore_opts(
         // computed MD5 is not the object's public plaintext ETag.
         preserve_etag: oi.etag.clone(),
         shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
+        restore_copyback: true,
+        //expires:           oi.expires,
         ..Default::default()
     })
 }
@@ -5532,6 +5531,18 @@ impl LifecycleOps for ObjectInfo {
         if self.transitioned_object.status != lifecycle::TRANSITION_COMPLETE {
             return false;
         }
+        if rustfs_utils::http::metadata_compat::get_consistent_str(
+            &self.user_defined,
+            rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_COPYBACK,
+        ) == Some(rustfs_utils::http::metadata_compat::RESTORE_COPYBACK_PROTOCOL_V1)
+            && self
+                .user_defined
+                .get(metadata_keys::RESTORE)
+                .and_then(|value| rustfs_filemeta::parse_restore_obj_status(value).ok())
+                .is_some_and(|status| status.is_restore_in_progress == Some(true))
+        {
+            return false;
+        }
         !is_restored_object_on_disk(&self.user_defined)
     }
 }
@@ -5543,25 +5554,25 @@ pub trait RestoreRequestOps {
 impl RestoreRequestOps for RestoreRequest {
     fn validate(&self, _api: Arc<ECStore>) -> Result<(), std::io::Error> {
         // SELECT type requires select_parameters, and vice versa
-        if self.type_.as_ref().is_none_or(|t| t.as_str() != RestoreRequestType::SELECT) && self.select_parameters.is_some() {
+        if self.type_.as_ref().is_none_or(|t| t.as_str() != "SELECT") && self.select_parameters.is_some() {
             return Err(std::io::Error::other("Select parameters can only be specified with SELECT request type"));
         }
         if let Some(type_) = &self.type_
-            && type_.as_str() == RestoreRequestType::SELECT
+            && type_.as_str() == "SELECT"
             && self.select_parameters.is_none()
         {
             return Err(std::io::Error::other("SELECT restore request requires select parameters to be specified"));
         }
 
         // OutputLocation is only valid for SELECT requests
-        if self.type_.as_ref().is_none_or(|t| t.as_str() != RestoreRequestType::SELECT) && self.output_location.is_some() {
+        if self.type_.as_ref().is_none_or(|t| t.as_str() != "SELECT") && self.output_location.is_some() {
             return Err(std::io::Error::other("OutputLocation can only be specified with SELECT request type"));
         }
         select_restore_s3_location(self)?;
 
         // Days must not be specified with SELECT requests
         if let Some(type_) = &self.type_
-            && type_.as_str() == RestoreRequestType::SELECT
+            && type_.as_str() == "SELECT"
             && self.days.is_some_and(|d| d > 0)
         {
             return Err(std::io::Error::other("Days cannot be specified with SELECT restore request"));
@@ -5603,7 +5614,7 @@ pub fn lifecycle_version_delete_target(oi: &ObjectInfo) -> Option<ObjectToDelete
 }
 
 pub async fn eval_action_from_lifecycle(
-    lc: &BucketLifecycleConfiguration,
+    lc: &PersistedLifecycleConfiguration,
     lock_config: Option<&ObjectLockConfiguration>,
     oi: &ObjectInfo,
 ) -> lifecycle::Event {
@@ -6308,14 +6319,19 @@ mod tests {
     use rustfs_data_usage::TierStats;
     use rustfs_filemeta::metadata_keys;
     use rustfs_filemeta::{FileInfo, FileMeta};
+    use rustfs_gateway_types::dto::MetadataEntry;
+    use rustfs_gateway_types::persistence::{
+        PersistedDefaultRetention as DefaultRetention, PersistedObjectLockConfiguration as ObjectLockConfiguration,
+        PersistedObjectLockRule as ObjectLockRule,
+    };
+    use rustfs_gateway_types::persistence::{
+        PersistedLifecycleConfiguration, PersistedLifecycleExpiration, PersistedLifecycleRule,
+        PersistedNoncurrentVersionExpiration, PersistedTransition,
+    };
+    use rustfs_lifecycle::{RestoreOutputLocation as OutputLocation, RestoreRequest, RestoreS3Location as S3Location};
     #[cfg(feature = "test-util")]
     use rustfs_s3_client::transition_api::ReaderImpl;
     use rustfs_scanner_metrics::metrics::{IlmAction, global_metrics};
-    use s3s::dto::{
-        BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule, MetadataEntry,
-        NoncurrentVersionExpiration, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule,
-        OutputLocation, RestoreRequest, RestoreRequestType, S3Location, Timestamp, Transition, TransitionStorageClass,
-    };
     use serial_test::serial;
     use sha2::{Digest, Sha256};
     use std::collections::HashMap;
@@ -7151,25 +7167,6 @@ mod tests {
             0,
             "tier generation lease should release after EOF"
         );
-        let unread = get_transitioned_object_reader_with_tier_manager(
-            &object_info.bucket,
-            &object_info.name,
-            &None,
-            &HeaderMap::new(),
-            &object_info,
-            &ObjectOptions::default(),
-            &manager,
-            None,
-        )
-        .await
-        .expect("open an unread transitioned response");
-        assert_eq!(TierConfigMgr::active_operation_lease_count(&manager, &tier).await, 1);
-        drop(unread);
-        assert_eq!(
-            TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
-            0,
-            "canceling an unread response must release the tier lease"
-        );
     }
 
     #[cfg(feature = "test-util")]
@@ -9341,7 +9338,7 @@ mod tests {
             output_location,
             select_parameters: None,
             tier: None,
-            type_: Some(RestoreRequestType::from_static(RestoreRequestType::SELECT)),
+            type_: Some("SELECT".to_owned()),
         }
     }
 
@@ -9658,12 +9655,12 @@ mod tests {
 
     #[test]
     fn lifecycle_rule_has_date_expiration_detects_enabled_date_rule() {
-        let lc = BucketLifecycleConfiguration {
+        let lc = PersistedLifecycleConfiguration {
             expiry_updated_at: None,
-            rules: vec![LifecycleRule {
-                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
-                expiration: Some(LifecycleExpiration {
-                    date: Some(Timestamp::from(OffsetDateTime::now_utc())),
+            rules: vec![PersistedLifecycleRule {
+                status: "Enabled".to_owned(),
+                expiration: Some(PersistedLifecycleExpiration {
+                    date: Some(lifecycle::format_lifecycle_date(OffsetDateTime::now_utc()).unwrap()),
                     ..Default::default()
                 }),
                 id: Some("rule-date".to_string()),
@@ -10180,14 +10177,15 @@ mod tests {
     #[test]
     fn should_defer_date_expiry_for_recent_config_update_respects_grace_window() {
         let now = OffsetDateTime::now_utc();
-        let recent = BucketLifecycleConfiguration {
-            expiry_updated_at: Some(Timestamp::from(now - time::Duration::seconds(1))),
+        let recent = PersistedLifecycleConfiguration {
+            expiry_updated_at: Some(lifecycle::format_lifecycle_date(now - time::Duration::seconds(1)).unwrap()),
             rules: Vec::new(),
         };
-        let stale = BucketLifecycleConfiguration {
-            expiry_updated_at: Some(Timestamp::from(
-                now - time::Duration::seconds(DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS + 1),
-            )),
+        let stale = PersistedLifecycleConfiguration {
+            expiry_updated_at: Some(
+                lifecycle::format_lifecycle_date(now - time::Duration::seconds(DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS + 1))
+                    .unwrap(),
+            ),
             rules: Vec::new(),
         };
 
@@ -10315,12 +10313,12 @@ mod tests {
         assert_eq!(deleted.version_id, None);
     }
 
-    fn expired_delete_marker_lifecycle() -> BucketLifecycleConfiguration {
-        BucketLifecycleConfiguration {
+    fn expired_delete_marker_lifecycle() -> PersistedLifecycleConfiguration {
+        PersistedLifecycleConfiguration {
             expiry_updated_at: None,
-            rules: vec![LifecycleRule {
-                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
-                expiration: Some(LifecycleExpiration {
+            rules: vec![PersistedLifecycleRule {
+                status: "Enabled".to_owned(),
+                expiration: Some(PersistedLifecycleExpiration {
                     expired_object_delete_marker: Some(true),
                     ..Default::default()
                 }),
@@ -10336,12 +10334,12 @@ mod tests {
         }
     }
 
-    fn latest_expiration_lifecycle() -> BucketLifecycleConfiguration {
-        BucketLifecycleConfiguration {
+    fn latest_expiration_lifecycle() -> PersistedLifecycleConfiguration {
+        PersistedLifecycleConfiguration {
             expiry_updated_at: None,
-            rules: vec![LifecycleRule {
-                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
-                expiration: Some(LifecycleExpiration {
+            rules: vec![PersistedLifecycleRule {
+                status: "Enabled".to_owned(),
+                expiration: Some(PersistedLifecycleExpiration {
                     days: Some(1),
                     ..Default::default()
                 }),
@@ -10357,12 +10355,12 @@ mod tests {
         }
     }
 
-    fn current_and_noncurrent_expiration_lifecycle() -> BucketLifecycleConfiguration {
-        BucketLifecycleConfiguration {
+    fn current_and_noncurrent_expiration_lifecycle() -> PersistedLifecycleConfiguration {
+        PersistedLifecycleConfiguration {
             expiry_updated_at: None,
-            rules: vec![LifecycleRule {
-                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
-                expiration: Some(LifecycleExpiration {
+            rules: vec![PersistedLifecycleRule {
+                status: "Enabled".to_owned(),
+                expiration: Some(PersistedLifecycleExpiration {
                     days: Some(30),
                     ..Default::default()
                 }),
@@ -10370,7 +10368,7 @@ mod tests {
                 del_marker_expiration: None,
                 filter: None,
                 id: Some("expire-current-and-noncurrent".to_string()),
-                noncurrent_version_expiration: Some(NoncurrentVersionExpiration {
+                noncurrent_version_expiration: Some(PersistedNoncurrentVersionExpiration {
                     noncurrent_days: Some(1),
                     newer_noncurrent_versions: None,
                 }),
@@ -10381,12 +10379,12 @@ mod tests {
         }
     }
 
-    fn all_versions_expiration_lifecycle() -> BucketLifecycleConfiguration {
-        BucketLifecycleConfiguration {
+    fn all_versions_expiration_lifecycle() -> PersistedLifecycleConfiguration {
+        PersistedLifecycleConfiguration {
             expiry_updated_at: None,
-            rules: vec![LifecycleRule {
-                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
-                expiration: Some(LifecycleExpiration {
+            rules: vec![PersistedLifecycleRule {
+                status: "Enabled".to_owned(),
+                expiration: Some(PersistedLifecycleExpiration {
                     days: Some(1),
                     expired_object_all_versions: Some(true),
                     ..Default::default()
@@ -10405,29 +10403,29 @@ mod tests {
 
     fn lock_enabled_without_default_retention() -> ObjectLockConfiguration {
         ObjectLockConfiguration {
-            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            object_lock_enabled: Some("Enabled".to_owned()),
             rule: None,
         }
     }
 
     fn lock_enabled_with_default_retention() -> ObjectLockConfiguration {
         ObjectLockConfiguration {
-            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            object_lock_enabled: Some("Enabled".to_owned()),
             rule: Some(ObjectLockRule {
                 default_retention: Some(DefaultRetention {
                     days: Some(30),
-                    mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
+                    mode: Some("COMPLIANCE".to_owned()),
                     years: None,
                 }),
             }),
         }
     }
 
-    fn latest_transition_lifecycle() -> BucketLifecycleConfiguration {
-        BucketLifecycleConfiguration {
+    fn latest_transition_lifecycle() -> PersistedLifecycleConfiguration {
+        PersistedLifecycleConfiguration {
             expiry_updated_at: None,
-            rules: vec![LifecycleRule {
-                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+            rules: vec![PersistedLifecycleRule {
+                status: "Enabled".to_owned(),
                 expiration: None,
                 abort_incomplete_multipart_upload: None,
                 del_marker_expiration: None,
@@ -10436,10 +10434,10 @@ mod tests {
                 noncurrent_version_expiration: None,
                 noncurrent_version_transitions: None,
                 prefix: None,
-                transitions: Some(vec![Transition {
+                transitions: Some(vec![PersistedTransition {
                     days: Some(1),
                     date: None,
-                    storage_class: Some(TransitionStorageClass::from_static("WARM")),
+                    storage_class: Some("WARM".to_owned()),
                 }]),
             }],
         }
@@ -12976,8 +12974,9 @@ mod tests {
 
         for (name, object, lock_config, expected) in cases {
             let object_opts = lifecycle::object_opts_from_object_info(&object);
+            let evaluator_lock_config = lock_config.clone().map(Arc::new);
             let batch_event = Evaluator::new(Arc::new(lifecycle.clone()))
-                .with_lock_retention(lock_config.clone().map(Arc::new))
+                .with_lock_retention(evaluator_lock_config)
                 .eval(&[object_opts])
                 .await
                 .expect("batch lifecycle evaluation should succeed")
@@ -14234,10 +14233,7 @@ mod tests {
         let object = current_object_with_metadata(
             ReplicationStatusType::Completed,
             HashMap::from([
-                (
-                    metadata_keys::OBJECT_LOCK_MODE.to_string(),
-                    s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
-                ),
+                (metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string()),
                 (metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until),
             ]),
         );
@@ -14249,7 +14245,7 @@ mod tests {
 
     #[tokio::test]
     async fn restored_copy_expiry_is_not_blocked_by_retention() {
-        let lifecycle = BucketLifecycleConfiguration {
+        let lifecycle = PersistedLifecycleConfiguration {
             expiry_updated_at: None,
             rules: Vec::new(),
         };
@@ -14259,10 +14255,7 @@ mod tests {
         let mut object = current_object_with_metadata(
             ReplicationStatusType::Completed,
             HashMap::from([
-                (
-                    metadata_keys::OBJECT_LOCK_MODE.to_string(),
-                    ObjectLockRetentionMode::COMPLIANCE.to_string(),
-                ),
+                (metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string()),
                 (metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until),
             ]),
         );
@@ -14479,7 +14472,7 @@ mod tests {
                 "removed" => {
                     metadata_sys::delete(&bucket, BUCKET_LIFECYCLE_CONFIG).await.unwrap();
                 }
-                "disabled" => changed.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::DISABLED),
+                "disabled" => changed.rules[0].status = "Disabled".to_owned(),
                 "longer-days" => {
                     changed.rules[0]
                         .noncurrent_version_expiration
@@ -14595,7 +14588,7 @@ mod tests {
         );
     }
 
-    async fn install_test_expiry_policy(ecstore: &Arc<ECStore>, bucket: &str) -> Arc<BucketLifecycleConfiguration> {
+    async fn install_test_expiry_policy(ecstore: &Arc<ECStore>, bucket: &str) -> Arc<PersistedLifecycleConfiguration> {
         let xml = br#"<LifecycleConfiguration><Rule><ID>expire</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter><Expiration><Days>0</Days></Expiration><NoncurrentVersionExpiration><NoncurrentDays>0</NoncurrentDays></NoncurrentVersionExpiration></Rule></LifecycleConfiguration>"#;
         metadata_sys::update_in(&ecstore.ctx, bucket, BUCKET_LIFECYCLE_CONFIG, xml.to_vec())
             .await
@@ -15288,7 +15281,11 @@ mod tests {
     #[tokio::test]
     #[serial]
     async fn tier_free_version_recovery_production_entrypoint_enqueues_seeded_item() {
-        let (disk_paths, ecstore) = setup_test_env().await;
+        let (disk_paths, ecstore) =
+            temp_env::async_with_vars([(super::ENV_TIER_FREE_VERSION_RECOVERY_ENABLED, Some("false"))], async {
+                setup_test_env().await
+            })
+            .await;
         let tier = format!("RECOVERY{}", Uuid::new_v4().simple()).to_uppercase();
         crate::services::tier::test_util::register_mock_tier(&ecstore.tier_config_mgr(), &tier).await;
         let runtime_state = install_unconsumed_runtime_expiry_worker(&ecstore, 1).await;

crates/ecstore/src/bucket/lifecycle/core.rs

@@ -17,7 +17,7 @@ use crate::object_api::ObjectInfo;
 pub use rustfs_lifecycle::{
     Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate, ObjectOpts,
     RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due,
-    expected_expiry_time, expiration_action_has_valid_target,
+    expected_expiry_time, expiration_action_has_valid_target, format_lifecycle_date, parse_lifecycle_date,
 };
 
 pub fn object_opts_from_object_info(oi: &ObjectInfo) -> ObjectOpts {

crates/ecstore/src/bucket/lifecycle/metadata_boundary.rs

@@ -14,7 +14,8 @@
 
 use std::sync::Arc;
 
-use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration};
+use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
+use rustfs_gateway_types::persistence::PersistedObjectLockConfiguration as ObjectLockConfiguration;
 use time::OffsetDateTime;
 use uuid::Uuid;
 
@@ -24,7 +25,7 @@ use crate::error::{Error, Result};
 
 #[derive(Debug)]
 pub(crate) struct LifecycleExpiryConfigs {
-    pub(crate) lifecycle: Option<Arc<BucketLifecycleConfiguration>>,
+    pub(crate) lifecycle: Option<Arc<PersistedLifecycleConfiguration>>,
     pub(crate) object_lock: Option<Arc<ObjectLockConfiguration>>,
     pub(crate) bucket_incarnation_id: Uuid,
     pub(crate) table_bucket_enabled: bool,
@@ -100,7 +101,7 @@ pub(crate) async fn get_expiry_configs_for_options(
     })
 }
 
-pub(crate) async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
+pub(crate) async fn get_lifecycle_config(bucket: &str) -> Result<(PersistedLifecycleConfiguration, OffsetDateTime)> {
     metadata_sys::get_lifecycle_config(bucket).await
 }
 
@@ -110,15 +111,15 @@ mod tests {
     use crate::bucket::metadata::BucketMetadata;
     use crate::bucket::metadata_sys::{self, test_support::isolated_store_over_temp_disks};
     use crate::storage_api_contracts::bucket::MakeBucketOptions;
-    use s3s::dto::{ExpirationStatus, LifecycleExpiration, LifecycleRule};
+    use rustfs_gateway_types::persistence::{PersistedLifecycleExpiration, PersistedLifecycleRule};
     use serial_test::serial;
 
-    fn lifecycle_config() -> BucketLifecycleConfiguration {
-        BucketLifecycleConfiguration {
+    fn lifecycle_config() -> PersistedLifecycleConfiguration {
+        PersistedLifecycleConfiguration {
             expiry_updated_at: None,
-            rules: vec![LifecycleRule {
-                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
-                expiration: Some(LifecycleExpiration {
+            rules: vec![PersistedLifecycleRule {
+                status: "Enabled".to_owned(),
+                expiration: Some(PersistedLifecycleExpiration {
                     days: Some(1),
                     ..Default::default()
                 }),

crates/ecstore/src/bucket/lifecycle/object_lock_boundary.rs

@@ -22,7 +22,7 @@ pub(crate) fn is_object_locked_by_metadata(user_defined: &HashMap<String, String
 }
 
 pub(crate) fn check_object_lock_for_deletion_with_config(
-    config: Option<&s3s::dto::ObjectLockConfiguration>,
+    config: Option<&rustfs_gateway_types::persistence::PersistedObjectLockConfiguration>,
     obj_info: &ObjectInfo,
     bypass_governance: bool,
 ) -> crate::error::Result<Option<ObjectLockBlockReason>> {

crates/ecstore/src/bucket/lifecycle/replication_sink.rs

@@ -82,12 +82,13 @@ mod tests {
     use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge};
     use crate::object_api::{ObjectInfo, ObjectOptions};
     use crate::storage_api_contracts::object::ObjectToDelete;
-    use rustfs_scanner_metrics::metrics::IlmAction;
-    use s3s::dto::{
-        BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication,
-        DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus,
-        VersioningConfiguration,
+    use rustfs_gateway_types::dto::Status;
+    use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
+    use rustfs_gateway_types::persistence::{
+        PersistedOptionalReplicationStatus, PersistedReplicationConfiguration, PersistedReplicationDestination,
+        PersistedReplicationRule, PersistedReplicationStatus,
     };
+    use rustfs_scanner_metrics::metrics::IlmAction;
     use uuid::Uuid;
 
     use super::*;
@@ -160,27 +161,27 @@ mod tests {
             for purge_enabled in [false, true] {
                 let snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
                     VersioningConfiguration {
-                        status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
+                        status: Some("Enabled".to_owned()),
                         ..Default::default()
                     },
-                    Some(ReplicationConfiguration {
+                    Some(PersistedReplicationConfiguration {
                         role: String::new(),
-                        rules: vec![ReplicationRule {
-                            delete_marker_replication: Some(DeleteMarkerReplication {
+                        rules: vec![PersistedReplicationRule {
+                            delete_marker_replication: Some(PersistedOptionalReplicationStatus {
                                 status: Some(if marker_enabled {
-                                    DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)
+                                    Status::ENABLED.to_string()
                                 } else {
-                                    DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)
+                                    Status::DISABLED.to_string()
                                 }),
                             }),
-                            delete_replication: Some(DeleteReplication {
+                            delete_replication: Some(PersistedReplicationStatus {
                                 status: if purge_enabled {
-                                    DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED)
+                                    Status::ENABLED.to_string()
                                 } else {
-                                    DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED)
+                                    Status::DISABLED.to_string()
                                 },
                             }),
-                            destination: Destination {
+                            destination: PersistedReplicationDestination {
                                 bucket: "arn:rustfs:replication:target".to_string(),
                                 ..Default::default()
                             },
@@ -190,7 +191,7 @@ mod tests {
                             prefix: Some(String::new()),
                             priority: Some(1),
                             source_selection_criteria: None,
-                            status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
+                            status: Status::ENABLED.to_string(),
                         }],
                     }),
                 );

crates/ecstore/src/bucket/lifecycle/runtime_boundary.rs

@@ -14,7 +14,7 @@
 
 use std::sync::Arc;
 
-use s3s::dto::BucketLifecycleConfiguration;
+use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
 use tokio::sync::RwLock;
 use tokio_util::sync::CancellationToken;
 
@@ -52,7 +52,7 @@ pub(crate) fn deployment_id() -> Option<String> {
     sources::deployment_id()
 }
 
-pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<BucketLifecycleConfiguration> {
+pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<PersistedLifecycleConfiguration> {
     sources::bucket_lifecycle_config(bucket).await
 }
 

crates/ecstore/src/bucket/metadata.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use super::msgp_decode::{read_msgp_ext8_time, skip_msgp_value, write_msgp_time};
-use super::object_lock::{ObjectLockApi, ObjectLockStatusExt};
+use super::object_lock::ObjectLockApi;
 use super::versioning::VersioningApi;
 use super::{quota::BucketQuota, target::BucketTargets};
 use crate::bucket::replication::invalid_replication_config_status_field;
@@ -25,12 +25,17 @@ use crate::object_api::WriteCompletion;
 use crate::runtime::sources as runtime_sources;
 use crate::store::ECStore;
 use byteorder::{BigEndian, ByteOrder, LittleEndian};
-use rustfs_policy::policy::BucketPolicy;
-use s3s::dto::{
-    AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, BucketVersioningStatus, CORSConfiguration,
-    NotificationConfiguration, ObjectLockConfiguration, PublicAccessBlockConfiguration, ReplicationConfiguration,
-    RequestPaymentConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration,
+use rustfs_gateway_types::cors_tagging::{PersistedCorsConfiguration as CORSConfiguration, PersistedTagging as Tagging};
+use rustfs_gateway_types::persistence::{
+    PersistedAccelerateConfiguration as AccelerateConfiguration,
+    PersistedBucketEncryptionConfiguration as ServerSideEncryptionConfiguration,
+    PersistedBucketLoggingStatus as BucketLoggingStatus, PersistedLifecycleConfiguration,
+    PersistedNotificationConfiguration as NotificationConfiguration, PersistedObjectLockConfiguration as ObjectLockConfiguration,
+    PersistedPublicAccessBlockConfiguration as PublicAccessBlockConfiguration, PersistedReplicationConfiguration,
+    PersistedRequestPaymentConfiguration as RequestPaymentConfiguration,
+    PersistedVersioningConfiguration as VersioningConfiguration, PersistedWebsiteConfiguration as WebsiteConfiguration,
 };
+use rustfs_policy::policy::BucketPolicy;
 use serde::Serializer;
 use sha2::{Digest, Sha256};
 use std::collections::HashMap;
@@ -40,23 +45,11 @@ use time::{Date, OffsetDateTime, PrimitiveDateTime, Time as CivilTime, UtcOffset
 use tracing::error;
 use uuid::Uuid;
 
-// The serving-layer DTO impls for the storage-level Object Lock traits live
-// here because this module owns the persisted `ObjectLockConfiguration`
-// during the s3s ratchet migration (rustfs/backlog#1842).
+// The persisted Object Lock configuration answers the storage-level trait
+// here because this module owns the persisted `ObjectLockConfiguration`.
 impl ObjectLockApi for ObjectLockConfiguration {
     fn enabled(&self) -> bool {
-        self.object_lock_enabled
-            .as_ref()
-            .is_some_and(|v| v.as_str() == s3s::dto::ObjectLockEnabled::ENABLED)
-    }
-}
-
-impl ObjectLockStatusExt for s3s::dto::ObjectLockLegalHoldStatus {
-    fn valid(&self) -> bool {
-        matches!(
-            self.as_str(),
-            s3s::dto::ObjectLockLegalHoldStatus::ON | s3s::dto::ObjectLockLegalHoldStatus::OFF
-        )
+        self.object_lock_enabled.as_deref() == Some("Enabled")
     }
 }
 
@@ -256,38 +249,6 @@ pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
 pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
 pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
 pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
-/// Shared tag-entry validation for object requests and persisted bucket tags.
-pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
-    fn is_valid_tag_text(value: &str) -> bool {
-        static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
-        VALID_TAG_TEXT
-            .get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
-            .as_ref()
-            .is_ok_and(|pattern| pattern.is_match(value))
-    }
-    let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
-    for tag in tag_set {
-        let key = tag
-            .key
-            .as_deref()
-            .filter(|key| !key.is_empty())
-            .ok_or("Tag key cannot be empty")?;
-        if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
-            return Err("The TagKey you have provided is invalid");
-        }
-        let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
-        if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
-            return Err("The TagValue you have provided is invalid");
-        }
-        if !keys.insert(key) {
-            return Err("Cannot provide multiple Tags with the same key");
-        }
-    }
-    Ok(())
-}
-
-pub const MAX_BUCKET_TAGS: usize = 50;
-
 pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
 pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
 pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
@@ -471,6 +432,8 @@ pub struct BucketMetadata {
     pub bucket_incarnation_id: Uuid,
     pub(crate) bucket_incarnation_sidecar: bool,
     pub(crate) bucket_creation_committed: bool,
+    // A legacy default is not proof of an interrupted creation.
+    pub(crate) bucket_creation_commit_record_present: bool,
     pub policy_config_json: Vec<u8>,
     pub notification_config_xml: Vec<u8>,
     pub lifecycle_config_xml: Vec<u8>,
@@ -519,13 +482,13 @@ pub struct BucketMetadata {
 
     pub policy_config: Option<BucketPolicy>,
     pub notification_config: Option<NotificationConfiguration>,
-    pub lifecycle_config: Option<BucketLifecycleConfiguration>,
+    pub lifecycle_config: Option<PersistedLifecycleConfiguration>,
     pub object_lock_config: Option<ObjectLockConfiguration>,
     pub versioning_config: Option<VersioningConfiguration>,
     pub sse_config: Option<ServerSideEncryptionConfiguration>,
     pub tagging_config: Option<Tagging>,
     pub quota_config: Option<BucketQuota>,
-    pub replication_config: Option<ReplicationConfiguration>,
+    pub replication_config: Option<PersistedReplicationConfiguration>,
     pub bucket_target_config: Option<BucketTargets>,
     pub bucket_target_config_meta: Option<HashMap<String, String>>,
     pub cors_config: Option<CORSConfiguration>,
@@ -546,6 +509,7 @@ impl Default for BucketMetadata {
             bucket_incarnation_id: Uuid::nil(),
             bucket_incarnation_sidecar: false,
             bucket_creation_committed: false,
+            bucket_creation_commit_record_present: false,
             policy_config_json: Default::default(),
             notification_config_xml: Default::default(),
             lifecycle_config_xml: Default::default(),
@@ -784,7 +748,10 @@ impl BucketMetadata {
                 "TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?,
                 "DurabilityConfigUpdatedAt" => self.durability_config_updated_at = read_msgp_time_value(rd)?,
                 "OnDemandMigrationConfigUpdatedAt" => self.on_demand_migration_config_updated_at = read_msgp_time_value(rd)?,
-                "BucketCreationCommitted" => self.bucket_creation_committed = read_msgp_bool(rd)?,
+                "BucketCreationCommitted" => {
+                    self.bucket_creation_committed = read_msgp_bool(rd)?;
+                    self.bucket_creation_commit_record_present = true;
+                }
                 other => {
                     tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field");
                     skip_msgp_value(rd)?;
@@ -797,8 +764,10 @@ impl BucketMetadata {
 
     /// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
     pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
-        // Map size: MinIO fields (25) + RustFS extensions (22)
-        let map_len: u32 = 47;
+        // Preserve the absence of a legacy commit record across config rewrites.
+        let commit_record_present = self.bucket_creation_commit_record_present || self.bucket_creation_committed;
+        // Map size: MinIO fields (25) + RustFS extensions (21) + optional commit record.
+        let map_len: u32 = 46 + u32::from(commit_record_present);
         rmp::encode::write_map_len(wr, map_len)?;
 
         // MinIO field order (same as Go struct)
@@ -879,8 +848,10 @@ impl BucketMetadata {
         write_msgp_time(wr, self.durability_config_updated_at)?;
         rmp::encode::write_str(wr, "OnDemandMigrationConfigUpdatedAt")?;
         write_msgp_time(wr, self.on_demand_migration_config_updated_at)?;
-        rmp::encode::write_str(wr, "BucketCreationCommitted")?;
-        rmp::encode::write_bool(wr, self.bucket_creation_committed)?;
+        if commit_record_present {
+            rmp::encode::write_str(wr, "BucketCreationCommitted")?;
+            rmp::encode::write_bool(wr, self.bucket_creation_committed)?;
+        }
 
         Ok(())
     }
@@ -1022,20 +993,6 @@ impl BucketMetadata {
                 self.encryption_config_updated_at = updated;
             }
             BUCKET_TAGGING_CONFIG => {
-                if !data.is_empty() {
-                    let tagging = deserialize::<Tagging>(&data).map_err(|_| {
-                        Error::InvalidArgument(self.name.clone(), String::new(), "Invalid bucket tagging XML".to_owned())
-                    })?;
-                    if tagging.tag_set.len() > MAX_BUCKET_TAGS {
-                        return Err(Error::InvalidArgument(
-                            self.name.clone(),
-                            String::new(),
-                            "Cannot have more than 50 tags per bucket".to_owned(),
-                        ));
-                    }
-                    validate_tag_set_entries(&tagging.tag_set)
-                        .map_err(|message| Error::InvalidArgument(self.name.clone(), String::new(), message.to_owned()))?;
-                }
                 self.tagging_config_xml = data;
                 // Drop the parsed form (like lifecycle above) so clearing the
                 // payload can't leave stale parsed tags to be cached.
@@ -1064,9 +1021,11 @@ impl BucketMetadata {
                     None
                 } else {
                     let config = deserialize::<VersioningConfiguration>(&data)?;
-                    if config.status.as_ref().is_some_and(|status| {
-                        !matches!(status.as_str(), BucketVersioningStatus::ENABLED | BucketVersioningStatus::SUSPENDED)
-                    }) {
+                    if config
+                        .status
+                        .as_ref()
+                        .is_some_and(|status| !matches!(status.as_str(), "Enabled" | "Suspended"))
+                    {
                         return Err(Error::other("bucket versioning configuration has an invalid status"));
                     }
                     Some(config)
@@ -1079,7 +1038,7 @@ impl BucketMetadata {
                 let config = if data.is_empty() {
                     None
                 } else {
-                    let config = deserialize::<ReplicationConfiguration>(&data)?;
+                    let config = deserialize::<PersistedReplicationConfiguration>(&data)?;
                     if let Some(field) = invalid_replication_config_status_field(&config) {
                         return Err(Error::other(format!("replication field {field} has an invalid status")));
                     }
@@ -1273,8 +1232,8 @@ impl BucketMetadata {
         }
         self.lifecycle_config = None;
         if !self.lifecycle_config_xml.is_empty()
-            && let Err(e) =
-                deserialize::<BucketLifecycleConfiguration>(&self.lifecycle_config_xml).map(|c| self.lifecycle_config = Some(c))
+            && let Err(e) = deserialize::<PersistedLifecycleConfiguration>(&self.lifecycle_config_xml)
+                .map(|c| self.lifecycle_config = Some(c))
         {
             tracing::warn!(
                 event = "bucket_metadata_parse_failed",
@@ -1360,8 +1319,8 @@ impl BucketMetadata {
         }
         self.replication_config = None;
         if !self.replication_config_xml.is_empty()
-            && let Err(e) =
-                deserialize::<ReplicationConfiguration>(&self.replication_config_xml).map(|c| self.replication_config = Some(c))
+            && let Err(e) = deserialize::<PersistedReplicationConfiguration>(&self.replication_config_xml)
+                .map(|c| self.replication_config = Some(c))
         {
             tracing::warn!(
                 event = "bucket_metadata_parse_failed",
@@ -1549,11 +1508,7 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence(
     bucket: &str,
     parse: bool,
 ) -> Result<(BucketMetadata, bool)> {
-    let (metadata_res, incarnation_res) = tokio::join!(
-        Box::pin(read_bucket_metadata(api.clone(), bucket)),
-        Box::pin(load_bucket_incarnation(api.clone(), bucket)),
-    );
-    let (mut bm, persisted) = match metadata_res {
+    let (mut bm, persisted) = match read_bucket_metadata(api.clone(), bucket).await {
         Ok(res) => (res, true),
         Err(err) => {
             if err != Error::ConfigNotFound {
@@ -1564,7 +1519,7 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence(
         }
     };
 
-    let incarnation = incarnation_res?;
+    let incarnation = load_bucket_incarnation(api.clone(), bucket).await?;
     if persisted {
         if bm.name.is_empty() {
             bm.name = bucket.to_string();
@@ -1741,6 +1696,13 @@ mod test {
         let mut bm = BucketMetadata::unmarshal(&body[4..]).expect("unmarshal recovered blob");
         assert_eq!(bm.name, "interop");
         assert!(!bm.bucket_creation_committed, "legacy metadata must default to uncommitted");
+        assert!(!bm.bucket_creation_commit_record_present, "legacy metadata has no creation-commit record");
+        let rewritten = BucketMetadata::unmarshal(&bm.marshal_msg().expect("rewrite legacy metadata"))
+            .expect("read rewritten legacy metadata");
+        assert!(
+            !rewritten.bucket_creation_commit_record_present,
+            "rewriting legacy metadata must not fabricate a creation intent"
+        );
         bm.parse_all_configs().expect("parse recovered configs");
         assert!(bm.lifecycle_config.is_some());
     }
@@ -1758,6 +1720,7 @@ mod test {
 
         assert_eq!(bm.name, new.name);
         assert!(new.bucket_creation_committed);
+        assert!(new.bucket_creation_commit_record_present);
         assert!(!bm.bucket_incarnation_id.is_nil());
         assert_eq!(bm.bucket_incarnation_id, new.bucket_incarnation_id);
     }
@@ -1830,31 +1793,6 @@ mod test {
     /// backlog#2292: a replicated config is stamped with the source
     /// `updated_at` it was given, not the local clock, while the plain
     /// `update_config` entry keeps stamping the local clock.
-    #[test]
-    fn bucket_tag_updates_reject_unreadable_shapes_without_replacing_metadata() {
-        let mut metadata = BucketMetadata::new("validated-tags");
-        let valid = br#"<Tagging><TagSet><Tag><Key>team</Key><Value></Value></Tag></TagSet></Tagging>"#.to_vec();
-        metadata
-            .update_config(BUCKET_TAGGING_CONFIG, valid.clone())
-            .expect("empty value is valid");
-        let updated = metadata.tagging_config_updated_at;
-        for invalid in [
-            "<Tagging><TagSet><Tag><Key>team</Key></Tag></TagSet></Tagging>".to_owned(),
-            "<Tagging><TagSet><Tag><Key></Key><Value>x</Value></Tag></TagSet></Tagging>".to_owned(),
-            "<Tagging><TagSet><Tag><Key>team</Key><Value>x</Value></Tag><Tag><Key>team</Key><Value>y</Value></Tag></TagSet></Tagging>".to_owned(),
-            format!("<Tagging><TagSet>{}</TagSet></Tagging>", (0..51).map(|i| format!("<Tag><Key>k{i}</Key><Value>v</Value></Tag>")).collect::<String>()),
-        ] {
-            assert!(metadata.update_config_at(BUCKET_TAGGING_CONFIG, invalid.into_bytes(), updated + time::Duration::hours(1)).is_err());
-            assert_eq!(metadata.tagging_config_xml, valid);
-            assert_eq!(metadata.tagging_config_updated_at, updated);
-        }
-        metadata.tagging_config_xml = b"legacy corrupt tags".to_vec();
-        metadata
-            .update_config(BUCKET_TAGGING_CONFIG, Vec::new())
-            .expect("deletion repairs legacy corruption");
-        assert!(metadata.tagging_config_xml.is_empty());
-    }
-
     #[test]
     fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
         let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);
@@ -1887,17 +1825,15 @@ mod test {
 
     #[test]
     fn object_locking_requires_lock_metadata_not_plain_versioning() {
-        use s3s::dto::ObjectLockEnabled;
-
         let mut bm = BucketMetadata::new("test-bucket");
         bm.versioning_config = Some(VersioningConfiguration {
-            status: Some(s3s::dto::BucketVersioningStatus::from_static("Enabled")),
+            status: Some("Enabled".to_owned()),
             ..Default::default()
         });
         assert!(!bm.object_locking());
 
         bm.object_lock_config = Some(ObjectLockConfiguration {
-            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            object_lock_enabled: Some("Enabled".to_owned()),
             ..Default::default()
         });
         assert!(bm.object_locking());

crates/ecstore/src/bucket/metadata_sys.rs

@@ -30,14 +30,18 @@ use crate::storage_api_contracts::heal::HealOperations as _;
 use crate::storage_api_contracts::namespace::NamespaceLocking as _;
 use crate::store::{ECStore, await_bucket_namespace_operation};
 use futures::future::join_all;
+use rustfs_gateway_types::cors_tagging::{PersistedCorsConfiguration as CORSConfiguration, PersistedTagging as Tagging};
+use rustfs_gateway_types::persistence::{
+    PersistedAccelerateConfiguration as AccelerateConfiguration,
+    PersistedBucketEncryptionConfiguration as ServerSideEncryptionConfiguration,
+    PersistedBucketLoggingStatus as BucketLoggingStatus, PersistedLifecycleConfiguration,
+    PersistedNotificationConfiguration as NotificationConfiguration, PersistedObjectLockConfiguration as ObjectLockConfiguration,
+    PersistedPublicAccessBlockConfiguration as PublicAccessBlockConfiguration, PersistedReplicationConfiguration,
+    PersistedRequestPaymentConfiguration as RequestPaymentConfiguration,
+    PersistedVersioningConfiguration as VersioningConfiguration, PersistedWebsiteConfiguration as WebsiteConfiguration,
+};
 use rustfs_heal_contracts::heal_channel::HealOpts;
 use rustfs_policy::policy::BucketPolicy;
-use s3s::dto::ReplicationConfiguration;
-use s3s::dto::{
-    AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, CORSConfiguration, NotificationConfiguration,
-    ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, PublicAccessBlockConfiguration,
-    RequestPaymentConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration,
-};
 use std::collections::HashSet;
 use std::time::Duration;
 use std::{
@@ -168,7 +172,7 @@ pub(crate) fn object_lock_config_state_from_authoritative_metadata(bm: &BucketMe
     if bm.lock_enabled {
         return Ok(ObjectLockConfigState::Configured {
             config: ObjectLockConfiguration {
-                object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+                object_lock_enabled: Some("Enabled".to_owned()),
                 rule: None,
             },
             updated_at: bm.object_lock_config_updated_at,
@@ -212,17 +216,10 @@ pub(crate) fn configured_object_lock_state_for_tests(
 ) -> ObjectLockConfigState {
     ObjectLockConfigState::Configured {
         config: ObjectLockConfiguration {
-            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
-            rule: Some(s3s::dto::ObjectLockRule {
-                default_retention: Some(s3s::dto::DefaultRetention {
-                    mode: Some(s3s::dto::ObjectLockRetentionMode::from_static(match mode {
-                        crate::bucket::object_lock::types::RetentionMode::Governance => {
-                            s3s::dto::ObjectLockRetentionMode::GOVERNANCE
-                        }
-                        crate::bucket::object_lock::types::RetentionMode::Compliance => {
-                            s3s::dto::ObjectLockRetentionMode::COMPLIANCE
-                        }
-                    })),
+            object_lock_enabled: Some("Enabled".to_owned()),
+            rule: Some(rustfs_gateway_types::persistence::PersistedObjectLockRule {
+                default_retention: Some(rustfs_gateway_types::persistence::PersistedDefaultRetention {
+                    mode: Some(mode.as_str().to_owned()),
                     days: Some(days),
                     years: None,
                 }),
@@ -233,7 +230,7 @@ pub(crate) fn configured_object_lock_state_for_tests(
 }
 
 fn validate_authoritative_object_lock_config(config: &ObjectLockConfiguration) -> Result<()> {
-    if config.object_lock_enabled.as_ref().map(ObjectLockEnabled::as_str) != Some(ObjectLockEnabled::ENABLED) {
+    if config.object_lock_enabled.as_deref() != Some("Enabled") {
         return Err(Error::other("persisted bucket Object Lock enabled state is invalid"));
     }
 
@@ -246,7 +243,7 @@ fn validate_authoritative_object_lock_config(config: &ObjectLockConfiguration) -
     if !retention
         .mode
         .as_ref()
-        .is_some_and(|mode| matches!(mode.as_str(), ObjectLockRetentionMode::COMPLIANCE | ObjectLockRetentionMode::GOVERNANCE))
+        .is_some_and(|mode| matches!(mode.as_str(), "COMPLIANCE" | "GOVERNANCE"))
     {
         return Err(Error::other("persisted bucket Object Lock retention mode is invalid"));
     }
@@ -683,7 +680,6 @@ pub(crate) async fn update_validated_in<F>(
     bucket: &str,
     config_file: &str,
     data: Vec<u8>,
-    expected_incarnation_id: Option<Uuid>,
     validate: F,
 ) -> Result<OffsetDateTime>
 where
@@ -691,7 +687,7 @@ where
 {
     Box::pin(async {
         let sys = require_bucket_metadata_sys_in(ctx)?;
-        let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
+        let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
         validate()?;
         update_under_config_write_guard(sys, &guard, config_file, data, None).await
     })
@@ -1396,7 +1392,7 @@ pub async fn get_public_access_block_config(bucket: &str) -> Result<(PublicAcces
     bucket_meta_sys.get_public_access_block_config(bucket).await
 }
 
-pub async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
+pub async fn get_lifecycle_config(bucket: &str) -> Result<(PersistedLifecycleConfiguration, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
 
@@ -1436,7 +1432,6 @@ pub(crate) async fn get_object_lock_config_state_in(
 
 /// Re-read the Object Lock state and bucket incarnation from the same
 /// authoritative metadata blob while the caller holds the transaction lock.
-#[cfg(all(test, feature = "test-util"))]
 pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
     ctx: &crate::runtime::instance::InstanceContext,
     bucket: &str,
@@ -1455,27 +1450,6 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_with_guard_
     get_delete_config_from_disk_in(ctx, bucket, guard, None).await
 }
 
-#[allow(clippy::type_complexity)]
-pub(crate) async fn get_object_lock_and_quota_config_from_disk_with_guard_in(
-    ctx: &crate::runtime::instance::InstanceContext,
-    bucket: &str,
-    guard: Option<&rustfs_lock::NamespaceLockGuard>,
-) -> Result<(
-    ObjectLockConfigState,
-    Uuid,
-    OffsetDateTime,
-    std::result::Result<(Option<BucketQuota>, Uuid, OffsetDateTime), String>,
-)> {
-    let (state, incarnation_id, updated_at, quota_res) =
-        get_delete_and_quota_config_from_disk_in(ctx, bucket, guard, None, true).await?;
-    Ok((
-        state,
-        incarnation_id,
-        updated_at,
-        quota_res.unwrap_or(Ok((None, incarnation_id, updated_at))),
-    ))
-}
-
 /// Validate queued expiration against the same fresh metadata used for Object
 /// Lock admission. The caller holds the transaction read lock through commit,
 /// so a successful lifecycle update cannot be followed by an old-policy delete.
@@ -1485,24 +1459,6 @@ pub(crate) async fn get_delete_config_from_disk_in(
     guard: Option<&rustfs_lock::NamespaceLockGuard>,
     expiry_policy: Option<&rustfs_lifecycle::ExpiryPolicySnapshot>,
 ) -> Result<(ObjectLockConfigState, Uuid, OffsetDateTime)> {
-    let (state, incarnation_id, updated_at, _) =
-        get_delete_and_quota_config_from_disk_in(ctx, bucket, guard, expiry_policy, false).await?;
-    Ok((state, incarnation_id, updated_at))
-}
-
-#[allow(clippy::type_complexity)]
-async fn get_delete_and_quota_config_from_disk_in(
-    ctx: &crate::runtime::instance::InstanceContext,
-    bucket: &str,
-    guard: Option<&rustfs_lock::NamespaceLockGuard>,
-    expiry_policy: Option<&rustfs_lifecycle::ExpiryPolicySnapshot>,
-    include_quota: bool,
-) -> Result<(
-    ObjectLockConfigState,
-    Uuid,
-    OffsetDateTime,
-    Option<std::result::Result<(Option<BucketQuota>, Uuid, OffsetDateTime), String>>,
-)> {
     let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
 
@@ -1518,7 +1474,7 @@ async fn get_delete_and_quota_config_from_disk_in(
                 .await?
         }
     };
-    match &authority {
+    match authority {
         BucketMetadataAuthority::Authoritative(metadata)
             if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
         {
@@ -1530,11 +1486,11 @@ async fn get_delete_and_quota_config_from_disk_in(
                 }
             }
             let object_lock_config_updated_at = metadata.object_lock_config_updated_at;
-            let state = object_lock_config_state_from_authoritative_metadata(metadata)?;
-            let incarnation_id = metadata.bucket_incarnation_id;
-            let quota_res = include_quota
-                .then(|| quota_config_and_incarnation_from_authority(bucket, authority).map_err(|err| err.to_string()));
-            Ok((state, incarnation_id, object_lock_config_updated_at, quota_res))
+            Ok((
+                object_lock_config_state_from_authoritative_metadata(&metadata)?,
+                metadata.bucket_incarnation_id,
+                object_lock_config_updated_at,
+            ))
         }
         BucketMetadataAuthority::Authoritative(_) => {
             Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))
@@ -1605,23 +1561,6 @@ pub(crate) async fn get_quota_config_and_incarnation_from_disk_for_options_in(
         return Err(Error::other("quota snapshot metadata transaction fence was released"));
     }
 
-    #[cfg(test)]
-    let has_injected_error = sys.object_lock_disk_read_errors.read().await.contains(bucket);
-    #[cfg(not(test))]
-    let has_injected_error = false;
-    if !ctx.lock_manager().is_disabled()
-        && !has_injected_error
-        && let Some(fresh_quota) = snapshot.take_fresh_put_quota()
-    {
-        if snapshot_guard.is_lock_lost()
-            || snapshot_guard.is_released()
-            || !snapshot.is_valid_for_destructive_put(store.id, bucket, opts.expected_bucket_incarnation_id.unwrap_or_default())
-        {
-            return Err(Error::other("quota snapshot fence or bucket incarnation changed during the disk read"));
-        }
-        return fresh_quota.map_err(Error::other);
-    }
-
     // The snapshot carries the caller's lifecycle fence, excluding deletion.
     // Its transaction guard excludes config writes, so a same-name object
     // owner need not recursively acquire the bucket namespace here.
@@ -1672,7 +1611,7 @@ pub(crate) async fn get_cached_quota_config_and_incarnation_in(
     bucket_meta_sys.cached_quota_config_and_incarnation(bucket).await
 }
 
-pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
+pub async fn get_replication_config(bucket: &str) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
 
@@ -1682,7 +1621,7 @@ pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfigur
 pub(crate) async fn get_replication_config_in(
     ctx: &crate::runtime::instance::InstanceContext,
     bucket: &str,
-) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
+) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
     let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
 
@@ -1808,22 +1747,27 @@ struct MetadataPublishGuard {
     _guard: tokio::sync::OwnedMutexGuard<MetadataPublishLockState>,
 }
 
-static LOCAL_META_MUTATION_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
-
-pub(crate) fn bump_local_meta_mutation_gen() {
-    LOCAL_META_MUTATION_GEN.fetch_add(1, std::sync::atomic::Ordering::Release);
+#[derive(Debug, Clone)]
+pub(crate) struct PendingBucketCreationProof {
+    pub(crate) generation: Uuid,
+    pub(crate) intent: Vec<u8>,
+    pub(crate) versioning_requested: bool,
+    pub(crate) witnesses: Vec<crate::disk::os::NativeBucketCreationWitness>,
 }
 
-pub(crate) fn current_local_meta_mutation_gen() -> u64 {
-    LOCAL_META_MUTATION_GEN.load(std::sync::atomic::Ordering::Acquire)
+impl PendingBucketCreationProof {
+    pub(crate) fn matches_metadata(&self, metadata: &BucketMetadata) -> Result<bool> {
+        Ok(!metadata.bucket_creation_committed
+            && metadata.bucket_incarnation_sidecar
+            && metadata.bucket_incarnation_id == self.generation
+            && metadata.marshal_msg()? == self.intent)
+    }
 }
 
-type LocalAuthorityCache = Arc<StdMutex<HashMap<String, (u64, Arc<BucketMetadata>)>>>;
-
 #[derive(Debug, Clone)]
 pub struct BucketMetadataSys {
     metadata_map: Arc<RwLock<HashMap<String, Arc<BucketMetadata>>>>,
-    local_authority_cache: LocalAuthorityCache,
+    pending_bucket_creations: Arc<RwLock<HashMap<String, PendingBucketCreationProof>>>,
     /// Serializes metadata-map commits and their derived cache updates for one
     /// bucket. Namespace locks, when present, are acquired before this lock.
     metadata_publish_locks: Arc<MetadataPublishLockRegistry>,
@@ -1867,7 +1811,7 @@ impl BucketMetadataSys {
     pub fn new(api: Arc<ECStore>) -> Self {
         Self {
             metadata_map: Arc::new(RwLock::new(HashMap::new())),
-            local_authority_cache: Arc::new(StdMutex::new(HashMap::new())),
+            pending_bucket_creations: Arc::new(RwLock::new(HashMap::new())),
             metadata_publish_locks: Arc::new(MetadataPublishLockRegistry {
                 locks: StdMutex::new(HashMap::new()),
             }),
@@ -1898,17 +1842,83 @@ impl BucketMetadataSys {
             .expect("bucket metadata object store should still be live")
     }
 
-    fn object_store_if_live(&self) -> Option<Arc<ECStore>> {
-        self.api.upgrade()
+    pub(crate) async fn pending_bucket_creation(&self, bucket: &str) -> Option<PendingBucketCreationProof> {
+        self.pending_bucket_creations.read().await.get(bucket).cloned()
     }
 
-    /// Whether the backing `ECStore` has been dropped.
-    ///
-    /// Used by `InstanceContext::clear_bucket_metadata_sys_if_stale` during
-    /// `ECStore::drop` so a shared bootstrap context does not retain a
-    /// `BucketMetadataSys` whose weak handle is dangling.
-    pub(crate) fn is_store_stale(&self) -> bool {
-        self.api.upgrade().is_none()
+    pub(crate) async fn grant_pending_bucket_creation(&self, bucket: &str, proof: PendingBucketCreationProof) {
+        self.pending_bucket_creations.write().await.insert(bucket.to_owned(), proof);
+    }
+
+    pub(crate) async fn revoke_pending_bucket_creation(&self, bucket: &str) {
+        self.pending_bucket_creations.write().await.remove(bucket);
+    }
+
+    pub(crate) async fn consume_pending_bucket_creation(
+        &self,
+        bucket: &str,
+        metadata: &BucketMetadata,
+        witnesses: &[crate::disk::os::NativeBucketCreationWitness],
+    ) -> Result<bool> {
+        let mut pending = self.pending_bucket_creations.write().await;
+        let Some(proof) = pending.get(bucket) else {
+            return Ok(false);
+        };
+        if !proof.matches_metadata(metadata)?
+            || proof.witnesses.len() != witnesses.len()
+            || !proof
+                .witnesses
+                .iter()
+                .zip(witnesses)
+                .all(|(old, current)| old.matches(current))
+            || !crate::disk::os::consume_native_creation_witnesses(witnesses)
+        {
+            return Ok(false);
+        }
+        pending.remove(bucket);
+        Ok(true)
+    }
+
+    #[cfg(test)]
+    pub(crate) async fn refresh_bucket_for_test(&self, bucket: &str) -> Result<()> {
+        let expected = self.metadata_map.read().await.get(bucket).cloned();
+        let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
+        let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
+        self.load_bucket_under_namespace(bucket, MetadataLoadMode::Refresh, expected.as_ref(), &guard)
+            .await
+    }
+
+    #[cfg(test)]
+    pub(crate) async fn publish_persisted_metadata_for_test(&self, bucket: &str) -> Result<()> {
+        let expected = self.metadata_map.read().await.get(bucket).cloned();
+        let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
+        let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
+        let (metadata, persisted) = await_bucket_namespace_operation(
+            Some(&guard),
+            bucket,
+            "test cached metadata load",
+            load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
+        )
+        .await?;
+        self.publish_if_unchanged(bucket, expected.as_ref(), metadata, persisted, &guard)
+            .await
+    }
+
+    #[cfg(test)]
+    pub(crate) async fn migrate_bucket_creation_commit_for_test(&self, bucket: &str) -> Result<()> {
+        self.migrate_bucket_creation_commit(bucket).await.map(|_| ())
+    }
+
+    #[cfg(test)]
+    pub(crate) async fn legacy_migration_reports_missing_bucket_for_test(&self, bucket: &str) -> Result<bool> {
+        Ok(matches!(
+            self.migrate_legacy_metadata(bucket).await?,
+            BucketMetadataAuthority::MissingBucket
+        ))
+    }
+
+    fn object_store_if_live(&self) -> Option<Arc<ECStore>> {
+        self.api.upgrade()
     }
 
     fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
@@ -2097,6 +2107,7 @@ impl BucketMetadataSys {
         expected: Option<&Arc<BucketMetadata>>,
         namespace_guard: &rustfs_lock::NamespaceLockGuard,
     ) -> Result<()> {
+        self.revoke_pending_bucket_creation(bucket).await;
         if !await_bucket_namespace_operation(
             Some(namespace_guard),
             bucket,
@@ -2180,6 +2191,7 @@ impl BucketMetadataSys {
         persisted: bool,
         namespace_guard: &rustfs_lock::NamespaceLockGuard,
     ) -> Result<()> {
+        self.revoke_pending_bucket_creation(bucket).await;
         if !persisted {
             let _publish_guard = self
                 .lock_metadata_publish(bucket, namespace_guard, "refreshed bucket metadata absence publish")
@@ -2241,7 +2253,7 @@ impl BucketMetadataSys {
     }
 
     pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
-        bump_local_meta_mutation_gen();
+        self.revoke_pending_bucket_creation(&bucket).await;
         if !is_meta_bucketname(&bucket) {
             let publish_lock = self.metadata_publish_lock(&bucket);
             let _publish_guard = publish_lock.lock().await;
@@ -2260,10 +2272,10 @@ impl BucketMetadataSys {
     ///
     /// Returns `true` if an entry was present. Reserved meta buckets are ignored.
     pub async fn remove(&self, bucket: &str) -> bool {
-        bump_local_meta_mutation_gen();
         if is_meta_bucketname(bucket) {
             return false;
         }
+        self.revoke_pending_bucket_creation(bucket).await;
         let publish_lock = self.metadata_publish_lock(bucket);
         let _publish_guard = publish_lock.lock().await;
         let mut map = self.metadata_map.write().await;
@@ -2281,7 +2293,7 @@ impl BucketMetadataSys {
     }
 
     async fn _reset(&mut self) {
-        bump_local_meta_mutation_gen();
+        self.pending_bucket_creations.write().await.clear();
         let mut map = self.metadata_map.write().await;
         map.clear();
         drop(map);
@@ -2420,6 +2432,7 @@ impl BucketMetadataSys {
     /// server's metadata never leaks into the ambient (first) instance.
     pub(crate) async fn persist_and_set(&self, bm: BucketMetadata) -> Result<()> {
         let mut bm = bm;
+        self.revoke_pending_bucket_creation(&bm.name).await;
         bm.save_with_store(self.object_store()).await?;
 
         self.set(bm.name.clone(), Arc::new(bm)).await;
@@ -2428,6 +2441,7 @@ impl BucketMetadataSys {
     }
 
     async fn persist_new_and_set(&self, mut bm: BucketMetadata) -> Result<()> {
+        self.revoke_pending_bucket_creation(&bm.name).await;
         bm.bucket_creation_committed = true;
         bm.save_with_store_committed(self.object_store()).await?;
         save_bucket_incarnation(self.object_store(), &bm.name, bm.bucket_incarnation_id).await?;
@@ -2437,7 +2451,9 @@ impl BucketMetadataSys {
     }
 
     async fn persist_new_bucket_metadata_intent(&self, mut bm: BucketMetadata) -> Result<()> {
+        self.revoke_pending_bucket_creation(&bm.name).await;
         bm.bucket_creation_committed = false;
+        bm.bucket_creation_commit_record_present = true;
         bm.save_with_store(self.object_store()).await?;
         save_bucket_incarnation(self.object_store(), &bm.name, bm.bucket_incarnation_id).await?;
         bm.bucket_incarnation_sidecar = true;
@@ -2447,6 +2463,7 @@ impl BucketMetadataSys {
     }
 
     async fn commit_bucket_metadata(&self, mut bm: BucketMetadata) -> Result<()> {
+        self.revoke_pending_bucket_creation(&bm.name).await;
         bm.parse_all_configs()?;
         bm.bucket_incarnation_sidecar = true;
         // The caller holds the full bucket creation fence (lifecycle, metadata
@@ -2495,6 +2512,7 @@ impl BucketMetadataSys {
         bucket: &str,
         namespace_guard: &rustfs_lock::NamespaceLockGuard,
     ) -> Result<()> {
+        self.revoke_pending_bucket_creation(bucket).await;
         let expected = self.metadata_map.read().await.get(bucket).cloned();
         if !self
             .bucket_exists(bucket, namespace_guard, "peer bucket metadata existence check")
@@ -2551,6 +2569,7 @@ impl BucketMetadataSys {
             #[cfg(test)]
             self.lazy_disk_loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
 
+            self.revoke_pending_bucket_creation(bucket).await;
             let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
             let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
             #[cfg(test)]
@@ -2876,6 +2895,7 @@ impl BucketMetadataSys {
     /// revalidated at write quorum, so a stale snapshot can neither revert an
     /// acknowledged configuration update nor outlive a delete/recreate.
     async fn migrate_bucket_creation_commit(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
+        self.revoke_pending_bucket_creation(bucket).await;
         let transaction_lock = self
             .object_store()
             .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
@@ -2953,6 +2973,7 @@ impl BucketMetadataSys {
         &self,
         bucket: &str,
     ) -> Result<BucketMetadataAuthority> {
+        self.revoke_pending_bucket_creation(bucket).await;
         #[cfg(test)]
         if self.object_lock_disk_read_errors.write().await.remove(bucket) {
             return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
@@ -3085,77 +3106,30 @@ impl BucketMetadataSys {
             return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
         }
 
-        let store = self.object_store();
-        let can_use_local_authority_cache = !store.ctx.lock_manager().is_disabled() && store.all_disks_local_and_online();
-        let gen_before = current_local_meta_mutation_gen();
-        let cached_authority = can_use_local_authority_cache
-            .then(|| {
-                self.local_authority_cache
-                    .lock()
-                    .unwrap_or_else(std::sync::PoisonError::into_inner)
-                    .get(bucket)
-                    .filter(|(cached_gen, _)| *cached_gen == gen_before)
-                    .map(|(_, bm)| Arc::clone(bm))
-            })
-            .flatten();
-        if let Some(cached) = cached_authority {
-            await_bucket_namespace_operation(
-                Some(guard),
-                bucket,
-                "bucket metadata authoritative snapshot",
-                std::future::ready(Ok::<_, Error>(())),
-            )
-            .await?;
-            if current_local_meta_mutation_gen() == gen_before && store.all_disks_local_and_online() {
-                return Ok(BucketMetadataAuthority::Authoritative(cached));
-            }
-        }
-
-        let (bucket_info_res, metadata_res) = tokio::join!(
-            Box::pin(await_bucket_namespace_operation(
-                Some(guard),
-                bucket,
-                "bucket metadata snapshot existence check",
-                async {
-                    store
-                        .get_bucket_info_from_sets_at_read_quorum(
-                            bucket,
-                            &crate::storage_api_contracts::bucket::BucketOptions::default(),
-                        )
-                        .await
-                },
-            )),
-            Box::pin(await_bucket_namespace_operation(
-                Some(guard),
-                bucket,
-                "bucket metadata authoritative snapshot",
-                load_bucket_metadata_parse_with_presence(Arc::clone(&store), bucket, true),
-            )),
-        );
-        match bucket_info_res {
+        match await_bucket_namespace_operation(Some(guard), bucket, "bucket metadata snapshot existence check", async {
+            self.object_store()
+                .get_bucket_info_from_sets_at_read_quorum(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
+                .await
+        })
+        .await
+        {
             Ok(_) => {}
             Err(Error::VolumeNotFound) => return Ok(BucketMetadataAuthority::MissingBucket),
             Err(err) => return Err(err),
         }
 
-        let (metadata, persisted) = metadata_res?;
+        let (metadata, persisted) = await_bucket_namespace_operation(
+            Some(guard),
+            bucket,
+            "bucket metadata authoritative snapshot",
+            load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
+        )
+        .await?;
         if persisted && metadata.lock_enabled && !metadata.bucket_creation_committed {
             return Err(Error::ErasureWriteQuorum);
         }
         if persisted {
-            let metadata = Arc::new(metadata);
-            if can_use_local_authority_cache
-                && metadata.bucket_incarnation_sidecar
-                && !metadata.bucket_incarnation_id.is_nil()
-                && current_local_meta_mutation_gen() == gen_before
-                && store.all_disks_local_and_online()
-            {
-                self.local_authority_cache
-                    .lock()
-                    .unwrap_or_else(std::sync::PoisonError::into_inner)
-                    .insert(bucket.to_string(), (gen_before, Arc::clone(&metadata)));
-            }
-            Ok(BucketMetadataAuthority::Authoritative(metadata))
+            Ok(BucketMetadataAuthority::Authoritative(Arc::new(metadata)))
         } else {
             Ok(BucketMetadataAuthority::Fabricated)
         }
@@ -3175,7 +3149,7 @@ impl BucketMetadataSys {
         }
     }
 
-    pub async fn get_lifecycle_config(&self, bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
+    pub async fn get_lifecycle_config(&self, bucket: &str) -> Result<(PersistedLifecycleConfiguration, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 
         match ConfigState::of(&bm.lifecycle_config_xml, &bm.lifecycle_config).require(bucket, BUCKET_LIFECYCLE_CONFIG)? {
@@ -3298,7 +3272,7 @@ impl BucketMetadataSys {
         }))
     }
 
-    pub async fn get_replication_config(&self, bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
+    pub async fn get_replication_config(&self, bucket: &str) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 
         if !bm.replication_config_xml.is_empty() && bm.replication_config.is_none() {
@@ -3400,6 +3374,7 @@ mod tests {
     };
     use crate::bucket::target::{BucketTarget, BucketTargetType, Credentials};
     use crate::config::com::read_config;
+    use crate::disk::{DiskAPI as _, RUSTFS_META_TMP_BUCKET};
     use crate::storage_api_contracts::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
     use byteorder::{ByteOrder as _, LittleEndian};
     use serial_test::serial;
@@ -3428,7 +3403,7 @@ mod tests {
         ),
         (
             BUCKET_TAGGING_CONFIG,
-            r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-rust</Value></Tag></TagSet></Tagging>"#.as_bytes(),
+            r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
         ),
         (BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
         (
@@ -3454,6 +3429,180 @@ mod tests {
         ),
     ];
 
+    #[tokio::test]
+    #[serial(storage_class_env)]
+    async fn persistence_entry_failures_revoke_pending_creation_before_cache_publish() {
+        let (dirs, store) = isolated_store_over_temp_disks().await;
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let sys = bucket_metadata_sys_of(&store.ctx).expect("instance metadata system");
+        let sys = sys.read().await.clone();
+        let options = MakeBucketOptions {
+            lock_enabled: true,
+            ..Default::default()
+        };
+        for entry in ["save", "persist", "persist-new", "commit"] {
+            let bucket = format!("revoke-{entry}-{}", Uuid::new_v4());
+            store.fail_next_bucket_creation_after_intent_for_test(&bucket);
+            store
+                .make_bucket(&bucket, &options)
+                .await
+                .expect_err("stop the real producer before physical publication");
+            sys.pending_bucket_creation(&bucket)
+                .await
+                .expect("the producer must leave a real live creation permission");
+            let (intent, persisted) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
+                .await
+                .expect("read the original producer intent");
+            assert!(persisted, "{entry}: the complete original intent must be persisted");
+            let original_intent = intent.marshal_msg().expect("encode the original intent");
+
+            // Keep the creation fence order used by the real producer.
+            let lifecycle_guard = store
+                .acquire_bucket_lifecycle_write_lock(&bucket)
+                .await
+                .expect("hold the bucket lifecycle fence");
+            let transaction_guard = acquire_bucket_metadata_transaction_lock_in(&store.ctx, &bucket)
+                .await
+                .expect("hold the metadata transaction fence");
+            let namespace_lock = store
+                .new_ns_lock(&bucket, &bucket)
+                .await
+                .expect("create the exact bucket namespace lock");
+            let namespace_guard = namespace_lock
+                .get_write_lock(crate::set_disk::get_lock_acquire_timeout())
+                .await
+                .expect("hold the exact bucket namespace fence");
+
+            // A file at each native staging root causes actual IO to fail before
+            // metadata can reach the cache-publishing set call. Preserve the
+            // original staging directories so no fixture data is discarded.
+            for dir in &dirs {
+                let staging = dir.path().join(RUSTFS_META_TMP_BUCKET);
+                let saved = dir.path().join(".rustfs.sys/revocation-saved-tmp");
+                std::fs::rename(&staging, &saved).expect("preserve the native staging directory");
+                std::fs::write(&staging, b"block metadata staging with a non-directory")
+                    .expect("install an actual non-directory IO obstruction");
+            }
+            let result = match entry {
+                "save" => sys.save(intent).await,
+                "persist" => sys.persist_and_set(intent).await,
+                "persist-new" => sys.persist_new_and_set(intent).await,
+                "commit" => sys.commit_bucket_metadata(intent).await,
+                _ => unreachable!(),
+            };
+            for dir in &dirs {
+                let staging = dir.path().join(RUSTFS_META_TMP_BUCKET);
+                let saved = dir.path().join(".rustfs.sys/revocation-saved-tmp");
+                std::fs::remove_file(&staging).expect("remove only the fixture's staging obstruction");
+                std::fs::rename(&saved, &staging).expect("restore the complete original staging directory");
+            }
+            drop((namespace_guard, transaction_guard, lifecycle_guard));
+            result.expect_err("native persistence must fail before its downstream cache set");
+            assert!(
+                sys.pending_bucket_creation(&bucket).await.is_none(),
+                "{entry}: the entry revoker must retire live permission even when persistence fails"
+            );
+            let (after_failure, present) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
+                .await
+                .expect("read the intent after the failed persistence entry");
+            assert!(present, "{entry}: a failed persistence entry must retain the original metadata");
+            assert_eq!(
+                after_failure
+                    .marshal_msg()
+                    .expect("encode the original intent after persistence failure"),
+                original_intent,
+                "{entry}: native IO failure must not change the original generation or complete intent"
+            );
+            let error = store
+                .make_bucket(&bucket, &options)
+                .await
+                .expect_err("a failed ordinary persistence entry cannot preserve creation permission");
+            assert!(error.to_string().contains("administrator recovery"), "{entry}: {error}");
+            let (after_retry, _) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
+                .await
+                .expect("read the original intent after the rejected retry");
+            assert_eq!(
+                after_retry.marshal_msg().expect("encode the intent after the rejected retry"),
+                original_intent,
+                "{entry}: the rejected retry must preserve the complete original intent"
+            );
+            for disk in store.pools[0].disk_set[0].disks.read().await.iter() {
+                assert!(
+                    matches!(
+                        disk.as_ref().expect("native disk").stat_volume(&bucket).await,
+                        Err(crate::disk::error::DiskError::VolumeNotFound)
+                    ),
+                    "{entry}: failed persistence and the rejected retry must not publish a physical volume"
+                );
+            }
+        }
+    }
+
+    #[tokio::test]
+    #[serial(storage_class_env)]
+    async fn metadata_reset_revokes_real_pending_creation_without_changing_persisted_intent() {
+        let (_dirs, store) = isolated_store_over_temp_disks().await;
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = format!("revoke-reset-{}", Uuid::new_v4());
+        let options = MakeBucketOptions {
+            lock_enabled: true,
+            ..Default::default()
+        };
+        store.fail_next_bucket_creation_after_intent_for_test(&bucket);
+        store
+            .make_bucket(&bucket, &options)
+            .await
+            .expect_err("stop the real producer before physical publication");
+        let sys = bucket_metadata_sys_of(&store.ctx).expect("instance metadata system");
+        let mut sys = sys.read().await.clone();
+        sys.pending_bucket_creation(&bucket)
+            .await
+            .expect("the real producer must leave live creation permission before reset");
+        let (intent, persisted) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
+            .await
+            .expect("read the original intent before cache reset");
+        assert!(persisted, "cache reset starts from a complete real persisted intent");
+        let original_intent = intent.marshal_msg().expect("encode the original reset intent");
+        sys._reset().await;
+        assert!(
+            sys.pending_bucket_creation(&bucket).await.is_none(),
+            "reset must revoke the producer-issued permission shared with the instance metadata system"
+        );
+        let (after_reset, present) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
+            .await
+            .expect("read the original intent after cache reset");
+        assert!(present, "cache reset must retain the persisted intent");
+        assert_eq!(
+            after_reset.marshal_msg().expect("encode the intent after cache reset"),
+            original_intent,
+            "cache reset must leave the original generation and complete persisted intent unchanged"
+        );
+        let error = store
+            .make_bucket(&bucket, &options)
+            .await
+            .expect_err("reset cannot regrant the producer's old creation permission");
+        assert!(error.to_string().contains("administrator recovery"), "{error}");
+        let (after_retry, _) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
+            .await
+            .expect("read the intent after the reset-induced retry rejection");
+        assert_eq!(
+            after_retry
+                .marshal_msg()
+                .expect("encode the intent after the rejected reset retry"),
+            original_intent,
+            "the reset-induced rejection must preserve the original complete intent"
+        );
+        for disk in store.pools[0].disk_set[0].disks.read().await.iter() {
+            assert!(
+                matches!(
+                    disk.as_ref().expect("native disk").stat_volume(&bucket).await,
+                    Err(crate::disk::error::DiskError::VolumeNotFound)
+                ),
+                "cache reset and the rejected retry must not publish a physical volume"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn g_d3_003_new_writer_replication_loads_without_fail_closed_state() {
         let (dirs, store) = isolated_store_over_temp_disks().await;
@@ -3642,7 +3791,7 @@ mod tests {
         let mut versioned = BucketMetadata::new("authoritative-versioned");
         versioned.versioning_config_xml = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec();
         versioned.versioning_config = Some(VersioningConfiguration {
-            status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
+            status: Some("Enabled".to_owned()),
             ..Default::default()
         });
         sys.set("authoritative-versioned".to_string(), Arc::new(versioned)).await;
@@ -3765,25 +3914,24 @@ mod tests {
         else {
             panic!("legacy lock-enabled metadata must remain configured");
         };
-        assert_eq!(
-            config.object_lock_enabled.as_ref().map(|value| value.as_str()),
-            Some(ObjectLockEnabled::ENABLED)
-        );
+        assert_eq!(config.object_lock_enabled.as_deref(), Some("Enabled"));
         assert!(config.rule.is_none());
     }
 
     #[test]
     fn authoritative_object_lock_state_rejects_invalid_retention_periods() {
-        use s3s::dto::{DefaultRetention, ObjectLockRule};
+        use rustfs_gateway_types::persistence::{
+            PersistedDefaultRetention as DefaultRetention, PersistedObjectLockRule as ObjectLockRule,
+        };
 
         for (days, years) in [(Some(0), None), (Some(-1), None), (Some(1), Some(1)), (None, None)] {
             let mut metadata = BucketMetadata::new("invalid-object-lock");
             metadata.object_lock_config = Some(ObjectLockConfiguration {
-                object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+                object_lock_enabled: Some("Enabled".to_owned()),
                 rule: Some(ObjectLockRule {
                     default_retention: Some(DefaultRetention {
                         days,
-                        mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
+                        mode: Some("COMPLIANCE".to_owned()),
                         years,
                     }),
                 }),
@@ -3798,9 +3946,11 @@ mod tests {
 
     #[test]
     fn authoritative_object_lock_state_rejects_invalid_enabled_mode_and_rule() {
-        use s3s::dto::{DefaultRetention, ObjectLockRule};
+        use rustfs_gateway_types::persistence::{
+            PersistedDefaultRetention as DefaultRetention, PersistedObjectLockRule as ObjectLockRule,
+        };
 
-        for enabled in [None, Some(ObjectLockEnabled::from("Disabled".to_string()))] {
+        for enabled in [None, Some("Disabled".to_string())] {
             let mut metadata = BucketMetadata::new("invalid-object-lock-enabled");
             metadata.lock_enabled = true;
             metadata.object_lock_config = Some(ObjectLockConfiguration {
@@ -3812,11 +3962,11 @@ mod tests {
 
         let mut invalid_mode = BucketMetadata::new("invalid-object-lock-mode");
         invalid_mode.object_lock_config = Some(ObjectLockConfiguration {
-            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            object_lock_enabled: Some("Enabled".to_owned()),
             rule: Some(ObjectLockRule {
                 default_retention: Some(DefaultRetention {
                     days: Some(1),
-                    mode: Some(ObjectLockRetentionMode::from("invalid".to_string())),
+                    mode: Some("invalid".to_string()),
                     years: None,
                 }),
             }),
@@ -3825,7 +3975,7 @@ mod tests {
 
         let mut missing_retention = BucketMetadata::new("missing-object-lock-retention");
         missing_retention.object_lock_config = Some(ObjectLockConfiguration {
-            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            object_lock_enabled: Some("Enabled".to_owned()),
             rule: Some(ObjectLockRule { default_retention: None }),
         });
         assert!(object_lock_config_state_from_authoritative_metadata(&missing_retention).is_err());
@@ -3835,7 +3985,7 @@ mod tests {
     fn object_lock_ever_enabled_marker_survives_config_removal() {
         let mut metadata = BucketMetadata::new("lock-config-removed");
         let config = ObjectLockConfiguration {
-            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            object_lock_enabled: Some("Enabled".to_owned()),
             rule: None,
         };
         metadata
@@ -3853,10 +4003,7 @@ mod tests {
         else {
             panic!("an ever-enabled bucket must remain Object Lock configured");
         };
-        assert_eq!(
-            config.object_lock_enabled.as_ref().map(ObjectLockEnabled::as_str),
-            Some(ObjectLockEnabled::ENABLED)
-        );
+        assert_eq!(config.object_lock_enabled.as_deref(), Some("Enabled"));
     }
 
     #[tokio::test]
@@ -3872,7 +4019,7 @@ mod tests {
 
         let node = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore.clone())));
         let config = ObjectLockConfiguration {
-            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            object_lock_enabled: Some("Enabled".to_owned()),
             rule: None,
         };
         update_with_sys(
@@ -4556,13 +4703,8 @@ mod tests {
             std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
         }
         let mut bm = BucketMetadata::new(bucket);
-        if config_file == crate::bucket::metadata::BUCKET_TAGGING_CONFIG {
-            // Model legacy persisted corruption without passing the write-time validator.
-            bm.tagging_config_xml = raw.to_vec();
-        } else {
-            bm.update_config(config_file, raw.to_vec())
-                .expect("raw config should be stored");
-        }
+        bm.update_config(config_file, raw.to_vec())
+            .expect("raw config should be stored");
         sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
     }
 
@@ -4858,7 +5000,7 @@ mod tests {
     #[tokio::test]
     async fn update_config_with_persists_tagging_rewrite_across_disk_reload() {
         use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
-        use s3s::dto::Tag;
+        use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
 
         let (dirs, ecstore) = isolated_store_over_temp_disks().await;
 
@@ -5022,7 +5164,7 @@ mod tests {
     #[tokio::test]
     async fn concurrent_update_config_with_calls_do_not_lose_writes() {
         use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
-        use s3s::dto::Tag;
+        use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
 
         let (dirs, ecstore) = isolated_store_over_temp_disks().await;
         let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
@@ -5122,7 +5264,7 @@ mod tests {
         let mut persisted = BucketMetadata::new("reload-bucket");
         persisted.policy_config_json = b"persisted-marker".to_vec();
         let lock_config = ObjectLockConfiguration {
-            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            object_lock_enabled: Some("Enabled".to_owned()),
             rule: None,
         };
         persisted
@@ -5215,61 +5357,6 @@ mod tests {
         );
     }
 
-    #[tokio::test]
-    #[serial]
-    async fn validated_config_update_rejects_a_different_bucket_incarnation() {
-        let (_dirs, store) = isolated_store_over_temp_disks().await;
-        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
-        let bucket = "catalog-cleanup-incarnation";
-        store
-            .make_bucket(bucket, &MakeBucketOptions::default())
-            .await
-            .expect("create bucket");
-        let incarnation = store
-            .bucket_incarnation_id_from_disk(bucket)
-            .await
-            .expect("read bucket incarnation");
-        let called = std::sync::atomic::AtomicBool::new(false);
-        let result = store
-            .update_bucket_metadata_config_for_incarnation_validated(
-                bucket,
-                "quota.json",
-                br#"{"quota":1}"#.to_vec(),
-                Uuid::new_v4(),
-                || {
-                    called.store(true, std::sync::atomic::Ordering::SeqCst);
-                    Ok(())
-                },
-            )
-            .await;
-        assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
-        assert!(
-            !called.load(std::sync::atomic::Ordering::SeqCst),
-            "incarnation must be checked before the external fence"
-        );
-        let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
-            .await
-            .expect("read unchanged bucket metadata");
-        assert!(
-            metadata.quota_config_json.is_empty(),
-            "a stale cleanup must not modify the replacement bucket"
-        );
-        store
-            .update_bucket_metadata_config_for_incarnation_validated(
-                bucket,
-                "quota.json",
-                br#"{"quota":1}"#.to_vec(),
-                incarnation,
-                || Ok(()),
-            )
-            .await
-            .expect("the current incarnation permits the validated update");
-        let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
-            .await
-            .expect("read updated bucket metadata");
-        assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
-    }
-
     #[tokio::test]
     #[serial]
     async fn quota_admission_rejects_stale_cached_no_quota() {
@@ -5456,7 +5543,7 @@ mod tests {
     #[serial]
     async fn bucket_delete_waits_for_config_mutation_fence() {
         use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
-        use s3s::dto::Tag;
+        use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
 
         let (_dirs, store) = isolated_store_over_temp_disks().await;
         init_bucket_metadata_sys(store.clone(), Vec::new()).await;
@@ -5743,8 +5830,7 @@ mod tests {
         // update only needs one interleaving to show up.
         const ROUNDS: usize = 8;
         for round in 0..ROUNDS {
-            let tagging =
-                format!("<Tagging><TagSet><Tag><Key>round</Key><Value>{round}</Value></Tag></TagSet></Tagging>").into_bytes();
+            let tagging = format!("<Tagging><Round>{round}</Round></Tagging>").into_bytes();
             let policy = format!(r#"{{"Version":"2012-10-17","Round":{round}}}"#).into_bytes();
 
             let start = Arc::new(tokio::sync::Barrier::new(2));
@@ -5817,7 +5903,7 @@ mod tests {
 
         let blocked = tokio::spawn({
             let node_b = node_b.clone();
-            async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging><TagSet/></Tagging>".to_vec()).await }
+            async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec()).await }
         });
 
         // Long enough for the write to have finished had it not waited: the

crates/ecstore/src/bucket/object_lock/mod.rs

@@ -17,15 +17,8 @@ pub mod objectlock_sys;
 pub mod types;
 
 /// Whether a bucket Object Lock configuration has locking enabled. The
-/// serving-layer `ObjectLockConfiguration` DTO implements this in
-/// the bucket-metadata module, which owns the persisted configuration type
-/// during the s3s ratchet migration (rustfs/backlog#1842).
+/// persisted `ObjectLockConfiguration` implements this in the bucket-metadata
+/// module, which owns the persisted configuration type.
 pub trait ObjectLockApi {
     fn enabled(&self) -> bool;
 }
-
-/// Whether a legal-hold status value is one of the two valid wire values.
-/// Implemented for the serving-layer DTO in the bucket-metadata module.
-pub trait ObjectLockStatusExt {
-    fn valid(&self) -> bool;
-}

crates/ecstore/src/bucket/quota/checker/tests/reporting_retirement.rs

@@ -119,21 +119,6 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
             // Exercise the real storage entry point before testing the new
             // reporting semantics: the same ledger must admit and refuse
             // these writes even when a scanner floor claims a million bytes.
-            if protocol == 1 {
-                let mut bootstrap = PutObjReader::from_vec(vec![0x5a; 512]);
-                assert!(
-                    matches!(
-                        store
-                            .put_object(&bucket, "bootstrap", &mut bootstrap, &ObjectOptions::default())
-                            .await,
-                        Err(StorageError::SlowDown)
-                    ),
-                    "a missing authoritative ledger must not admit a write from scanner reporting"
-                );
-                crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
-                    .await
-                    .expect("bootstrap authoritative usage through the production worker");
-            }
             let mut too_large = PutObjReader::from_vec(vec![0x5a; 8192]);
             assert!(
                 matches!(

crates/ecstore/src/bucket/quota/reservation.rs

@@ -19,7 +19,7 @@ use crate::data_usage::compute_bucket_usage;
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::disk::{DiskAPI, error::DiskError};
 use crate::error::{Result, StorageError, is_err_object_not_found, is_err_version_not_found};
-use crate::object_api::{ObjectInfo, ObjectLockConfigSnapshot, ObjectOptions, QuotaAdmission};
+use crate::object_api::{ObjectInfo, ObjectOptions, QuotaAdmission};
 use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
 use crate::storage_api_contracts::namespace::NamespaceLocking;
 use crate::storage_api_contracts::{list::ListOperations as _, object::ObjectOperations};
@@ -28,7 +28,7 @@ use futures::{StreamExt, stream};
 use rustfs_lock::NamespaceLockGuard;
 use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
-use std::collections::{BTreeMap, HashSet};
+use std::collections::BTreeMap;
 use std::sync::Arc;
 use std::time::Duration;
 use time::OffsetDateTime;
@@ -36,8 +36,6 @@ use tracing::warn;
 use uuid::Uuid;
 
 const QUOTA_LEDGER_FORMAT_VERSION: u8 = 1;
-const RECONCILE_RETRY_SECONDS: i64 = 30;
-const EVENT_QUOTA_RECONCILIATION: &str = "quota_reconciliation";
 const MAX_ORPHANS_REAPED_PER_WRITE: usize = 64;
 const MAX_ORPHAN_PROBES_PER_WRITE: usize = 128;
 const ORPHAN_PROBE_CONCURRENCY: usize = 32;
@@ -98,10 +96,6 @@ struct QuotaLedger {
     version: u8,
     bucket_incarnation: Uuid,
     quota_revision_unix_nanos: i128,
-    /// Changes on every durable write, including zero-growth reservations.
-    /// Missing on an older writer's output, which invalidates a concurrent scan.
-    #[serde(default)]
-    writer_revision: Uuid,
     accounted_usage: u64,
     reservations: BTreeMap<Uuid, PersistedReservation>,
     #[serde(default)]
@@ -116,7 +110,6 @@ impl QuotaLedger {
             version: QUOTA_LEDGER_FORMAT_VERSION,
             bucket_incarnation,
             quota_revision_unix_nanos: quota_revision.unix_timestamp_nanos(),
-            writer_revision: Uuid::nil(),
             accounted_usage,
             reservations: BTreeMap::new(),
             reconcile_required: false,
@@ -492,23 +485,22 @@ impl QuotaContext {
             let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_data.ledger_object).await?;
             let ledger_guard = Arc::new(ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?);
             fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &ledger_data.ledger_object, None).await?;
-            let mut ledger =
-                load_current_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, bucket_incarnation, quota_revision)
-                    .await?;
-            let job = ReconcileJob::new(Arc::clone(&store), ledger_data.bucket.clone(), bucket_incarnation, quota_revision);
-            let Some(mut ledger) = ledger.take() else {
-                // Bootstrap has no authoritative usage yet. Release every
-                // request-owned fence by returning; the worker owns no request.
-                job.schedule();
-                return Err(StorageError::SlowDown);
-            };
+            let mut ledger = load_current_ledger_locked(
+                Arc::clone(&store),
+                &ledger_data.bucket,
+                &ledger_data.ledger_object,
+                bucket_incarnation,
+                quota_revision,
+            )
+            .await?;
+
             let growth = reservation.growth();
-            let current_usage = ledger.admitted_usage()?;
-            let expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
-            if ledger.reconcile_required
-                || (growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial())
-            {
-                job.schedule();
+            let mut current_usage = ledger.admitted_usage()?;
+            let mut expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
+            if growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial() {
+                reconcile_exact(&store, &ledger_data.bucket, &mut ledger).await?;
+                current_usage = ledger.admitted_usage()?;
+                expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
             }
             if growth > 0 && expected_usage > quota_limit {
                 return Err(StorageError::QuotaExceeded {
@@ -526,7 +518,7 @@ impl QuotaContext {
                 });
             }
             ledger.reserve(operation_id, reservation)?;
-            save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &mut ledger, &ledger_guard).await?;
+            save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &ledger, &ledger_guard).await?;
 
             Ok(QuotaReservation {
                 ledger: Some(ReservationLedgerData::Single(ledger_data)),
@@ -1090,7 +1082,7 @@ async fn mark_commit_started(data: &ReservationLedgerData) -> Result<()> {
         fence_namespace_mutations(&data.store, RUSTFS_META_BUCKET, &data.ledger_object, None).await?;
         let mut ledger = load_ledger_locked(Arc::clone(&data.store), &data.ledger_object).await?;
         ledger.mark_commit_started(data.operation_id, &data.reservation)?;
-        save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &mut ledger, &ledger_guard).await
+        save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &ledger, &ledger_guard).await
     })
     .await
     .map_err(|err| StorageError::other_with_context("quota commit marker task failed", err))?
@@ -1121,7 +1113,7 @@ async fn settle(data: &ReservationLedgerData, committed: bool) -> Result<()> {
         } else {
             ledger.abort(operation_id, &reservation)?;
         }
-        save_ledger_locked(Arc::clone(&store), &ledger_object, &mut ledger, &ledger_guard).await
+        save_ledger_locked(Arc::clone(&store), &ledger_object, &ledger, &ledger_guard).await
     })
     .await
     .map_err(|err| StorageError::other_with_context("quota ledger settlement task failed", err))?
@@ -1129,15 +1121,22 @@ async fn settle(data: &ReservationLedgerData, committed: bool) -> Result<()> {
 
 async fn load_current_ledger_locked(
     store: Arc<ECStore>,
+    bucket: &str,
     ledger_object: &str,
     bucket_incarnation: Uuid,
     quota_revision: OffsetDateTime,
-) -> Result<Option<QuotaLedger>> {
-    match load_ledger_locked(store, ledger_object).await {
-        Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(Some(ledger)),
-        Ok(ledger) if ledger.reservations.is_empty() => Ok(None),
+) -> Result<QuotaLedger> {
+    match load_ledger_locked(Arc::clone(&store), ledger_object).await {
+        Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(ledger),
+        Ok(ledger) if ledger.reservations.is_empty() && !ledger.reconcile_required => {
+            let usage = exact_bucket_usage(&store, bucket).await?;
+            Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage))
+        }
         Ok(_) => Err(StorageError::PartMissingOrCorrupt),
-        Err(StorageError::ConfigNotFound) => Ok(None),
+        Err(StorageError::ConfigNotFound) => {
+            let usage = exact_bucket_usage(&store, bucket).await?;
+            Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage))
+        }
         Err(err) => Err(err),
     }
 }
@@ -1246,11 +1245,15 @@ async fn reap_stale_reservations(store: Arc<ECStore>, bucket: &str, ledger_objec
     for (object, target) in orphan_commit_targets {
         fence_namespace_mutations(&store, bucket, &object, target).await?;
     }
-    let removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?;
+    let mut removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?;
+    if ledger.reservations.is_empty() && ledger.reconcile_required {
+        reconcile_exact(&store, bucket, &mut ledger).await?;
+        removed = true;
+    }
     if !removed && !cursor_changed {
         return Ok(());
     }
-    save_ledger_locked(store, ledger_object, &mut ledger, &ledger_guard).await
+    save_ledger_locked(store, ledger_object, &ledger, &ledger_guard).await
 }
 
 fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid]) -> Result<bool> {
@@ -1270,217 +1273,13 @@ fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid])
     Ok(removed)
 }
 
-#[derive(Debug, Serialize, Deserialize)]
-struct ReconcileControl {
-    bucket_incarnation: Uuid,
-    quota_revision_unix_nanos: i128,
-    next_attempt_at: i64,
-}
-
-#[derive(Clone)]
-struct ReconcileJob {
-    store: Arc<ECStore>,
-    bucket: String,
-    bucket_incarnation: Uuid,
-    quota_revision: OffsetDateTime,
-}
-
-type ReconcileKey = (Uuid, String, Uuid, i128);
-
-fn active_reconciliations() -> &'static std::sync::Mutex<HashSet<ReconcileKey>> {
-    static ACTIVE: std::sync::OnceLock<std::sync::Mutex<HashSet<ReconcileKey>>> = std::sync::OnceLock::new();
-    ACTIVE.get_or_init(Default::default)
-}
-
-struct ActiveReconciliation(ReconcileKey);
-
-impl Drop for ActiveReconciliation {
-    fn drop(&mut self) {
-        active_reconciliations()
-            .lock()
-            .unwrap_or_else(std::sync::PoisonError::into_inner)
-            .remove(&self.0);
-    }
-}
-
-impl ReconcileJob {
-    fn new(store: Arc<ECStore>, bucket: String, bucket_incarnation: Uuid, quota_revision: OffsetDateTime) -> Self {
-        Self {
-            store,
-            bucket,
-            bucket_incarnation,
-            quota_revision,
-        }
-    }
-
-    fn schedule(self) {
-        let key = (
-            self.store.id,
-            self.bucket.clone(),
-            self.bucket_incarnation,
-            self.quota_revision.unix_timestamp_nanos(),
-        );
-        if !active_reconciliations()
-            .lock()
-            .unwrap_or_else(std::sync::PoisonError::into_inner)
-            .insert(key.clone())
-        {
-            return;
-        }
-        let owner = ActiveReconciliation(key);
-        // The owner carries no object, metadata, lifecycle or reservation guard.
-        // Cancellation of the rejected request cannot cancel its reconciliation.
-        tokio::spawn(async move {
-            let _owner = owner;
-            if let Err(error) = self.run().await {
-                warn!(event = EVENT_QUOTA_RECONCILIATION, component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_QUOTA, state = "deferred", bucket = %self.bucket,
-                    error = %error, "Quota reconciliation deferred");
-            }
-        });
-    }
-
-    fn control_object(&self) -> String {
-        format!(
-            "{}.reconcile/{}/{}",
-            ledger_object(&self.bucket),
-            self.bucket_incarnation,
-            self.quota_revision.unix_timestamp_nanos()
-        )
-    }
-
-    async fn save_control(&self, object: &str, control: &ReconcileControl, guard: &Arc<NamespaceLockGuard>) -> Result<()> {
-        if guard.is_lock_lost() {
-            return Err(StorageError::SlowDown);
-        }
-        let mut opts = ObjectOptions {
-            max_parity: true,
-            no_lock: true,
-            ..Default::default()
-        };
-        let _ = opts.set_quota_admission(0, u64::MAX);
-        opts.add_owned_write_lock(Arc::clone(guard), RUSTFS_META_BUCKET, object);
-        opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
-        save_config_with_opts(Arc::clone(&self.store), object, serde_json::to_vec(control)?, &opts).await
-    }
-
-    async fn run(&self) -> Result<()> {
-        let control_object = self.control_object();
-        let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &control_object).await?;
-        let Ok(guard) = lock.get_write_lock_quiet(Duration::from_millis(50)).await else {
-            return Ok(());
-        };
-        let guard = Arc::new(guard);
-        let now = now_unix();
-        match read_config_no_lock(Arc::clone(&self.store), &control_object).await {
-            Ok(bytes) => {
-                let control: ReconcileControl = serde_json::from_slice(&bytes)?;
-                if control.bucket_incarnation != self.bucket_incarnation
-                    || control.quota_revision_unix_nanos != self.quota_revision.unix_timestamp_nanos()
-                {
-                    return Err(StorageError::PartMissingOrCorrupt);
-                }
-                if control.next_attempt_at > now {
-                    return Ok(());
-                }
-            }
-            Err(StorageError::ConfigNotFound) => {}
-            Err(error) => return Err(error),
-        }
-        fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &control_object, None).await?;
-        let mut control = ReconcileControl {
-            bucket_incarnation: self.bucket_incarnation,
-            quota_revision_unix_nanos: self.quota_revision.unix_timestamp_nanos(),
-            next_attempt_at: now.saturating_add(RECONCILE_RETRY_SECONDS),
-        };
-        // Persist the throttle before work, including bootstrap and failed scans.
-        self.save_control(&control_object, &control, &guard).await?;
-        let result = self.scan_and_publish(&guard).await;
-        control.next_attempt_at = now_unix().saturating_add(RECONCILE_RETRY_SECONDS);
-        self.save_control(&control_object, &control, &guard).await?;
-        result
-    }
-
-    /// Take a short authoritative configuration snapshot without recursively
-    /// acquiring a reader behind a queued metadata/lifecycle writer.
-    async fn configuration_snapshot(&self) -> Result<Arc<ObjectLockConfigSnapshot>> {
-        let snapshot = self.store.object_lock_config_snapshot_for_single_put(&self.bucket).await?;
-        let guard = snapshot
-            .metadata_transaction_guard_for(self.store.id, &self.bucket, Some(self.bucket_incarnation))
-            .ok_or(StorageError::PreconditionFailed)?;
-        let opts = ObjectOptions {
-            expected_bucket_incarnation_id: Some(self.bucket_incarnation),
-            object_lock_config_snapshot: Some(Arc::clone(&snapshot)),
-            ..Default::default()
-        };
-        let (quota, incarnation, revision) =
-            metadata_sys::get_quota_config_and_incarnation_from_disk_for_options_in(&self.store.ctx, &self.bucket, &opts, &guard)
-                .await?;
-        if incarnation != self.bucket_incarnation
-            || revision != self.quota_revision
-            || !quota.as_ref().is_some_and(|quota| {
-                quota.uses_durable_reservations() && quota.reservation_protocol != Some(QUOTA_RESERVATION_PROTOCOL_V2)
-            })
-        {
-            return Err(StorageError::PreconditionFailed);
-        }
-        Ok(snapshot)
-    }
-
-    async fn scan_and_publish(&self, owner: &NamespaceLockGuard) -> Result<()> {
-        let ledger_object = ledger_object(&self.bucket);
-        let expected_revision = {
-            // Lock order: reconcile owner, lifecycle, metadata, ledger.
-            // Request paths never wait for the reconcile owner.
-            let _configuration = self.configuration_snapshot().await?;
-            let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?;
-            let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
-            match load_ledger_locked(Arc::clone(&self.store), &ledger_object).await {
-                Ok(mut ledger) => {
-                    if !ledger.reservations.is_empty() {
-                        return Ok(());
-                    }
-                    // Seed a non-nil token even for a legacy ledger. An old
-                    // node drops this field on write and cannot pass the CAS.
-                    fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &ledger_object, None).await?;
-                    save_ledger_locked(Arc::clone(&self.store), &ledger_object, &mut ledger, &guard).await?;
-                    Some(ledger.writer_revision)
-                }
-                Err(StorageError::ConfigNotFound) => None,
-                Err(error) => return Err(error),
-            }
-        };
-        #[cfg(test)]
-        tests::pause_reconciliation(&self.bucket, tests::ReconcilePauseStage::BeforeScan).await?;
-        // Only the independent reconcile owner is held during this whole-bucket
-        // walk. Concurrent reserve/settle/reap writes invalidate its result.
-        let usage = exact_bucket_usage(&self.store, &self.bucket).await?;
-        #[cfg(test)]
-        tests::pause_reconciliation(&self.bucket, tests::ReconcilePauseStage::BeforePublish).await?;
-        let configuration = self.configuration_snapshot().await?;
-        let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?;
-        let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
-        let unchanged = match load_ledger_locked(Arc::clone(&self.store), &ledger_object).await {
-            Ok(ledger) => {
-                expected_revision == Some(ledger.writer_revision)
-                    && !ledger.writer_revision.is_nil()
-                    && ledger.reservations.is_empty()
-            }
-            Err(StorageError::ConfigNotFound) => expected_revision.is_none(),
-            Err(error) => return Err(error),
-        };
-        if !unchanged {
-            return Ok(());
-        }
-        if owner.is_lock_lost()
-            || !configuration.is_valid_for_destructive_put(self.store.id, &self.bucket, self.bucket_incarnation)
-        {
-            return Err(StorageError::SlowDown);
-        }
-        fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &ledger_object, None).await?;
-        let mut ledger = QuotaLedger::new(self.bucket_incarnation, self.quota_revision, usage);
-        save_ledger_locked(Arc::clone(&self.store), &ledger_object, &mut ledger, &guard).await
+async fn reconcile_exact(store: &Arc<ECStore>, bucket: &str, ledger: &mut QuotaLedger) -> Result<()> {
+    if !ledger.reservations.is_empty() {
+        return Err(StorageError::PartMissingOrCorrupt);
     }
+    ledger.accounted_usage = exact_bucket_usage(store, bucket).await?;
+    ledger.reconcile_required = false;
+    Ok(())
 }
 
 async fn exact_bucket_usage(store: &Arc<ECStore>, bucket: &str) -> Result<u64> {
@@ -2136,7 +1935,7 @@ async fn load_ledger_locked(store: Arc<ECStore>, ledger_object: &str) -> Result<
 async fn save_ledger_locked(
     store: Arc<ECStore>,
     ledger_object: &str,
-    ledger: &mut QuotaLedger,
+    ledger: &QuotaLedger,
     ledger_guard: &Arc<NamespaceLockGuard>,
 ) -> Result<()> {
     if ledger_guard.is_lock_lost() {
@@ -2160,71 +1959,9 @@ async fn save_ledger_locked(
     let _ = opts.set_quota_admission(0, u64::MAX);
     opts.add_owned_write_lock(Arc::clone(ledger_guard), RUSTFS_META_BUCKET, ledger_object);
     opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
-    ledger.writer_revision = Uuid::new_v4();
     save_config_with_opts(store, ledger_object, serde_json::to_vec(ledger)?, &opts).await
 }
 
-/// Advance the persisted reconciliation retry deadline for deterministic fixtures.
-/// The production reaper, ownership fence, scan and compare-and-swap still run.
-#[cfg(any(test, feature = "test-util"))]
-pub async fn retry_quota_reconciliation_for_test(store: Arc<ECStore>, bucket: &str) -> Result<()> {
-    let (_, incarnation, revision) = store.bucket_quota_and_incarnation_from_disk(bucket).await?;
-    let job = ReconcileJob::new(Arc::clone(&store), bucket.to_owned(), incarnation, revision);
-    let key = (store.id, bucket.to_owned(), incarnation, revision.unix_timestamp_nanos());
-    tokio::time::timeout(Duration::from_secs(10), async {
-        loop {
-            let active = active_reconciliations()
-                .lock()
-                .unwrap_or_else(std::sync::PoisonError::into_inner)
-                .contains(&key);
-            if !active {
-                break;
-            }
-            tokio::task::yield_now().await;
-        }
-    })
-    .await
-    .map_err(|_| StorageError::SlowDown)?;
-    reap_stale_reservations(Arc::clone(&store), bucket, &ledger_object(bucket)).await?;
-    let object = job.control_object();
-    {
-        let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
-        let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
-        match read_config_no_lock(Arc::clone(&store), &object).await {
-            Ok(bytes) => {
-                let mut control: ReconcileControl = serde_json::from_slice(&bytes)?;
-                control.next_attempt_at = 0;
-                fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &object, None).await?;
-                job.save_control(&object, &control, &guard).await?;
-            }
-            Err(StorageError::ConfigNotFound) => {}
-            Err(error) => return Err(error),
-        }
-    }
-    job.run().await
-}
-
-/// Bootstrap a test bucket through the real background reconciliation path.
-/// Fixtures call this after enabling V1 quota, before issuing writes whose
-/// contract is independent of the initial retryable admission response.
-#[cfg(any(test, feature = "test-util"))]
-pub async fn initialize_quota_ledger_for_test(store: Arc<ECStore>, bucket: &str) -> Result<()> {
-    let (quota, incarnation, revision) = store.bucket_quota_and_incarnation_from_disk(bucket).await?;
-    if !quota
-        .as_ref()
-        .is_some_and(|quota| quota.uses_durable_reservations() && !quota.uses_sharded_reservations())
-    {
-        return Ok(());
-    }
-    ReconcileJob::new(Arc::clone(&store), bucket.to_owned(), incarnation, revision)
-        .run()
-        .await?;
-    match load_current_ledger_locked(store, &ledger_object(bucket), incarnation, revision).await? {
-        Some(_) => Ok(()),
-        None => Err(StorageError::SlowDown),
-    }
-}
-
 #[cfg(any(test, feature = "test-util"))]
 #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
 pub fn fail_next_quota_ledger_save_for_test() {
@@ -2266,434 +2003,6 @@ mod tests {
     use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
     use serial_test::serial;
 
-    #[derive(Clone, Copy, PartialEq, Eq)]
-    pub(super) enum ReconcilePauseStage {
-        BeforeScan,
-        BeforePublish,
-    }
-
-    struct ReconcilePause {
-        bucket: String,
-        stage: ReconcilePauseStage,
-        armed: std::sync::atomic::AtomicBool,
-        scans: std::sync::atomic::AtomicUsize,
-        fail_scan: std::sync::atomic::AtomicBool,
-        reached: tokio::sync::Notify,
-        release: tokio::sync::Notify,
-    }
-
-    static RECONCILE_PAUSE: std::sync::Mutex<Option<Arc<ReconcilePause>>> = std::sync::Mutex::new(None);
-
-    struct ReconcilePauseGuard(Arc<ReconcilePause>);
-
-    impl ReconcilePauseGuard {
-        fn install(bucket: &str, stage: ReconcilePauseStage) -> Self {
-            let pause = Arc::new(ReconcilePause {
-                bucket: bucket.to_owned(),
-                stage,
-                armed: true.into(),
-                scans: 0.into(),
-                fail_scan: false.into(),
-                reached: tokio::sync::Notify::new(),
-                release: tokio::sync::Notify::new(),
-            });
-            *RECONCILE_PAUSE.lock().expect("reconcile hooks") = Some(Arc::clone(&pause));
-            Self(pause)
-        }
-
-        async fn reached(&self) {
-            tokio::time::timeout(Duration::from_secs(10), self.0.reached.notified())
-                .await
-                .expect("reconciliation reached barrier");
-        }
-    }
-
-    impl Drop for ReconcilePauseGuard {
-        fn drop(&mut self) {
-            self.0.release.notify_one();
-            let mut hook = RECONCILE_PAUSE.lock().expect("reconcile hooks");
-            if hook.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.0)) {
-                *hook = None;
-            }
-        }
-    }
-
-    pub(super) async fn pause_reconciliation(bucket: &str, stage: ReconcilePauseStage) -> Result<()> {
-        use std::sync::atomic::Ordering;
-        let hook = RECONCILE_PAUSE
-            .lock()
-            .expect("reconcile hooks")
-            .as_ref()
-            .filter(|hook| hook.bucket == bucket)
-            .cloned();
-        if let Some(hook) = hook {
-            if stage == ReconcilePauseStage::BeforeScan {
-                hook.scans.fetch_add(1, Ordering::SeqCst);
-            }
-            if stage == hook.stage && hook.armed.swap(false, Ordering::SeqCst) {
-                hook.reached.notify_one();
-                hook.release.notified().await;
-            }
-            if stage == ReconcilePauseStage::BeforeScan && hook.fail_scan.load(Ordering::SeqCst) {
-                return Err(StorageError::other("injected quota usage failure"));
-            }
-        }
-        Ok(())
-    }
-
-    async fn reconciliation_fixture(limit: u64) -> (Vec<tempfile::TempDir>, ReconcileJob) {
-        use crate::object_api::PutObjReader;
-        use crate::storage_api_contracts::object::ObjectIO;
-        let (directories, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
-        metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
-        crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
-        let bucket = format!("quota-reconcile-{}", Uuid::new_v4().simple());
-        store
-            .make_bucket(&bucket, &MakeBucketOptions::default())
-            .await
-            .expect("bucket");
-        let mut seed = PutObjReader::from_vec(vec![1; 10]);
-        store
-            .put_object(&bucket, "seed", &mut seed, &ObjectOptions::default())
-            .await
-            .expect("existing usage");
-        store
-            .update_bucket_metadata_config(
-                &bucket,
-                "quota.json",
-                serde_json::to_vec(&super::super::BucketQuota::new(Some(limit))).expect("quota JSON"),
-            )
-            .await
-            .expect("enable quota");
-        let (_, incarnation, revision) = store
-            .bucket_quota_and_incarnation_from_disk(&bucket)
-            .await
-            .expect("quota identity");
-        (directories, ReconcileJob::new(store, bucket, incarnation, revision))
-    }
-
-    async fn quota_reservation(job: &ReconcileJob, old: u64, new: u64) -> Result<QuotaReservation> {
-        begin(&job.store.ctx, &job.bucket, "candidate", &ObjectOptions::default(), 0, 0)
-            .await?
-            .reserve(old, new)
-            .await
-    }
-
-    async fn read_test_ledger(job: &ReconcileJob) -> QuotaLedger {
-        load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket))
-            .await
-            .expect("ledger")
-    }
-
-    async fn drain_reconciliation_scheduler(job: &ReconcileJob) {
-        let key = (
-            job.store.id,
-            job.bucket.clone(),
-            job.bucket_incarnation,
-            job.quota_revision.unix_timestamp_nanos(),
-        );
-        tokio::time::timeout(Duration::from_secs(10), async {
-            while active_reconciliations().lock().expect("active workers").contains(&key) {
-                tokio::task::yield_now().await;
-            }
-        })
-        .await
-        .expect("owned reconciliation finished");
-    }
-
-    async fn expire_reconciliation_backoff(job: &ReconcileJob) {
-        drain_reconciliation_scheduler(job).await;
-        let object = job.control_object();
-        let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
-        let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("owner guard"));
-        let mut control: ReconcileControl =
-            serde_json::from_slice(&read_config_no_lock(Arc::clone(&job.store), &object).await.expect("control"))
-                .expect("control JSON");
-        control.next_attempt_at = 0;
-        fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
-            .await
-            .expect("control fence");
-        job.save_control(&object, &control, &guard).await.expect("advance test clock");
-    }
-
-    async fn wait_for_reconcile_owner(job: &ReconcileJob) {
-        let object = job.control_object();
-        let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
-        let _guard = tokio::time::timeout(Duration::from_secs(10), lock.get_write_lock(get_lock_acquire_timeout()))
-            .await
-            .expect("worker finished")
-            .expect("owner guard");
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn quota_reconciliation_bootstrap_is_single_flight_without_request_guards() {
-        use std::sync::atomic::Ordering;
-        let (_directories, job) = reconciliation_fixture(10).await;
-        let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
-        assert!(
-            matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::SlowDown)),
-            "unknown usage cannot admit a reservation"
-        );
-        pause.reached().await;
-        assert!(
-            matches!(
-                load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
-                Err(StorageError::ConfigNotFound)
-            ),
-            "bootstrap must not publish zero usage"
-        );
-        assert!(matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::SlowDown)));
-        // Bypass the local scheduler, exercising the cross-node ownership boundary.
-        tokio::time::timeout(Duration::from_secs(2), job.run())
-            .await
-            .expect("contender must not wait for a scan")
-            .expect("another owner is active");
-        let lifecycle = tokio::time::timeout(Duration::from_secs(2), job.store.acquire_bucket_lifecycle_write_lock(&job.bucket))
-            .await
-            .expect("scan does not retain lifecycle readers")
-            .expect("lifecycle writer");
-        let metadata = tokio::time::timeout(
-            Duration::from_secs(2),
-            metadata_sys::acquire_bucket_metadata_transaction_lock_in(&job.store.ctx, &job.bucket),
-        )
-        .await
-        .expect("scan does not retain metadata readers")
-        .expect("metadata writer");
-        drop(metadata);
-        drop(lifecycle);
-        pause.0.release.notify_one();
-        wait_for_reconcile_owner(&job).await;
-        let ledger = read_test_ledger(&job).await;
-        assert_eq!(ledger.accounted_usage, 10);
-        assert!(ledger.reservations.is_empty());
-        assert!(!ledger.writer_revision.is_nil());
-        for _ in 0..3 {
-            assert!(matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::QuotaExceeded { .. })));
-            job.run().await.expect("persisted cooldown");
-        }
-        drain_reconciliation_scheduler(&job).await;
-        assert_eq!(
-            pause.0.scans.load(Ordering::SeqCst),
-            1,
-            "denied requests cannot repeatedly scan a full bucket"
-        );
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn quota_reconciliation_rejects_concurrent_zero_growth_settle_reap_and_legacy_writes() {
-        let (_directories, job) = reconciliation_fixture(100).await;
-        job.run().await.expect("bootstrap");
-        for mutation in ["abort", "commit", "reap", "legacy"] {
-            expire_reconciliation_backoff(&job).await;
-            let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
-            let worker = tokio::spawn({
-                let job = job.clone();
-                async move { job.run().await }
-            });
-            pause.reached().await;
-            let initial = read_test_ledger(&job).await.writer_revision;
-            if mutation == "legacy" {
-                let object = ledger_object(&job.bucket);
-                let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("ledger lock");
-                let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("ledger writer"));
-                let mut document = serde_json::to_value(read_test_ledger(&job).await).expect("ledger JSON");
-                document.as_object_mut().expect("ledger map").remove("writer_revision");
-                fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
-                    .await
-                    .expect("ledger fence");
-                let mut opts = ObjectOptions {
-                    max_parity: true,
-                    no_lock: true,
-                    ..Default::default()
-                };
-                let _ = opts.set_quota_admission(0, u64::MAX);
-                opts.add_owned_write_lock(Arc::clone(&guard), RUSTFS_META_BUCKET, &object);
-                opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
-                save_config_with_opts(
-                    Arc::clone(&job.store),
-                    &object,
-                    serde_json::to_vec(&document).expect("old node JSON"),
-                    &opts,
-                )
-                .await
-                .expect("old node drops new field");
-            } else {
-                let mut reservation = quota_reservation(&job, 10, 10)
-                    .await
-                    .expect("zero-growth admission while scanning");
-                let reserved = read_test_ledger(&job).await.writer_revision;
-                assert_ne!(reserved, initial, "even zero-growth reserve must invalidate a scan");
-                reservation.mark_commit_started().await.expect("commit marker");
-                assert_ne!(read_test_ledger(&job).await.writer_revision, reserved);
-                match mutation {
-                    "abort" => reservation.abort().await,
-                    "commit" => reservation.commit().await,
-                    "reap" => {
-                        reservation.defer_after_fence();
-                        reap_stale_reservations(Arc::clone(&job.store), &job.bucket, &ledger_object(&job.bucket))
-                            .await
-                            .expect("reap orphan");
-                    }
-                    _ => unreachable!(),
-                }
-            }
-            let written = read_test_ledger(&job).await;
-            assert!(written.reservations.is_empty());
-            assert_ne!(written.writer_revision, initial);
-            pause.0.release.notify_one();
-            worker.await.expect("worker join").expect("stale scan discarded");
-            let current = read_test_ledger(&job).await;
-            assert_eq!(
-                current.writer_revision, written.writer_revision,
-                "{mutation}: stale usage must not replace a concurrent writer"
-            );
-            assert_eq!(current.accounted_usage, written.accounted_usage);
-        }
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn quota_reconciliation_failure_is_throttled_and_deletions_eventually_reclaim_usage() {
-        use std::sync::atomic::Ordering;
-        let (_directories, job) = reconciliation_fixture(10).await;
-        let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
-        pause.0.fail_scan.store(true, Ordering::SeqCst);
-        pause.0.release.notify_one();
-        assert!(job.run().await.is_err());
-        assert!(matches!(
-            load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
-            Err(StorageError::ConfigNotFound)
-        ));
-        job.run().await.expect("failure cooldown");
-        assert_eq!(pause.0.scans.load(Ordering::SeqCst), 1);
-        pause.0.fail_scan.store(false, Ordering::SeqCst);
-        expire_reconciliation_backoff(&job).await;
-        job.run().await.expect("recover after failure");
-        assert_eq!(read_test_ledger(&job).await.accounted_usage, 10);
-        job.store
-            .delete_object(&job.bucket, "seed", ObjectOptions::default())
-            .await
-            .expect("delete stored usage");
-        assert!(
-            matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::QuotaExceeded { .. })),
-            "stale usage remains conservative during cooldown"
-        );
-        expire_reconciliation_backoff(&job).await;
-        job.run().await.expect("refresh deleted usage");
-        assert_eq!(read_test_ledger(&job).await.accounted_usage, 0);
-        quota_reservation(&job, 0, 1)
-            .await
-            .expect("delete must not leave bucket permanently full")
-            .abort()
-            .await;
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn quota_reconciliation_survives_waiter_cancellation_and_rechecks_configuration() {
-        let (_directories, job) = reconciliation_fixture(100).await;
-        let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
-        let caller = tokio::spawn({
-            let job = job.clone();
-            async move {
-                job.schedule();
-                std::future::pending::<()>().await
-            }
-        });
-        pause.reached().await;
-        caller.abort();
-        assert!(caller.await.expect_err("cancel caller").is_cancelled());
-        job.store
-            .update_bucket_metadata_config(
-                &job.bucket,
-                "quota.json",
-                serde_json::to_vec(&super::super::BucketQuota::new(Some(200))).expect("updated quota"),
-            )
-            .await
-            .expect("quota changes while scanning");
-        pause.0.release.notify_one();
-        wait_for_reconcile_owner(&job).await;
-        assert!(
-            matches!(
-                load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
-                Err(StorageError::ConfigNotFound)
-            ),
-            "old config cannot publish bootstrap usage"
-        );
-        drop(pause);
-        let (_, incarnation, revision) = job
-            .store
-            .bucket_quota_and_incarnation_from_disk(&job.bucket)
-            .await
-            .expect("current configuration");
-        let current = ReconcileJob::new(Arc::clone(&job.store), job.bucket.clone(), incarnation, revision);
-        let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
-        let caller = tokio::spawn({
-            let job = current.clone();
-            async move {
-                job.schedule();
-                std::future::pending::<()>().await
-            }
-        });
-        pause.reached().await;
-        caller.abort();
-        assert!(caller.await.expect_err("cancel new caller").is_cancelled());
-        pause.0.release.notify_one();
-        wait_for_reconcile_owner(&current).await;
-        assert!(
-            read_test_ledger(&current).await.matches(incarnation, revision),
-            "owned scan publishes after its waiter is canceled"
-        );
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn quota_reconciliation_cannot_publish_into_recreated_bucket() {
-        use crate::storage_api_contracts::bucket::DeleteBucketOptions;
-        let (_directories, job) = reconciliation_fixture(100).await;
-        let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
-        let worker = tokio::spawn({
-            let job = job.clone();
-            async move { job.run().await }
-        });
-        pause.reached().await;
-        job.store
-            .delete_object(&job.bucket, "seed", ObjectOptions::default())
-            .await
-            .expect("empty old bucket");
-        job.store
-            .delete_bucket(&job.bucket, &DeleteBucketOptions::default())
-            .await
-            .expect("retire bucket while scan is paused");
-        job.store
-            .make_bucket(&job.bucket, &MakeBucketOptions::default())
-            .await
-            .expect("replacement bucket");
-        job.store
-            .update_bucket_metadata_config(
-                &job.bucket,
-                "quota.json",
-                serde_json::to_vec(&super::super::BucketQuota::new(Some(100))).expect("new quota"),
-            )
-            .await
-            .expect("replacement quota");
-        pause.0.release.notify_one();
-        assert!(worker.await.expect("old worker join").is_err(), "old incarnation must not publish");
-        assert!(matches!(
-            load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
-            Err(StorageError::ConfigNotFound)
-        ));
-        drop(pause);
-        initialize_quota_ledger_for_test(Arc::clone(&job.store), &job.bucket)
-            .await
-            .expect("replacement can bootstrap independently");
-        let replacement = read_test_ledger(&job).await;
-        assert_ne!(replacement.bucket_incarnation, job.bucket_incarnation);
-        assert_eq!(replacement.accounted_usage, 0);
-    }
-
     fn ledger(accounted_usage: u64) -> QuotaLedger {
         QuotaLedger::new(Uuid::new_v4(), OffsetDateTime::now_utc(), accounted_usage)
     }
@@ -2781,7 +2090,8 @@ mod tests {
             .expect("persist crash-left reservation fixture");
 
         assert!(matches!(
-            load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc()).await,
+            load_current_ledger_locked(Arc::clone(&store), &bucket, &ledger_path, new_incarnation, OffsetDateTime::now_utc())
+                .await,
             Err(StorageError::PartMissingOrCorrupt)
         ));
         assert!(matches!(
@@ -2799,10 +2109,10 @@ mod tests {
             Err(StorageError::ConfigNotFound)
         ));
         let replacement =
-            load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc())
+            load_current_ledger_locked(Arc::clone(&store), &bucket, &ledger_path, new_incarnation, OffsetDateTime::now_utc())
                 .await
                 .expect("new quota admission must not inherit a fresh stale reservation");
-        assert!(replacement.is_none(), "replacement must await authoritative bootstrap before admission");
+        assert!(replacement.reservations.is_empty());
     }
 
     #[test]

crates/ecstore/src/bucket/remote_s3_client.rs

@@ -40,9 +40,7 @@ use aws_smithy_runtime_api::client::http::{
     HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
 };
 use aws_smithy_runtime_api::client::interceptors::Intercept;
-use aws_smithy_runtime_api::client::interceptors::context::{
-    BeforeDeserializationInterceptorContextMut, BeforeTransmitInterceptorContextMut,
-};
+use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
 use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
 use aws_smithy_runtime_api::client::result::ConnectorError;
 use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
@@ -157,8 +155,6 @@ pub struct RemoteS3EndpointSpec {
     pub ca_cert_pem: Option<String>,
     pub connect_timeout: Option<Duration>,
     pub read_timeout: Option<Duration>,
-    /// Entire SDK call, including non-streaming response bodies and retries.
-    pub operation_timeout: Option<Duration>,
     /// How many wire requests one logical call may cost. Every consumer
     /// declares it; see [`RemoteS3RetryPolicy`].
     pub retry: RemoteS3RetryPolicy,
@@ -256,78 +252,6 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
     Ok(builder.build())
 }
 
-/// Only installed on non-streaming upload operations. The budget begins at
-/// response headers, independently of the potentially long request upload.
-#[derive(Debug)]
-pub(crate) struct UploadResponseDeadline(pub Duration);
-
-impl Intercept for UploadResponseDeadline {
-    fn name(&self) -> &'static str {
-        "RustfsUploadResponseDeadline"
-    }
-
-    fn modify_before_deserialization(
-        &self,
-        context: &mut BeforeDeserializationInterceptorContextMut<'_>,
-        _runtime_components: &RuntimeComponents,
-        _cfg: &mut ConfigBag,
-    ) -> Result<(), BoxError> {
-        let response = context.response_mut();
-        let inner = std::mem::replace(response.body_mut(), SdkBody::taken());
-        *response.body_mut() = SdkBody::from_body_1_x(UploadResponseBody {
-            inner: Some(inner),
-            deadline: Box::pin(tokio::time::sleep(self.0)),
-        });
-        Ok(())
-    }
-}
-
-struct UploadResponseBody {
-    inner: Option<SdkBody>,
-    deadline: std::pin::Pin<Box<tokio::time::Sleep>>,
-}
-
-impl http_body::Body for UploadResponseBody {
-    type Data = bytes::Bytes;
-    type Error = BoxError;
-
-    fn poll_frame(
-        mut self: std::pin::Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-    ) -> std::task::Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
-        use std::future::Future as _;
-        use std::task::Poll;
-        if self.inner.is_none() {
-            return Poll::Ready(None);
-        }
-        // The absolute response budget never renews on data, empty frames,
-        // or trailers. A retry gets its own bounded response window.
-        if self.deadline.as_mut().poll(cx).is_ready() {
-            self.inner.take();
-            return Poll::Ready(Some(Err(std::io::Error::new(
-                std::io::ErrorKind::TimedOut,
-                "replication response body deadline exceeded",
-            )
-            .into())));
-        }
-        let Some(inner) = self.inner.as_mut() else {
-            return Poll::Ready(None);
-        };
-        let result = std::pin::Pin::new(inner).poll_frame(cx);
-        if matches!(result, Poll::Ready(None) | Poll::Ready(Some(Err(_)))) {
-            self.inner.take();
-        }
-        result
-    }
-
-    fn is_end_stream(&self) -> bool {
-        self.inner.as_ref().is_none_or(http_body::Body::is_end_stream)
-    }
-    fn size_hint(&self) -> http_body::SizeHint {
-        self.inner.as_ref().map(http_body::Body::size_hint).unwrap_or_default()
-    }
-}
-
 /// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
 /// signing: SigV4 excludes `user-agent` from the canonical request, so the
 /// signature stays valid.
@@ -385,7 +309,7 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
         config_builder = config_builder.http_client(http_client);
     }
 
-    if spec.connect_timeout.is_some() || spec.read_timeout.is_some() || spec.operation_timeout.is_some() {
+    if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
         let mut timeouts = TimeoutConfig::builder();
         if let Some(connect_timeout) = spec.connect_timeout {
             timeouts = timeouts.connect_timeout(connect_timeout);
@@ -393,9 +317,6 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
         if let Some(read_timeout) = spec.read_timeout {
             timeouts = timeouts.read_timeout(read_timeout);
         }
-        if let Some(operation_timeout) = spec.operation_timeout {
-            timeouts = timeouts.operation_timeout(operation_timeout);
-        }
         config_builder = config_builder.timeout_config(timeouts.build());
     }
 
@@ -753,7 +674,6 @@ mod tests {
             ca_cert_pem: None,
             connect_timeout: None,
             read_timeout: None,
-            operation_timeout: None,
             retry: RemoteS3RetryPolicy::Disabled,
             user_agent_suffix: "",
         }
@@ -1107,16 +1027,11 @@ mod tests {
         spec.user_agent_suffix = "RustFS-Test/0.0";
         spec.connect_timeout = Some(Duration::from_secs(5));
         spec.read_timeout = Some(Duration::from_secs(5));
-        spec.operation_timeout = Some(Duration::from_secs(12));
         let config = build_remote_s3_config(&spec)
             .await
             .expect("spec should build")
             .http_client(http_client)
             .build();
-        assert_eq!(
-            config.timeout_config().and_then(TimeoutConfig::operation_timeout),
-            Some(Duration::from_secs(12))
-        );
         S3Client::from_conf(config)
             .head_bucket()
             .bucket("bucket")

crates/ecstore/src/bucket/replication/replication_config_boundary.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+pub use rustfs_replication::config_cache_serde;
 pub use rustfs_replication::{
     ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_READ_ONLY_HISTORICAL_FIELDS,
     REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION,

crates/ecstore/src/bucket/replication/replication_lifecycle_bridge.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use s3s::dto::ReplicationConfiguration;
+use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
 
 use super::replication_config_boundary::ReplicationConfigurationExt as _;
 use super::replication_filemeta_boundary::{
@@ -42,7 +42,7 @@ impl ReplicationLifecycleBridge {
         dead_code,
         reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
     )]
-    pub(crate) fn new_config(config: ReplicationConfiguration) -> ReplicationLifecycleConfig {
+    pub(crate) fn new_config(config: PersistedReplicationConfiguration) -> ReplicationLifecycleConfig {
         ReplicationConfig::new(Some(config), None)
     }
 
@@ -118,16 +118,17 @@ impl ReplicationLifecycleBridge {
 
 #[cfg(test)]
 mod tests {
-    use s3s::dto::{Destination, ReplicationRule, ReplicationRuleStatus};
+    use rustfs_gateway_types::dto::Status;
+    use rustfs_gateway_types::persistence::{PersistedReplicationDestination, PersistedReplicationRule};
 
     use super::super::replication_filemeta_boundary::{ReplicateTargetDecision, VersionPurgeStatusType};
     use super::*;
 
-    fn replication_rule() -> ReplicationRule {
-        ReplicationRule {
+    fn replication_rule() -> PersistedReplicationRule {
+        PersistedReplicationRule {
             delete_marker_replication: None,
             delete_replication: None,
-            destination: Destination {
+            destination: PersistedReplicationDestination {
                 bucket: "arn:aws:s3:::target-bucket".to_string(),
                 ..Default::default()
             },
@@ -137,13 +138,13 @@ mod tests {
             prefix: Some(String::new()),
             priority: Some(1),
             source_selection_criteria: None,
-            status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
+            status: Status::ENABLED.to_string(),
         }
     }
 
     #[test]
     fn has_pending_version_purge_preserves_replication_active_rule_behavior() {
-        let config = ReplicationLifecycleBridge::new_config(ReplicationConfiguration {
+        let config = ReplicationLifecycleBridge::new_config(PersistedReplicationConfiguration {
             role: String::new(),
             rules: vec![replication_rule()],
         });

crates/ecstore/src/bucket/replication/replication_metadata_boundary.rs

@@ -17,8 +17,8 @@ use std::sync::Arc;
 use crate::bucket::{metadata::BucketMetadata, metadata_sys};
 use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
 use crate::runtime::instance::InstanceContext;
+use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
 use rustfs_utils::path::path_join_buf;
-use s3s::dto::ReplicationConfiguration;
 use time::OffsetDateTime;
 
 use super::replication_error_boundary::{Error, Result};
@@ -33,15 +33,14 @@ pub(crate) struct ReplicationMetadataStore;
 impl ReplicationMetadataStore {
     pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
     pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
-    pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
     pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
     pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
 
-    pub(crate) async fn replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
+    pub(crate) async fn replication_config(bucket: &str) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
         metadata_sys::get_replication_config(bucket).await
     }
 
-    pub(crate) async fn optional_replication_config(bucket: &str) -> Result<Option<ReplicationConfiguration>> {
+    pub(crate) async fn optional_replication_config(bucket: &str) -> Result<Option<PersistedReplicationConfiguration>> {
         let config = match Self::replication_config(bucket).await {
             Ok((config, _)) => Some(config),
             Err(err) => {
@@ -117,10 +116,6 @@ mod tests {
             ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
             "config/replication/mrf.bin.recovery"
         );
-        assert_eq!(
-            ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
-            "config/replication/mrf.bin.replay"
-        );
         assert_eq!(
             ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
             "config/replication/force-delete.bin"

crates/ecstore/src/bucket/replication/replication_object_config.rs

@@ -16,7 +16,9 @@ use std::{collections::HashMap, fmt, sync::Arc};
 
 use super::replication_filemeta_boundary::metadata_keys;
 use crate::bucket::metadata::BucketMetadata;
-use s3s::dto::{BucketVersioningStatus, ReplicationConfiguration, ReplicationRuleStatus, VersioningConfiguration};
+use rustfs_gateway_types::dto::Status;
+use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
+use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
 use serde::{Deserialize, Serialize};
 use time::OffsetDateTime;
 use tracing::error;
@@ -39,7 +41,7 @@ use super::replication_target_boundary::{BucketTargets, ReplicationTargetStore};
 use super::replication_versioning_boundary::ReplicationVersioningStore;
 use super::runtime_boundary as runtime_sources;
 
-pub(crate) async fn get_replication_config(bucket: &str) -> Result<Option<ReplicationConfiguration>> {
+pub(crate) async fn get_replication_config(bucket: &str) -> Result<Option<PersistedReplicationConfiguration>> {
     let config = ReplicationMetadataStore::optional_replication_config(bucket).await?;
     validate_delete_replication_config(&VersioningConfiguration::default(), config.as_ref())?;
     Ok(config)
@@ -64,7 +66,7 @@ impl DeleteReplicationConfigSnapshot {
     #[cfg(test)]
     pub(crate) fn from_configs_for_test(
         versioning: VersioningConfiguration,
-        replication: Option<ReplicationConfiguration>,
+        replication: Option<PersistedReplicationConfiguration>,
     ) -> Self {
         let metadata = replication.map(|config| {
             let mut metadata = BucketMetadata::new("test-bucket");
@@ -78,7 +80,7 @@ impl DeleteReplicationConfigSnapshot {
         &self.versioning
     }
 
-    pub fn replication_config(&self) -> Option<&ReplicationConfiguration> {
+    pub fn replication_config(&self) -> Option<&PersistedReplicationConfiguration> {
         self.metadata
             .as_ref()
             .and_then(|metadata| metadata.replication_config.as_ref())
@@ -101,7 +103,7 @@ impl DeleteReplicationConfigSnapshot {
     pub(crate) fn active_delete_marker_rules_require_tags(&self, object: &str) -> bool {
         self.replication_config().is_some_and(|config| {
             config.rules.iter().any(|rule| {
-                if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
+                if rule.status == Status::DISABLED.as_str() {
                     return false;
                 }
                 if !object.starts_with(rule.prefix()) {
@@ -122,12 +124,12 @@ impl DeleteReplicationConfigSnapshot {
 
 fn validate_delete_replication_config(
     versioning: &VersioningConfiguration,
-    config: Option<&ReplicationConfiguration>,
+    config: Option<&PersistedReplicationConfiguration>,
 ) -> Result<()> {
     if versioning
         .status
         .as_ref()
-        .is_some_and(|status| !matches!(status.as_str(), BucketVersioningStatus::ENABLED | BucketVersioningStatus::SUSPENDED))
+        .is_some_and(|status| !matches!(status.as_str(), "Enabled" | "Suspended"))
     {
         return Err(super::replication_error_boundary::Error::other(
             "bucket versioning configuration has an invalid status",
@@ -144,7 +146,7 @@ fn validate_delete_replication_config(
         let role = config.role.trim();
         let mut role_destination = None;
         for rule in &config.rules {
-            if rule.status.as_str() == ReplicationRuleStatus::ENABLED {
+            if rule.status.as_str() == Status::ENABLED.as_str() {
                 let destination = rule.destination.bucket.trim();
                 if role.is_empty() && destination.is_empty() {
                     return Err(super::replication_error_boundary::Error::other(
@@ -169,7 +171,7 @@ fn validate_delete_replication_config(
     Ok(())
 }
 
-fn replication_config_from_metadata(metadata: &BucketMetadata) -> Result<Option<&ReplicationConfiguration>> {
+fn replication_config_from_metadata(metadata: &BucketMetadata) -> Result<Option<&PersistedReplicationConfiguration>> {
     if !metadata.replication_config_xml.is_empty() && metadata.replication_config.is_none() {
         return Err(super::replication_error_boundary::Error::other(
             "persisted bucket replication configuration is invalid",
@@ -250,12 +252,15 @@ pub(crate) async fn load_delete_replication_config_in(
 
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct ReplicationConfig {
-    pub config: Option<ReplicationConfiguration>,
+    /// Carried in the scanner's persisted data usage cache, so its serde form
+    /// is the one the s3s DTO wrote there (see `config_cache_serde` in the replication crate).
+    #[serde(with = "super::replication_config_boundary::config_cache_serde")]
+    pub config: Option<PersistedReplicationConfiguration>,
     pub remotes: Option<BucketTargets>,
 }
 
 impl ReplicationConfig {
-    pub fn new(config: Option<ReplicationConfiguration>, remotes: Option<BucketTargets>) -> Self {
+    pub fn new(config: Option<PersistedReplicationConfiguration>, remotes: Option<BucketTargets>) -> Self {
         Self { config, remotes }
     }
 
@@ -468,7 +473,7 @@ fn check_replicate_delete_with_config(
     oi: &ObjectInfo,
     del_opts: &ObjectOptions,
     source_error: bool,
-    config: Option<&ReplicationConfiguration>,
+    config: Option<&PersistedReplicationConfiguration>,
     existing_delete_marker: bool,
     trust_persisted_replica_status: bool,
 ) -> ReplicateDecision {
@@ -587,7 +592,7 @@ pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplic
 }
 
 fn metadata_target_should_replicate(
-    cfg: &ReplicationConfiguration,
+    cfg: &PersistedReplicationConfiguration,
     opts: &ObjectOpts,
     mopts: &MustReplicateOptions,
     arn: &str,
@@ -597,20 +602,21 @@ fn metadata_target_should_replicate(
 
 #[cfg(test)]
 mod tests {
-    use s3s::dto::{
-        DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination,
-        ReplicaModifications, ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, SourceSelectionCriteria, Tag,
+    use rustfs_gateway_types::dto::Status;
+    use rustfs_gateway_types::persistence::{
+        PersistedOptionalReplicationStatus, PersistedReplicationDestination, PersistedReplicationFilter,
+        PersistedReplicationRule, PersistedReplicationStatus, PersistedReplicationTag, PersistedSourceSelectionCriteria,
     };
 
     use super::super::replication_filemeta_boundary::VersionPurgeStatusType;
     use super::super::replication_target_boundary::BucketTarget;
     use super::*;
 
-    fn replication_rule() -> ReplicationRule {
-        ReplicationRule {
+    fn replication_rule() -> PersistedReplicationRule {
+        PersistedReplicationRule {
             delete_marker_replication: None,
             delete_replication: None,
-            destination: Destination {
+            destination: PersistedReplicationDestination {
                 bucket: "arn:aws:s3:::target-bucket".to_string(),
                 ..Default::default()
             },
@@ -620,7 +626,7 @@ mod tests {
             prefix: Some(String::new()),
             priority: Some(1),
             source_selection_criteria: None,
-            status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
+            status: Status::ENABLED.to_string(),
         }
     }
 
@@ -629,11 +635,11 @@ mod tests {
         let arn = "arn:rustfs:replication:us-east-1:target:bucket";
         let mut rule = replication_rule();
         rule.destination.bucket = arn.to_string();
-        rule.filter = Some(ReplicationRuleFilter {
+        rule.filter = Some(PersistedReplicationFilter {
             prefix: Some("admitted/".to_string()),
             ..Default::default()
         });
-        let cfg = ReplicationConfiguration {
+        let cfg = PersistedReplicationConfiguration {
             role: String::new(),
             rules: vec![rule],
         };
@@ -671,7 +677,7 @@ mod tests {
         assert!(!empty.replicate(&ObjectOpts::default()));
 
         let config = ReplicationConfig::new(
-            Some(ReplicationConfiguration {
+            Some(PersistedReplicationConfiguration {
                 role: String::new(),
                 rules: vec![replication_rule()],
             }),
@@ -725,7 +731,7 @@ mod tests {
     fn delete_snapshot_rejects_enabled_rules_without_a_destination() {
         let mut rule = replication_rule();
         rule.destination.bucket.clear();
-        let config = ReplicationConfiguration {
+        let config = PersistedReplicationConfiguration {
             role: String::new(),
             rules: vec![rule],
         };
@@ -741,7 +747,7 @@ mod tests {
         let first = replication_rule();
         let mut second = replication_rule();
         second.destination.bucket = "arn:aws:s3:::other-target".to_string();
-        let config = ReplicationConfiguration {
+        let config = PersistedReplicationConfiguration {
             role: "arn:aws:s3:::role-target".to_string(),
             rules: vec![first, second],
         };
@@ -752,47 +758,47 @@ mod tests {
     #[test]
     fn delete_snapshot_rejects_unknown_status_values() {
         let invalid_versioning = VersioningConfiguration {
-            status: Some("Enabld".to_string().into()),
+            status: Some("Enabld".to_string()),
             ..Default::default()
         };
         assert!(validate_delete_replication_config(&invalid_versioning, None).is_err());
 
         let mut invalid_rule = replication_rule();
-        invalid_rule.status = "Enabld".to_string().into();
-        let config = ReplicationConfiguration {
+        invalid_rule.status = "Enabld".to_string();
+        let config = PersistedReplicationConfiguration {
             role: String::new(),
             rules: vec![invalid_rule],
         };
         assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
 
         let mut invalid_delete = replication_rule();
-        invalid_delete.delete_replication = Some(DeleteReplication {
-            status: "Enabld".to_string().into(),
+        invalid_delete.delete_replication = Some(PersistedReplicationStatus {
+            status: "Enabld".to_string(),
         });
-        let config = ReplicationConfiguration {
+        let config = PersistedReplicationConfiguration {
             role: String::new(),
             rules: vec![invalid_delete],
         };
         assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
 
         let mut invalid_delete_marker = replication_rule();
-        invalid_delete_marker.delete_marker_replication = Some(DeleteMarkerReplication {
-            status: Some("Enabld".to_string().into()),
+        invalid_delete_marker.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
+            status: Some("Enabld".to_string()),
         });
-        let config = ReplicationConfiguration {
+        let config = PersistedReplicationConfiguration {
             role: String::new(),
             rules: vec![invalid_delete_marker],
         };
         assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
 
         let mut invalid_replica_modifications = replication_rule();
-        invalid_replica_modifications.source_selection_criteria = Some(SourceSelectionCriteria {
-            replica_modifications: Some(ReplicaModifications {
-                status: "Enabld".to_string().into(),
+        invalid_replica_modifications.source_selection_criteria = Some(PersistedSourceSelectionCriteria {
+            replica_modifications: Some(PersistedReplicationStatus {
+                status: "Enabld".to_string(),
             }),
             sse_kms_encrypted_objects: None,
         });
-        let config = ReplicationConfiguration {
+        let config = PersistedReplicationConfiguration {
             role: String::new(),
             rules: vec![invalid_replica_modifications],
         };
@@ -804,11 +810,11 @@ mod tests {
         let mut metadata = BucketMetadata::new("bucket");
         metadata.versioning_config_xml = b"configured".to_vec();
         metadata.versioning_config = Some(VersioningConfiguration {
-            status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
+            status: Some("Enabled".to_owned()),
             ..Default::default()
         });
         metadata.replication_config_xml = b"configured".to_vec();
-        metadata.replication_config = Some(ReplicationConfiguration {
+        metadata.replication_config = Some(PersistedReplicationConfiguration {
             role: String::new(),
             rules: vec![replication_rule()],
         });
@@ -848,7 +854,7 @@ mod tests {
         );
 
         let mut inconsistent = BucketMetadata::new("bucket");
-        inconsistent.replication_config = Some(ReplicationConfiguration {
+        inconsistent.replication_config = Some(PersistedReplicationConfiguration {
             role: String::new(),
             rules: vec![replication_rule()],
         });
@@ -863,11 +869,11 @@ mod tests {
         let arn = "arn:rustfs:replication:us-east-1:target:bucket";
         let mut rule = replication_rule();
         rule.destination.bucket = arn.to_string();
-        rule.delete_marker_replication = Some(DeleteMarkerReplication {
-            status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
+        rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
+            status: Some(Status::ENABLED.to_string()),
         });
         let mut metadata = BucketMetadata::new("bucket");
-        metadata.replication_config = Some(ReplicationConfiguration {
+        metadata.replication_config = Some(PersistedReplicationConfiguration {
             role: String::new(),
             rules: vec![rule],
         });
@@ -898,27 +904,27 @@ mod tests {
     fn delete_marker_source_read_is_required_only_for_tag_filtered_rules() {
         let mut prefix_rule = replication_rule();
         prefix_rule.prefix = Some("logs/".to_string());
-        prefix_rule.delete_marker_replication = Some(DeleteMarkerReplication {
-            status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
+        prefix_rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
+            status: Some(Status::ENABLED.to_string()),
         });
 
         let prefix_snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
             VersioningConfiguration {
-                status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
+                status: Some("Enabled".to_owned()),
                 ..Default::default()
             },
-            Some(ReplicationConfiguration {
+            Some(PersistedReplicationConfiguration {
                 role: String::new(),
                 rules: vec![prefix_rule],
             }),
         );
 
         let mut tag_rule = replication_rule();
-        tag_rule.delete_marker_replication = Some(DeleteMarkerReplication {
-            status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
+        tag_rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
+            status: Some(Status::ENABLED.to_string()),
         });
-        tag_rule.filter = Some(ReplicationRuleFilter {
-            tag: Some(Tag {
+        tag_rule.filter = Some(PersistedReplicationFilter {
+            tag: Some(PersistedReplicationTag {
                 key: Some("class".to_string()),
                 value: Some("audit".to_string()),
             }),
@@ -926,10 +932,10 @@ mod tests {
         });
         let tag_snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
             VersioningConfiguration {
-                status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
+                status: Some("Enabled".to_owned()),
                 ..Default::default()
             },
-            Some(ReplicationConfiguration {
+            Some(PersistedReplicationConfiguration {
                 role: String::new(),
                 rules: vec![tag_rule],
             }),
@@ -944,14 +950,14 @@ mod tests {
         let arn = "arn:rustfs:replication:us-east-1:target:bucket";
         let mut rule = replication_rule();
         rule.destination.bucket = arn.to_string();
-        rule.delete_replication = Some(DeleteReplication {
-            status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
+        rule.delete_replication = Some(PersistedReplicationStatus {
+            status: Status::ENABLED.to_string(),
         });
-        rule.delete_marker_replication = Some(DeleteMarkerReplication {
-            status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)),
+        rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
+            status: Some(Status::DISABLED.to_string()),
         });
         let config = ReplicationConfig::new(
-            Some(ReplicationConfiguration {
+            Some(PersistedReplicationConfiguration {
                 role: String::new(),
                 rules: vec![rule],
             }),
@@ -985,22 +991,22 @@ mod tests {
     #[test]
     fn live_delete_does_not_trust_persisted_replica_status() {
         let mut rule = replication_rule();
-        rule.delete_replication = Some(DeleteReplication {
-            status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
+        rule.delete_replication = Some(PersistedReplicationStatus {
+            status: Status::ENABLED.to_string(),
         });
-        rule.source_selection_criteria = Some(SourceSelectionCriteria {
-            replica_modifications: Some(ReplicaModifications {
-                status: s3s::dto::ReplicaModificationsStatus::from_static(s3s::dto::ReplicaModificationsStatus::DISABLED),
+        rule.source_selection_criteria = Some(PersistedSourceSelectionCriteria {
+            replica_modifications: Some(PersistedReplicationStatus {
+                status: Status::DISABLED.to_string(),
             }),
             sse_kms_encrypted_objects: None,
         });
-        let replication = ReplicationConfiguration {
+        let replication = PersistedReplicationConfiguration {
             role: String::new(),
             rules: vec![rule],
         };
         let snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
             VersioningConfiguration {
-                status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
+                status: Some("Enabled".to_owned()),
                 ..Default::default()
             },
             Some(replication.clone()),
@@ -1036,14 +1042,14 @@ mod tests {
         let arn = "arn:rustfs:replication:us-east-1:target:bucket";
         let mut rule = replication_rule();
         rule.destination.bucket = arn.to_string();
-        rule.delete_replication = Some(DeleteReplication {
-            status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
+        rule.delete_replication = Some(PersistedReplicationStatus {
+            status: Status::ENABLED.to_string(),
         });
-        rule.delete_marker_replication = Some(DeleteMarkerReplication {
-            status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)),
+        rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
+            status: Some(Status::DISABLED.to_string()),
         });
         let config = ReplicationConfig::new(
-            Some(ReplicationConfiguration {
+            Some(PersistedReplicationConfiguration {
                 role: String::new(),
                 rules: vec![rule],
             }),

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -16,8 +16,8 @@ use super::replication_config_store::ReplicationConfigStore;
 use super::replication_error_boundary::{Error as EcstoreError, is_err_object_not_found, is_err_version_not_found};
 use super::replication_filemeta_boundary::{
     MrfOpKind, MrfReplicateEntry, REPLICATE_HEAL_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedTargetInfo,
-    ReplicationGenerationSnapshot, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
-    replicate_decision_for_admitted_targets, replication_statuses_map, version_purge_statuses_map,
+    ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, replicate_decision_for_admitted_targets,
+    replication_statuses_map, version_purge_statuses_map,
 };
 use super::replication_lock_boundary::ReplicationLockTiming;
 use super::replication_logging::{EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION};
@@ -86,10 +86,6 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
 const MRF_PENDING_CAP: usize = 200_000;
 const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
 const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
-/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
-/// re-attempts every retained entry, so a target that stays down costs one
-/// attempt per entry per interval.
-const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
 
 #[derive(Debug, Default)]
 pub struct DurableMrfBacklog {
@@ -619,7 +615,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
 }
 
 /// Acknowledge only the generation read by the recovery leader. Entries appended
-/// after that generation are retained as a suffix and replayed on the next pass.
+/// after that generation are retained as a suffix and replayed on the next startup.
 /// Lock order: recovery leader lock -> MRF journal object lock.
 async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     storage: Arc<S>,
@@ -671,7 +667,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     Err(EcstoreError::PreconditionFailed)
 }
 
-/// Acquires the MRF recovery leader lock for one journal replay pass.
+/// Acquires the MRF recovery leader lock for the startup replay.
 /// Returns `None` (after logging) when the lock cannot be created or another
 /// node is already processing the backlog.
 async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
@@ -1022,7 +1018,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
 }
 
 /// Acknowledges the replayed MRF prefix and returns the retained backlog.
-/// On acknowledgement failure the backlog is preserved for the next replay pass and
+/// On acknowledgement failure the backlog is preserved for the next startup and
 /// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
 async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     storage: &Arc<S>,
@@ -1037,7 +1033,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
                 component = LOG_COMPONENT_ECSTORE,
                 subsystem = LOG_SUBSYSTEM_REPLICATION,
                 error = %error,
-                "Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
+                "Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
             );
             match read_mrf_entries(storage.clone()).await {
                 Ok(current) => current,
@@ -1055,120 +1051,6 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     }
 }
 
-fn unix_millis_now() -> i64 {
-    i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
-}
-
-/// Whether a periodic pass may run, given the start time of the last pass on
-/// any node. A stamp more than one interval ahead of this node's clock is skew
-/// or a bad value, so it does not suppress replay.
-fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
-    let Some(last_started_ms) = last_started_ms else {
-        return true;
-    };
-    let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
-    now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
-}
-
-async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
-    let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
-        .await
-        .ok()?;
-    std::str::from_utf8(&data).ok()?.trim().parse().ok()
-}
-
-async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
-    if let Err(error) = ReplicationConfigStore::save(
-        storage.clone(),
-        ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
-        started_ms.to_string().into_bytes(),
-    )
-    .await
-    {
-        warn!(
-            component = LOG_COMPONENT_ECSTORE,
-            subsystem = LOG_SUBSYSTEM_REPLICATION,
-            error = %error,
-            "Failed to record the MRF replay start time"
-        );
-    }
-}
-
-/// One MRF journal replay pass. The recovery leader reads the journal,
-/// re-drives every entry, and acknowledges the replayed prefix; entries that
-/// could not be admitted, and entries appended meanwhile, stay for the next pass.
-///
-/// Every node runs the periodic loop, and the leader lock only serializes
-/// passes. With `min_interval`, a pass is skipped when a pass on any node
-/// started within that interval, read from the shared replay stamp, so a
-/// cluster replays once per interval rather than once per node. The startup
-/// pass passes `None` and always runs. A pass that finds entries records its
-/// start time. Returns whether the journal was read.
-/// Lock order: recovery leader lock -> replay stamp object lock.
-async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
-    let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
-        return false;
-    };
-    let started_ms = unix_millis_now();
-    if let Some(interval) = min_interval
-        && !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
-    {
-        return false;
-    }
-
-    let Some(entries) = load_mrf_recovery_entries(storage).await else {
-        return true;
-    };
-    if !entries.is_empty() {
-        write_mrf_replay_stamp(storage, started_ms).await;
-    }
-    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
-
-    let total = entries.len();
-    let mut queued_count = 0usize;
-    let mut retry_entries = Vec::new();
-
-    for entry in entries.iter() {
-        let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
-            continue;
-        };
-
-        if admission == ReplicationQueueAdmission::Missed {
-            retry_entries.push(entry.clone());
-        } else if admission == ReplicationQueueAdmission::Queued {
-            queued_count += 1;
-        }
-    }
-
-    let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
-    let retained_count = retained.len();
-    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
-
-    if queued_count > 0 {
-        info!(
-            component = LOG_COMPONENT_ECSTORE,
-            subsystem = LOG_SUBSYSTEM_REPLICATION,
-            recovered = queued_count,
-            total,
-            retained = retained_count,
-            "Replayed MRF entries admitted for retry"
-        );
-    }
-    true
-}
-
-/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
-/// one pass at a time across the cluster, including the startup pass, and the
-/// shared replay stamp spaces passes from different nodes by `period`.
-fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
-    tokio::spawn(async move {
-        loop {
-            tokio::time::sleep(period).await;
-            replay_mrf_journal(&storage, Some(period)).await;
-        }
-    })
-}
-
 #[derive(Debug, thiserror::Error)]
 #[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
 struct ResyncActiveConflictError {
@@ -1289,7 +1171,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
         // Start background tasks
         pool.start_mrf_persister().await;
         pool.start_mrf_processor().await;
-        pool.start_mrf_replayer().await;
         pool.start_force_delete_processor().await;
 
         pool
@@ -1581,7 +1462,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Queues a replica task
     pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
-        // A source generation already queued or being uploaded is not driven a
+        // A version that is already queued or being uploaded is not driven a
         // second time: the scanner heal pass sees it as PENDING until the
         // first upload lands and would otherwise re-queue it every cycle
         // (backlog#2362). The key is released when the worker finishes, or
@@ -1758,23 +1639,52 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Starts the MRF processor — one-shot at startup.
     ///
-    /// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
-    /// the pass while the server runs.
+    /// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
+    /// rewrites any entries that could not be admitted for a later startup retry.
     async fn start_mrf_processor(&self) {
         let storage = self.storage.clone();
-        let handle = tokio::spawn(async move {
-            replay_mrf_journal(&storage, None).await;
-        });
-        self.task_handles.lock().await.push(handle);
-    }
 
-    /// Starts the runtime MRF replayer — ongoing background task.
-    ///
-    /// Entries reach the journal while the server runs (worker-queue overflow,
-    /// delete-marker purge intents whose watch window drained with targets still
-    /// dirty). Replaying only at startup left them waiting for a restart.
-    async fn start_mrf_replayer(&self) {
-        let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
+        let handle = tokio::spawn(async move {
+            let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
+                return;
+            };
+
+            let Some(entries) = load_mrf_recovery_entries(&storage).await else {
+                return;
+            };
+            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
+
+            let total = entries.len();
+            let mut queued_count = 0usize;
+            let mut retry_entries = Vec::new();
+
+            for entry in entries.iter() {
+                let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
+                    continue;
+                };
+
+                if admission == ReplicationQueueAdmission::Missed {
+                    retry_entries.push(entry.clone());
+                } else if admission == ReplicationQueueAdmission::Queued {
+                    queued_count += 1;
+                }
+            }
+
+            let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
+            let retained_count = retained.len();
+            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
+
+            if queued_count > 0 {
+                info!(
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REPLICATION,
+                    recovered = queued_count,
+                    total,
+                    retained = retained_count,
+                    "Replayed MRF entries admitted for retry"
+                );
+            }
+        });
         self.task_handles.lock().await.push(handle);
     }
 
@@ -1845,7 +1755,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
     /// writes them to the on-disk MRF file every flush interval (default 10s,
     /// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
     /// entries accumulate. Runtime writers append under the journal lock; only the
-    /// recovery leader of a replay pass may remove an acknowledged prefix.
+    /// startup recovery leader may remove an acknowledged prefix.
     async fn start_mrf_persister(&self) {
         let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
             return;
@@ -2404,47 +2314,28 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 }
 
 /// Object versions currently queued or being uploaded, keyed by bucket,
-/// object name, version and source generation. `queue_replica_task` admits a generation only once
+/// object name and version. `queue_replica_task` admits a version only once
 /// while it is in flight; the scanner heal pass and MRF replays that arrive
 /// in the meantime are `Skipped` instead of driving a second complete upload
 /// (backlog#2362). Entries are removed when the worker finishes the task or
 /// when no worker accepted it.
 #[derive(Debug, Default)]
 pub(crate) struct ReplicationInFlight {
-    keys: std::sync::Mutex<std::collections::HashSet<ReplicationInFlightKey>>,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-struct ReplicationInFlightKey {
-    bucket: String,
-    object: String,
-    version_id: Option<uuid::Uuid>,
-    generation: ReplicationGenerationSnapshot,
-}
-
-impl From<&ReplicateObjectInfo> for ReplicationInFlightKey {
-    fn from(info: &ReplicateObjectInfo) -> Self {
-        Self {
-            bucket: info.bucket.clone(),
-            object: info.name.clone(),
-            version_id: info.version_id,
-            generation: info.replication_generation.clone(),
-        }
-    }
+    keys: std::sync::Mutex<std::collections::HashSet<(String, String, Option<uuid::Uuid>)>>,
 }
 
 impl ReplicationInFlight {
-    fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<ReplicationInFlightKey>> {
+    fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<(String, String, Option<uuid::Uuid>)>> {
         self.keys.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
     }
 
-    /// Claim `ri`; a metadata mutation is independent of its earlier generation.
+    /// Claim `ri`; `false` when the same version is already in flight.
     fn try_begin(&self, ri: &ReplicateObjectInfo) -> bool {
-        self.lock().insert(ReplicationInFlightKey::from(ri))
+        self.lock().insert((ri.bucket.clone(), ri.name.clone(), ri.version_id))
     }
 
     fn finish(&self, ri: &ReplicateObjectInfo) {
-        self.lock().remove(&ReplicationInFlightKey::from(ri));
+        self.lock().remove(&(ri.bucket.clone(), ri.name.clone(), ri.version_id));
     }
 
     #[cfg(test)]
@@ -2457,21 +2348,26 @@ impl ReplicationInFlight {
 /// including when replication panics.
 struct ReplicationInFlightGuard {
     in_flight: Arc<ReplicationInFlight>,
-    key: ReplicationInFlightKey,
+    key: ReplicateObjectInfo,
 }
 
 impl ReplicationInFlightGuard {
     fn new(in_flight: Arc<ReplicationInFlight>, ri: &ReplicateObjectInfo) -> Self {
         Self {
             in_flight,
-            key: ReplicationInFlightKey::from(ri),
+            key: ReplicateObjectInfo {
+                bucket: ri.bucket.clone(),
+                name: ri.name.clone(),
+                version_id: ri.version_id,
+                ..Default::default()
+            },
         }
     }
 }
 
 impl Drop for ReplicationInFlightGuard {
     fn drop(&mut self) {
-        self.in_flight.lock().remove(&self.key);
+        self.in_flight.finish(&self.key);
     }
 }
 
@@ -3525,8 +3421,6 @@ mod tests {
         allow_first_read: Notify,
         read_count: AtomicUsize,
         delete_storage_calls: AtomicUsize,
-        delete_error: StdMutex<EcstoreError>,
-        replay_stamp: StdMutex<Option<Vec<u8>>>,
         reported_size: StdMutex<Option<i64>>,
         stream_read_bytes: Arc<AtomicUsize>,
         write_count: AtomicUsize,
@@ -3594,24 +3488,6 @@ mod tests {
             _h: Self::HeaderMap,
             _opts: &Self::ObjectOptions,
         ) -> Result<Self::GetObjectReader, Self::Error> {
-            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
-                let stamp = self
-                    .shared
-                    .replay_stamp
-                    .lock()
-                    .expect("test replay stamp lock should not be poisoned")
-                    .clone()
-                    .ok_or(EcstoreError::FileNotFound)?;
-                return Ok(Self::GetObjectReader {
-                    object_info: ObjectInfo {
-                        size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
-                        ..Default::default()
-                    },
-                    stream: Box::new(Cursor::new(stamp)),
-                    buffered_body: None,
-                    body_source: Default::default(),
-                });
-            }
             if !object.ends_with("/.replication/resync.bin")
                 && !object.ends_with("config/replication/mrf.bin")
                 && !object.ends_with("config/replication/force-delete.bin")
@@ -3675,16 +3551,6 @@ mod tests {
                 let lock = self.new_ns_lock(bucket, object).await?;
                 Some(lock.get_write_lock(Duration::from_secs(10)).await?)
             };
-            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
-                let mut stamp = Vec::new();
-                data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
-                *self
-                    .shared
-                    .replay_stamp
-                    .lock()
-                    .expect("test replay stamp lock should not be poisoned") = Some(stamp);
-                return Ok(ObjectInfo::default());
-            }
             if opts.http_preconditions.is_some()
                 && let Some(replacement) = self
                     .shared
@@ -3806,12 +3672,7 @@ mod tests {
             _opts: Self::ObjectOptions,
         ) -> Result<Self::ObjectInfo, Self::Error> {
             self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
-            Err(self
-                .shared
-                .delete_error
-                .lock()
-                .expect("delete error lock should not be poisoned")
-                .clone())
+            Err(EcstoreError::NotImplemented)
         }
 
         async fn delete_objects(
@@ -4068,48 +3929,6 @@ mod tests {
         assert_eq!(current_queue(&pool, "in-flight-bucket").await, (3, 3 * 4096));
     }
 
-    #[tokio::test]
-    async fn queue_replica_task_admits_metadata_changes_during_an_initial_upload() {
-        let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
-        let (tx, mut rx) = mpsc::channel(4);
-        pool.workers.write().await.push(tx);
-        let initial = ReplicateObjectInfo {
-            bucket: "pending-tags-bucket".to_string(),
-            name: "object".to_string(),
-            version_id: Some(uuid::Uuid::new_v4()),
-            size: 4096,
-            replication_generation: ReplicationGenerationSnapshot {
-                mutation_id: Some(uuid::Uuid::new_v4().to_string()),
-                ..Default::default()
-            },
-            ..Default::default()
-        };
-        let metadata = ReplicateObjectInfo {
-            op_type: ReplicationType::Metadata,
-            user_tags: String::new(),
-            replication_generation: ReplicationGenerationSnapshot {
-                mutation_id: Some(uuid::Uuid::new_v4().to_string()),
-                ..Default::default()
-            },
-            ..initial.clone()
-        };
-        assert_eq!(pool.queue_replica_task(initial.clone()).await, ReplicationQueueAdmission::Queued);
-        assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Queued);
-        assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
-        assert_eq!(pool.in_flight.len(), 2);
-        drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &initial));
-        assert_eq!(pool.in_flight.len(), 1);
-        assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
-        for expected in [&initial, &metadata] {
-            let ReplicationOperation::Object(queued) = rx.recv().await.expect("queued operation") else {
-                panic!("expected object replication");
-            };
-            assert_eq!(queued.replication_generation, expected.replication_generation);
-        }
-        drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &metadata));
-        assert_eq!(pool.in_flight.len(), 0);
-    }
-
     #[tokio::test]
     async fn queue_replica_task_releases_the_version_when_no_worker_accepts_it() {
         let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
@@ -4352,8 +4171,6 @@ mod tests {
             allow_first_read: Notify::new(),
             read_count: AtomicUsize::new(0),
             delete_storage_calls: AtomicUsize::new(0),
-            delete_error: StdMutex::new(EcstoreError::NotImplemented),
-            replay_stamp: StdMutex::new(None),
             reported_size: StdMutex::new(None),
             stream_read_bytes: Arc::new(AtomicUsize::new(0)),
             write_count: AtomicUsize::new(0),
@@ -4404,103 +4221,6 @@ mod tests {
         }
     }
 
-    #[tokio::test]
-    async fn delete_worker_already_purged_source_completes_without_retry() {
-        let arn = "arn:test";
-        for delete_marker_version in [false, true] {
-            for version_id in [Uuid::new_v4(), Uuid::nil()] {
-                for error in [
-                    EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
-                    EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
-                    EcstoreError::FileVersionNotFound,
-                    EcstoreError::FileNotFound,
-                ] {
-                    let shared = empty_resync_shared_state();
-                    *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
-                    let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
-                    let delete = DeletedObjectReplicationInfo {
-                        bucket: "source".to_string(),
-                        delete_object: ReplicationDeletedObject {
-                            object_name: "purged".to_string(),
-                            version_id: (!delete_marker_version).then_some(version_id),
-                            delete_marker_version_id: delete_marker_version.then_some(version_id),
-                            replication_state: Some(ReplicationState {
-                                replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
-                                purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
-                                ..Default::default()
-                            }),
-                            ..Default::default()
-                        },
-                        op_type: ReplicationType::Delete,
-                        ..Default::default()
-                    };
-
-                    assert!(
-                        replicate_delete_with_outcome(delete, storage).await,
-                        "a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
-                    );
-                    assert_eq!(
-                        shared.delete_storage_calls.load(Ordering::SeqCst),
-                        2 + usize::from(!delete_marker_version && !version_id.is_nil()),
-                        "the worker must lock and attempt writeback, reading a non-null data version first"
-                    );
-                }
-            }
-        }
-    }
-
-    #[tokio::test]
-    async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
-        let arn = "arn:test";
-        for (version_purge, target_status, error) in [
-            (
-                true,
-                VersionPurgeStatusType::Pending,
-                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
-            ),
-            (true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
-            (
-                false,
-                VersionPurgeStatusType::Empty,
-                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
-            ),
-        ] {
-            let shared = empty_resync_shared_state();
-            *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
-            let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
-            let delete = DeletedObjectReplicationInfo {
-                bucket: "source".to_string(),
-                delete_object: ReplicationDeletedObject {
-                    object_name: "purged".to_string(),
-                    version_id: version_purge.then(Uuid::new_v4),
-                    delete_marker: !version_purge,
-                    replication_state: Some(ReplicationState {
-                        replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
-                        targets: if version_purge {
-                            HashMap::new()
-                        } else {
-                            HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
-                        },
-                        purge_targets: if version_purge {
-                            HashMap::from([(arn.to_string(), target_status)])
-                        } else {
-                            HashMap::new()
-                        },
-                        ..Default::default()
-                    }),
-                    ..Default::default()
-                },
-                op_type: ReplicationType::Delete,
-                ..Default::default()
-            };
-
-            assert!(
-                !replicate_delete_with_outcome(delete, storage).await,
-                "unfinished target work, source I/O failures, and marker creation must remain retryable"
-            );
-        }
-    }
-
     #[tokio::test]
     async fn delete_worker_preserves_persisted_admission_without_live_config() {
         let arn = "arn:test";
@@ -5074,9 +4794,10 @@ mod tests {
     #[tokio::test]
     async fn heal_queue_marks_missing_versioning_state_as_missed() {
         use super::super::replication_target_boundary::BucketTargets;
-        use s3s::dto::{
-            DeleteReplication, DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule,
-            ReplicationRuleStatus,
+        use rustfs_gateway_types::dto::Status;
+        use rustfs_gateway_types::persistence::{
+            PersistedReplicationConfiguration, PersistedReplicationDestination, PersistedReplicationRule,
+            PersistedReplicationStatus,
         };
 
         let arn = "arn:rustfs:replication:us-east-1:target:bucket";
@@ -5091,14 +4812,14 @@ mod tests {
                 ..Default::default()
             },
             ReplicationConfig::new(
-                Some(ReplicationConfiguration {
+                Some(PersistedReplicationConfiguration {
                     role: String::new(),
-                    rules: vec![ReplicationRule {
+                    rules: vec![PersistedReplicationRule {
                         delete_marker_replication: None,
-                        delete_replication: Some(DeleteReplication {
-                            status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
+                        delete_replication: Some(PersistedReplicationStatus {
+                            status: Status::ENABLED.to_string(),
                         }),
-                        destination: Destination {
+                        destination: PersistedReplicationDestination {
                             bucket: arn.to_string(),
                             ..Default::default()
                         },
@@ -5108,7 +4829,7 @@ mod tests {
                         prefix: Some(String::new()),
                         priority: Some(1),
                         source_selection_criteria: None,
-                        status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
+                        status: Status::ENABLED.to_string(),
                     }],
                 }),
                 Some(BucketTargets::default()),
@@ -5584,8 +5305,6 @@ mod tests {
                 allow_first_read: Notify::new(),
                 read_count: AtomicUsize::new(0),
                 delete_storage_calls: AtomicUsize::new(0),
-                delete_error: StdMutex::new(EcstoreError::NotImplemented),
-                replay_stamp: StdMutex::new(None),
                 reported_size: StdMutex::new(None),
                 stream_read_bytes: Arc::new(AtomicUsize::new(0)),
                 write_count: AtomicUsize::new(0),
@@ -6550,108 +6269,6 @@ mod tests {
         assert_eq!(retained.target_arns, entry.target_arns);
     }
 
-    #[tokio::test]
-    async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
-        let shared = empty_resync_shared_state();
-        let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
-        // A force-delete intent from a superseded generation replays as
-        // Skipped, so a pass that reaches it acknowledges it off the journal.
-        let stale_intent = |object: &str| MrfReplicateEntry {
-            bucket: "mrf-runtime-replay".to_string(),
-            object: object.to_string(),
-            op: MrfOpKind::Delete,
-            force_delete_id: Some(uuid::Uuid::new_v4()),
-            force_delete_local_commit: false,
-            target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
-            ..Default::default()
-        };
-        let journal_drained = || async {
-            tokio::time::timeout(Duration::from_secs(5), async {
-                while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
-                    tokio::time::sleep(Duration::from_millis(10)).await;
-                }
-            })
-            .await
-            .is_ok()
-        };
-
-        let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
-        for object in ["first", "second"] {
-            *shared.data.lock().expect("test data lock should not be poisoned") =
-                encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
-            assert!(
-                journal_drained().await,
-                "an MRF entry persisted while the server runs must be replayed without a restart ({object})"
-            );
-        }
-        handle.abort();
-    }
-
-    #[tokio::test]
-    async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
-        let shared = empty_resync_shared_state();
-        let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
-        let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
-        let interval = Duration::from_secs(1);
-        let persist_backlog = |object: &str| {
-            let entry = MrfReplicateEntry {
-                bucket: "mrf-node-replay".to_string(),
-                object: object.to_string(),
-                op: MrfOpKind::Delete,
-                force_delete_id: Some(uuid::Uuid::new_v4()),
-                force_delete_local_commit: false,
-                target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
-                ..Default::default()
-            };
-            *shared.data.lock().expect("test data lock should not be poisoned") =
-                encode_mrf_file(&[entry]).expect("MRF entry should encode");
-        };
-        let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
-
-        persist_backlog("first");
-        assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
-        assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
-
-        // Node B's timer fires within the same interval with a backlog present.
-        persist_backlog("second");
-        assert!(
-            !replay_mrf_journal(&node_b, Some(interval)).await,
-            "a node must not replay within one interval of another node's pass"
-        );
-        assert!(!journal_is_empty(), "the backlog should wait for the next interval");
-
-        assert!(
-            replay_mrf_journal(&node_b, None).await,
-            "the startup pass should replay regardless of the shared stamp"
-        );
-        assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
-
-        persist_backlog("third");
-        tokio::time::sleep(interval).await;
-        assert!(
-            replay_mrf_journal(&node_a, Some(interval)).await,
-            "a periodic pass should replay once the interval has elapsed"
-        );
-        assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
-    }
-
-    #[test]
-    fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
-        let interval = Duration::from_secs(300);
-        let now = 1_000_000_000;
-        assert!(mrf_replay_due(None, now, interval));
-        assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
-        assert!(mrf_replay_due(Some(now - 300_000), now, interval));
-        assert!(
-            !mrf_replay_due(Some(now + 1_000), now, interval),
-            "small clock skew should still space passes"
-        );
-        assert!(
-            mrf_replay_due(Some(now + 300_000), now, interval),
-            "a stamp an interval ahead must not suppress replay"
-        );
-    }
-
     #[test]
     fn mrf_entry_delete_marker_roundtrip() {
         let dm_vid = Uuid::new_v4();

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -85,6 +85,8 @@ use http_body_util::StreamBody;
 use metrics::counter;
 #[cfg(test)]
 use rmp_serde;
+#[cfg(test)]
+use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
 use rustfs_s3_types::EventName;
 use rustfs_utils::http::{
     AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_TAGGING_DIRECTIVE,
@@ -92,8 +94,6 @@ use rustfs_utils::http::{
     insert_str, replication_target_versions,
 };
 use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash};
-#[cfg(test)]
-use s3s::dto::ReplicationConfiguration;
 use std::collections::{HashMap, HashSet};
 use std::fmt::Display;
 use std::pin::Pin;
@@ -2809,22 +2809,17 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
             true
         }
         Err(e) => {
-            // A batch purge or another worker may have removed this version
-            // already. Remote failures still control the final outcome below.
-            let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
-            if !source_already_purged {
-                error!(
-                    event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
-                    bucket = %bucket,
-                    arn = %dobj.target_arn,
-                    object = %dobj.delete_object.object_name,
-                    operation = "apply_replication_delete_state",
-                    error = %e,
-                    "Replication target operation failed"
-                );
-            }
+            error!(
+                event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+                bucket = %bucket,
+                arn = %dobj.target_arn,
+                object = %dobj.delete_object.object_name,
+                operation = "apply_replication_delete_state",
+                error = %e,
+                "Replication target operation failed"
+            );
             send_event(EventArgs {
                 event_name,
                 bucket_name: bucket.clone(),
@@ -2837,7 +2832,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
                 },
                 ..Default::default()
             });
-            source_already_purged
+            false
         }
     };
 
@@ -3015,7 +3010,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
 /// If the marker disappears (deleted before or while the replica landed),
 /// purge the replicated marker from the targets, retrying failed targets on
 /// later rounds. When the window drains with targets still dirty, persist the
-/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
+/// purge intent as a durable MRF entry so the next startup replays it through
 /// `purge_stale_delete_marker_targets`.
 async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
     bucket: String,
@@ -3106,7 +3101,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
 /// Heal callers retry for the full watch window because the startup MRF
 /// processor runs before bucket metadata (and thus target clients) finishes
 /// initializing — the first pass can see `target_client_missing` and a later
-/// round resolves the client; replay passes are serial background work, so
+/// round resolves the client; the replay loop is serial and startup-only, so
 /// blocking it for up to the window per dirty entry is acceptable. Live
 /// callers run on replication workers where a down target would pin a worker
 /// for the whole window, so they attempt once and lean on the durable intent
@@ -3287,18 +3282,10 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
 
     let mut join_set = JoinSet::new();
     let mut all_succeeded = true;
-    let mut target_infos = Vec::with_capacity(tgt_arns.len());
 
     for arn in tgt_arns {
         let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &arn).await else {
             all_succeeded = false;
-            target_infos.push(force_delete_target_info(
-                dobj.op_type,
-                &arn,
-                "",
-                false,
-                Err(TARGET_CLIENT_UNAVAILABLE_ERROR.to_string()),
-            ));
             debug!(
                 event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
                 component = LOG_COMPONENT_ECSTORE,
@@ -3324,7 +3311,6 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
 
         let bucket = bucket.clone();
         let object_name = object_name.clone();
-        let op_type = dobj.op_type;
 
         join_set.spawn(async move {
             if ReplicationTargetStore::target_is_offline(&tgt_client).await {
@@ -3349,16 +3335,9 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
                     user_agent: "Internal: [Replication]".to_string(),
                     ..Default::default()
                 });
-                return force_delete_target_info(
-                    op_type,
-                    &tgt_client.arn,
-                    &tgt_client.endpoint,
-                    tgt_client.secure,
-                    Err(FORCE_DELETE_TARGET_OFFLINE_ERROR.to_string()),
-                );
+                return false;
             }
 
-            let started = Instant::now();
             if let Err(e) = tgt_client
                 .remove_object(&tgt_client.bucket, &object_name, None, replication_force_delete_remove_options())
                 .await
@@ -3385,25 +3364,16 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
                     user_agent: "Internal: [Replication]".to_string(),
                     ..Default::default()
                 });
-                return force_delete_target_info(
-                    op_type,
-                    &tgt_client.arn,
-                    &tgt_client.endpoint,
-                    tgt_client.secure,
-                    Err(e.to_string()),
-                );
+                return false;
             }
 
-            force_delete_target_info(op_type, &tgt_client.arn, &tgt_client.endpoint, tgt_client.secure, Ok(started.elapsed()))
+            true
         });
     }
 
     while let Some(result) = join_set.join_next().await {
         match result {
-            Ok(info) => {
-                all_succeeded &= info.replication_status == ReplicationStatusType::Completed;
-                target_infos.push(info);
-            }
+            Ok(success) => all_succeeded &= success,
             Err(error) => {
                 all_succeeded = false;
                 error!(
@@ -3420,8 +3390,6 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
         }
     }
 
-    record_force_delete_target_stats(bucket, &target_infos).await;
-
     if all_succeeded
         && let Some(operation_id) = dobj.delete_object.force_delete_id
         && let Err(error) = super::replication_pool::complete_force_delete_intent(storage, operation_id).await
@@ -3441,53 +3409,6 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
     all_succeeded
 }
 
-const FORCE_DELETE_TARGET_OFFLINE_ERROR: &str = "replication target is offline";
-
-/// One target's outcome of a replicated force-delete in the shape the
-/// replication statistics consume. A force-delete carries no per-version
-/// replication state, so the previous status is always `Pending`: that is
-/// the transition `ReplicationStats::update` counts as a failure, whereas an
-/// `Empty` previous status would drop the outcome and leave `replication
-/// status` reporting zero failures while the peer still holds the objects.
-fn force_delete_target_info(
-    op_type: ReplicationType,
-    arn: &str,
-    endpoint: &str,
-    secure: bool,
-    outcome: std::result::Result<std::time::Duration, String>,
-) -> ReplicatedTargetInfo {
-    let mut rinfo = ReplicatedTargetInfo {
-        arn: arn.to_string(),
-        op_type,
-        endpoint: endpoint.to_string(),
-        secure,
-        prev_replication_status: ReplicationStatusType::Pending,
-        ..Default::default()
-    };
-    match outcome {
-        Ok(duration) => {
-            rinfo.replication_status = ReplicationStatusType::Completed;
-            rinfo.duration = duration;
-        }
-        Err(error) => {
-            rinfo.replication_status = ReplicationStatusType::Failed;
-            rinfo.error = Some(error);
-        }
-    }
-    rinfo
-}
-
-async fn record_force_delete_target_stats(bucket: &str, targets: &[ReplicatedTargetInfo]) {
-    let Some(stats) = runtime_sources::replication_stats() else {
-        return;
-    };
-    for target in targets {
-        stats
-            .update(bucket, target, target.replication_status.clone(), target.prev_replication_status.clone())
-            .await;
-    }
-}
-
 fn unavailable_delete_target_info(dobj: &DeletedObjectReplicationInfo, arn: &str) -> ReplicatedTargetInfo {
     let mut rinfo = dobj
         .delete_object
@@ -6579,9 +6500,10 @@ mod tests {
 
     use super::super::replication_target_boundary::{BucketTarget, BucketTargets};
     use super::*;
-    use s3s::dto::{
-        BucketVersioningStatus, DeleteReplication, DeleteReplicationStatus, Destination, ExcludedPrefix, ReplicationRule,
-        ReplicationRuleStatus, VersioningConfiguration,
+    use rustfs_gateway_types::dto::Status;
+    use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
+    use rustfs_gateway_types::persistence::{
+        PersistedReplicationDestination, PersistedReplicationRule, PersistedReplicationStatus,
     };
     use std::collections::HashMap;
     use time::OffsetDateTime;
@@ -6609,7 +6531,6 @@ mod tests {
             health_check_duration: std::time::Duration::from_secs(5),
             replicate_sync: false,
             client: Arc::new(aws_sdk_s3::Client::from_conf(config)),
-            timeouts: Default::default(),
         })
     }
 
@@ -7524,51 +7445,6 @@ mod tests {
         server.join().expect("test HTTP server should finish");
     }
 
-    /// A force-delete the peer rejects must reach the replication failure
-    /// counters. A replicated force-delete used to only log the rejection, so
-    /// `replication status` reported zero failures while the sites diverged.
-    #[tokio::test]
-    async fn force_delete_target_outcomes_reach_replication_stats() {
-        let stats = super::super::replication_state::ReplicationStats::new();
-        let denied = force_delete_target_info(
-            ReplicationType::Delete,
-            "arn:rustfs:replication::denied:bucket",
-            "http://peer-a:9000",
-            false,
-            Err("remove_object failed: AccessDenied".to_string()),
-        );
-        assert_eq!(denied.replication_status, ReplicationStatusType::Failed);
-        assert_eq!(denied.prev_replication_status, ReplicationStatusType::Pending);
-        assert_eq!(denied.error.as_deref(), Some("remove_object failed: AccessDenied"));
-
-        let accepted = force_delete_target_info(
-            ReplicationType::Heal,
-            "arn:rustfs:replication::accepted:bucket",
-            "https://peer-b:9000",
-            true,
-            Ok(std::time::Duration::from_millis(3)),
-        );
-        assert_eq!(accepted.replication_status, ReplicationStatusType::Completed);
-        assert!(accepted.error.is_none());
-
-        for target in [&denied, &accepted] {
-            stats
-                .update(
-                    "bucket",
-                    target,
-                    target.replication_status.clone(),
-                    target.prev_replication_status.clone(),
-                )
-                .await;
-        }
-
-        let bucket_stats = stats.get("bucket").await;
-        let denied_stat = bucket_stats.stats.get(&denied.arn).expect("denied target: stats entry");
-        assert_eq!((denied_stat.failed.count, denied_stat.replicated_count), (1, 0));
-        let accepted_stat = bucket_stats.stats.get(&accepted.arn).expect("accepted target: stats entry");
-        assert_eq!((accepted_stat.failed.count, accepted_stat.replicated_count), (0, 1));
-    }
-
     /// AWS answers `HEAD ?versionId=<source marker id>` with a bodiless 400.
     /// The marker is verified through the latest version instead: a
     /// versionless 404 with `x-amz-delete-marker: true` proves it propagated.
@@ -8693,7 +8569,7 @@ mod tests {
             ..Default::default()
         };
         let rcfg = ReplicationConfig::new(
-            Some(ReplicationConfiguration {
+            Some(PersistedReplicationConfiguration {
                 role: role.to_string(),
                 rules: vec![],
             }),
@@ -8719,7 +8595,7 @@ mod tests {
         ReplicationVersioningStore::install_prefix_state_test_config(
             &bucket,
             VersioningConfiguration {
-                status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
+                status: Some("Enabled".to_owned()),
                 ..Default::default()
             },
         );
@@ -8752,22 +8628,20 @@ mod tests {
         ReplicationVersioningStore::install_prefix_state_test_config(
             &bucket,
             VersioningConfiguration {
-                status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
-                excluded_prefixes: Some(vec![ExcludedPrefix {
-                    prefix: Some("archive/".to_string()),
-                }]),
+                status: Some("Enabled".to_owned()),
+                excluded_prefixes: Some(vec![Some("archive/".to_string())]),
                 ..Default::default()
             },
         );
         let rcfg = ReplicationConfig::new(
-            Some(ReplicationConfiguration {
+            Some(PersistedReplicationConfiguration {
                 role: String::new(),
-                rules: vec![ReplicationRule {
+                rules: vec![PersistedReplicationRule {
                     delete_marker_replication: None,
-                    delete_replication: Some(DeleteReplication {
-                        status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
+                    delete_replication: Some(PersistedReplicationStatus {
+                        status: Status::ENABLED.to_string(),
                     }),
-                    destination: Destination {
+                    destination: PersistedReplicationDestination {
                         bucket: arn.to_string(),
                         ..Default::default()
                     },
@@ -8777,7 +8651,7 @@ mod tests {
                     prefix: Some(String::new()),
                     priority: Some(1),
                     source_selection_criteria: None,
-                    status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
+                    status: Status::ENABLED.to_string(),
                 }],
             }),
             Some(BucketTargets {
@@ -8813,14 +8687,14 @@ mod tests {
         let admitted_arn = "arn:rustfs:replication:us-east-1:target:admitted";
         let current_role = "arn:rustfs:replication:us-east-1:target:current";
         let rcfg = ReplicationConfig::new(
-            Some(ReplicationConfiguration {
+            Some(PersistedReplicationConfiguration {
                 role: current_role.to_string(),
-                rules: vec![ReplicationRule {
+                rules: vec![PersistedReplicationRule {
                     delete_marker_replication: None,
-                    delete_replication: Some(DeleteReplication {
-                        status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
+                    delete_replication: Some(PersistedReplicationStatus {
+                        status: Status::DISABLED.to_string(),
                     }),
-                    destination: Destination {
+                    destination: PersistedReplicationDestination {
                         bucket: current_role.to_string(),
                         ..Default::default()
                     },
@@ -8830,7 +8704,7 @@ mod tests {
                     prefix: Some(String::new()),
                     priority: Some(1),
                     source_selection_criteria: None,
-                    status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
+                    status: Status::ENABLED.to_string(),
                 }],
             }),
             Some(BucketTargets {
@@ -8878,12 +8752,12 @@ mod tests {
     #[tokio::test]
     async fn heal_rejects_semantically_invalid_replication_config() {
         let rcfg = ReplicationConfig::new(
-            Some(ReplicationConfiguration {
+            Some(PersistedReplicationConfiguration {
                 role: String::new(),
-                rules: vec![ReplicationRule {
+                rules: vec![PersistedReplicationRule {
                     delete_marker_replication: None,
                     delete_replication: None,
-                    destination: Destination {
+                    destination: PersistedReplicationDestination {
                         bucket: "arn:rustfs:replication:us-east-1:target:bucket".to_string(),
                         ..Default::default()
                     },
@@ -8893,7 +8767,7 @@ mod tests {
                     prefix: Some(String::new()),
                     priority: Some(1),
                     source_selection_criteria: None,
-                    status: ReplicationRuleStatus::from_static("Enabld"),
+                    status: "Enabld".to_owned(),
                 }],
             }),
             Some(BucketTargets::default()),

crates/ecstore/src/bucket/replication/replication_target_config_bridge.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use s3s::dto::ReplicationConfiguration;
+use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
 
 use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt};
 use super::replication_filemeta_boundary::ReplicationType;
@@ -20,7 +20,7 @@ use super::replication_filemeta_boundary::ReplicationType;
 pub(crate) struct ReplicationTargetConfigBridge;
 
 impl ReplicationTargetConfigBridge {
-    pub(crate) fn target_is_used_by_rules(config: &ReplicationConfiguration, arn: &str) -> bool {
+    pub(crate) fn target_is_used_by_rules(config: &PersistedReplicationConfiguration, arn: &str) -> bool {
         config
             .filter_target_arns(&ObjectOpts {
                 op_type: ReplicationType::All,
@@ -37,7 +37,7 @@ mod tests {
 
     #[test]
     fn target_config_bridge_matches_role_target() {
-        let config = ReplicationConfiguration {
+        let config = PersistedReplicationConfiguration {
             role: " arn:target ".to_string(),
             rules: Vec::new(),
         };

crates/ecstore/src/bucket/replication/replication_versioning_boundary.rs

@@ -15,7 +15,7 @@
 use super::replication_error_boundary::Result;
 use crate::bucket::{versioning::VersioningApi as _, versioning_sys::BucketVersioningSys};
 #[cfg(test)]
-use s3s::dto::VersioningConfiguration;
+use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
 #[cfg(test)]
 use std::{collections::HashMap, sync::LazyLock, sync::Mutex};
 

crates/ecstore/src/bucket/tagging/mod.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use s3s::dto::Tag;
+use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
 use std::cmp::Ordering;
 use std::collections::HashMap;
 use url::form_urlencoded;

crates/ecstore/src/bucket/utils.rs

@@ -15,9 +15,23 @@
 use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
 use crate::error::{Error, Result, StorageError};
 use regex::Regex;
+use rustfs_gateway_types::cors_tagging::{
+    CorsTaggingCodecError, PersistedCorsConfiguration, PersistedTagging, parse_cors, parse_tagging, serialize_cors,
+    serialize_tagging,
+};
+use rustfs_gateway_types::persistence::{
+    PersistedAccelerateConfiguration, PersistedBucketEncryptionConfiguration, PersistedBucketLoggingStatus,
+    PersistedLifecycleConfiguration, PersistedNotificationConfiguration, PersistedObjectLockConfiguration,
+    PersistedPublicAccessBlockConfiguration, PersistedReplicationConfiguration, PersistedRequestPaymentConfiguration,
+    PersistedVersioningConfiguration, PersistedWebsiteConfiguration, PersistenceCodecError, parse_accelerate,
+    parse_bucket_encryption, parse_bucket_logging, parse_lifecycle_legacy_dates, parse_notification, parse_object_lock,
+    parse_public_access_block, parse_replication, parse_request_payment, parse_versioning, parse_website, serialize_accelerate,
+    serialize_bucket_encryption, serialize_bucket_logging, serialize_lifecycle_legacy_dates, serialize_notification,
+    serialize_object_lock, serialize_public_access_block, serialize_replication, serialize_request_payment, serialize_versioning,
+    serialize_website,
+};
 use rustfs_utils::WindowsFileNameError;
 use rustfs_utils::path::SLASH_SEPARATOR;
-use s3s::xml;
 use tracing::instrument;
 
 pub fn is_meta_bucketname(name: &str) -> bool {
@@ -74,37 +88,173 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> {
     check_bucket_name_common(bucket_name, true)
 }
 
-// RUSTFS_COMPAT_TODO(s3gate-metadata-xml): the s3s codec reads persisted XML during migration. Remove after every supported writer uses the gateway codec and every retained metadata object and backup archive is verified or rewritten.
-pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
-where
-    T: for<'xml> xml::Deserialize<'xml>,
-{
-    let mut d = xml::Deserializer::new(input);
-    let ans = T::deserialize(&mut d)?;
-    d.expect_eof()?;
-    Ok(ans)
+/// A bucket configuration persisted as XML in the bucket metadata, together
+/// with the codec that reads and writes its stored bytes.
+///
+/// Every family reads and writes its stored bytes through the gateway
+/// persistence codecs (`rustfs_gateway_types::persistence` and
+/// `rustfs_gateway_types::cors_tagging`, rustfs/backlog#2745). The codecs keep
+/// the persisted bytes the `bucket_config_goldens` pin, so a document the old
+/// s3s codec wrote reads back and re-encodes unchanged.
+pub trait BucketConfigXml: Sized {
+    /// Decodes the stored bytes of one configuration document.
+    fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError>;
+
+    /// Encodes the value as RustFS stores it.
+    fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError>;
 }
 
-#[allow(
-    dead_code,
-    reason = "xml serialize helper with no caller in this port; the live sibling is deserialize (backlog#1823)"
-)]
-pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
-    let mut buf = Vec::with_capacity(256);
-    {
-        let mut ser = xml::Serializer::new(&mut buf);
-        val.serialize_content(&mut ser)?;
-    }
-    Ok(String::from_utf8(buf).unwrap())
+/// Why a persisted bucket configuration could not be decoded or encoded.
+#[derive(Debug, thiserror::Error)]
+pub enum BucketConfigXmlError {
+    /// A gateway persistence codec refused the bytes or the value.
+    #[error(transparent)]
+    Persisted(#[from] PersistenceCodecError),
+    /// The gateway CORS or Tagging codec refused the bytes or the value.
+    #[error(transparent)]
+    CorsTagging(#[from] CorsTaggingCodecError),
 }
 
-pub fn serialize<T: xml::Serialize>(val: &T) -> xml::SerResult<Vec<u8>> {
-    let mut buf = Vec::with_capacity(256);
-    {
-        let mut ser = xml::Serializer::new(&mut buf);
-        val.serialize(&mut ser)?;
+/// Decodes one persisted bucket configuration document.
+pub fn deserialize<T: BucketConfigXml>(input: &[u8]) -> std::result::Result<T, BucketConfigXmlError> {
+    T::decode_config_xml(input)
+}
+
+/// Encodes one bucket configuration as RustFS persists it.
+pub fn serialize<T: BucketConfigXml>(val: &T) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
+    val.encode_config_xml()
+}
+
+impl BucketConfigXml for PersistedReplicationConfiguration {
+    fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
+        Ok(parse_replication(input)?)
+    }
+
+    fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
+        Ok(serialize_replication(self)?)
+    }
+}
+
+impl BucketConfigXml for PersistedLifecycleConfiguration {
+    fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
+        Ok(parse_lifecycle_legacy_dates(input)?)
+    }
+
+    fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
+        Ok(serialize_lifecycle_legacy_dates(self)?)
+    }
+}
+
+// The gateway writers of these families are infallible: a value that exists is
+// always encodable, so only the readers can refuse bytes.
+impl BucketConfigXml for PersistedAccelerateConfiguration {
+    fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
+        Ok(parse_accelerate(input)?)
+    }
+
+    fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
+        Ok(serialize_accelerate(self))
+    }
+}
+
+impl BucketConfigXml for PersistedBucketLoggingStatus {
+    fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
+        Ok(parse_bucket_logging(input)?)
+    }
+
+    fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
+        Ok(serialize_bucket_logging(self))
+    }
+}
+
+impl BucketConfigXml for PersistedCorsConfiguration {
+    fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
+        Ok(parse_cors(input)?)
+    }
+
+    fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
+        Ok(serialize_cors(self))
+    }
+}
+
+impl BucketConfigXml for PersistedNotificationConfiguration {
+    fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
+        Ok(parse_notification(input)?)
+    }
+
+    fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
+        Ok(serialize_notification(self))
+    }
+}
+
+impl BucketConfigXml for PersistedObjectLockConfiguration {
+    fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
+        Ok(parse_object_lock(input)?)
+    }
+
+    fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
+        Ok(serialize_object_lock(self))
+    }
+}
+
+impl BucketConfigXml for PersistedPublicAccessBlockConfiguration {
+    fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
+        Ok(parse_public_access_block(input)?)
+    }
+
+    fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
+        Ok(serialize_public_access_block(self))
+    }
+}
+
+impl BucketConfigXml for PersistedRequestPaymentConfiguration {
+    fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
+        Ok(parse_request_payment(input)?)
+    }
+
+    fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
+        Ok(serialize_request_payment(self))
+    }
+}
+
+impl BucketConfigXml for PersistedBucketEncryptionConfiguration {
+    fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
+        Ok(parse_bucket_encryption(input)?)
+    }
+
+    fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
+        Ok(serialize_bucket_encryption(self))
+    }
+}
+
+impl BucketConfigXml for PersistedTagging {
+    fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
+        Ok(parse_tagging(input)?)
+    }
+
+    fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
+        Ok(serialize_tagging(self))
+    }
+}
+
+impl BucketConfigXml for PersistedVersioningConfiguration {
+    fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
+        Ok(parse_versioning(input)?)
+    }
+
+    fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
+        Ok(serialize_versioning(self))
+    }
+}
+
+impl BucketConfigXml for PersistedWebsiteConfiguration {
+    fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
+        Ok(parse_website(input)?)
+    }
+
+    fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
+        Ok(serialize_website(self))
     }
-    Ok(buf)
 }
 
 pub fn has_bad_path_component(path: &str) -> bool {
@@ -259,10 +409,6 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
 pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) -> Result<()> {
     check_put_object_args(bucket, object)?;
 
-    if !data_movement && !is_meta_bucketname(bucket) && has_xml_invalid_control_characters(object) {
-        return Err(StorageError::ObjectNameInvalid(bucket.to_owned(), object.to_owned()));
-    }
-
     if cfg!(target_os = "windows") && !data_movement && !is_meta_bucketname(bucket) && object.contains('\\') {
         return Err(StorageError::InvalidArgument(
             bucket.to_owned(),
@@ -274,14 +420,6 @@ pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) ->
     Ok(())
 }
 
-/// New public keys must remain representable in an unencoded XML listing.
-/// Existing keys still use the read/delete validators so they can be removed.
-pub fn has_xml_invalid_control_characters(object: &str) -> bool {
-    object
-        .chars()
-        .any(|character| character < ' ' && !matches!(character, '\t' | '\n' | '\r'))
-}
-
 pub fn check_copy_obj_args(bucket: &str, object: &str) -> Result<()> {
     check_bucket_and_object_names(bucket, object)
 }
@@ -446,6 +584,40 @@ mod tests {
     use super::*;
     use proptest::prelude::*;
 
+    /// Every bucket-configuration family reads and writes through its own
+    /// gateway codec (rustfs/backlog#2745): the family's golden document
+    /// decodes and re-encodes to the same bytes, and a document of another
+    /// family is refused instead of decoding into an empty configuration.
+    #[test]
+    fn every_bucket_config_family_uses_its_own_gateway_codec() {
+        fn assert_family<T: BucketConfigXml>(family: &str, own: &[u8], foreign: &[u8]) {
+            let value = deserialize::<T>(own).unwrap_or_else(|err| panic!("{family}: own document must decode: {err}"));
+            let encoded = serialize(&value).unwrap_or_else(|err| panic!("{family}: value must encode: {err}"));
+            assert_eq!(encoded, own, "{family}: re-encoding must keep the persisted bytes");
+            assert!(deserialize::<T>(foreign).is_err(), "{family}: a foreign root element must be refused");
+        }
+
+        macro_rules! golden {
+            ($family:literal) => {
+                include_bytes!(concat!("../../tests/fixtures/bucket-config-goldens/", $family, "/minio.out.xml"))
+            };
+        }
+        let tagging = golden!("tagging");
+        let versioning = golden!("versioning");
+
+        assert_family::<PersistedAccelerateConfiguration>("accelerate", golden!("accelerate"), tagging);
+        assert_family::<PersistedBucketLoggingStatus>("logging", golden!("logging"), tagging);
+        assert_family::<PersistedCorsConfiguration>("cors", golden!("cors"), tagging);
+        assert_family::<PersistedNotificationConfiguration>("notification", golden!("notification"), tagging);
+        assert_family::<PersistedObjectLockConfiguration>("object_lock", golden!("object_lock"), tagging);
+        assert_family::<PersistedPublicAccessBlockConfiguration>("public_access_block", golden!("public_access_block"), tagging);
+        assert_family::<PersistedRequestPaymentConfiguration>("request_payment", golden!("request_payment"), tagging);
+        assert_family::<PersistedBucketEncryptionConfiguration>("encryption", golden!("encryption"), tagging);
+        assert_family::<PersistedTagging>("tagging", tagging, versioning);
+        assert_family::<PersistedVersioningConfiguration>("versioning", versioning, tagging);
+        assert_family::<PersistedWebsiteConfiguration>("website", golden!("website"), tagging);
+    }
+
     /// rustfs#5785: keys whose path segments exceed the on-disk NAME_MAX
     /// budget must be rejected up front as ObjectNameInvalid (4xx), not leak
     /// ENAMETOOLONG as InternalError 500 from the disk layer.

crates/ecstore/src/bucket/versioning/mod.rs

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
 use rustfs_utils::string::match_simple;
-use s3s::dto::{BucketVersioningStatus, VersioningConfiguration};
 
 pub trait VersioningApi {
     fn enabled(&self) -> bool;
@@ -28,10 +28,10 @@ pub trait VersioningApi {
 
 impl VersioningApi for VersioningConfiguration {
     fn enabled(&self) -> bool {
-        self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED))
+        self.status.as_deref() == Some("Enabled")
     }
     fn prefix_enabled(&self, prefix: &str) -> bool {
-        if self.status != Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)) {
+        if self.status.as_deref() != Some("Enabled") {
             return false;
         }
 
@@ -44,12 +44,10 @@ impl VersioningApi for VersioningConfiguration {
         }
 
         if let Some(ref excluded_prefixes) = self.excluded_prefixes {
-            for p in excluded_prefixes.iter() {
-                if let Some(ref sprefix) = p.prefix {
-                    let pattern = format!("{sprefix}*");
-                    if match_simple(&pattern, prefix) {
-                        return false;
-                    }
+            for sprefix in excluded_prefixes.iter().flatten() {
+                let pattern = format!("{sprefix}*");
+                if match_simple(&pattern, prefix) {
+                    return false;
                 }
             }
         }
@@ -58,11 +56,11 @@ impl VersioningApi for VersioningConfiguration {
     }
 
     fn prefix_suspended(&self, prefix: &str) -> bool {
-        if self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)) {
+        if self.status.as_deref() == Some("Suspended") {
             return true;
         }
 
-        if self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)) {
+        if self.status.as_deref() == Some("Enabled") {
             if prefix.is_empty() {
                 return false;
             }
@@ -75,12 +73,10 @@ impl VersioningApi for VersioningConfiguration {
             }
 
             if let Some(ref excluded_prefixes) = self.excluded_prefixes {
-                for p in excluded_prefixes.iter() {
-                    if let Some(ref sprefix) = p.prefix {
-                        let pattern = format!("{sprefix}*");
-                        if match_simple(&pattern, prefix) {
-                            return true;
-                        }
+                for sprefix in excluded_prefixes.iter().flatten() {
+                    let pattern = format!("{sprefix}*");
+                    if match_simple(&pattern, prefix) {
+                        return true;
                     }
                 }
             }
@@ -92,14 +88,12 @@ impl VersioningApi for VersioningConfiguration {
         self.prefix_enabled(prefix) || self.prefix_suspended(prefix)
     }
     fn suspended(&self) -> bool {
-        self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED))
+        self.status.as_deref() == Some("Suspended")
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use s3s::dto::{BucketVersioningStatus, ExcludedPrefix};
-
     use super::*;
 
     struct LegacyVersioning;
@@ -134,10 +128,8 @@ mod tests {
     #[test]
     fn delete_state_treats_excluded_prefixes_as_unversioned() {
         let config = VersioningConfiguration {
-            status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
-            excluded_prefixes: Some(vec![ExcludedPrefix {
-                prefix: Some("archive/".to_string()),
-            }]),
+            status: Some("Enabled".to_owned()),
+            excluded_prefixes: Some(vec![Some("archive/".to_string())]),
             ..Default::default()
         };
 
@@ -146,7 +138,7 @@ mod tests {
         assert_eq!(config.delete_state("live/object"), (true, false));
 
         let suspended = VersioningConfiguration {
-            status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
+            status: Some("Suspended".to_owned()),
             ..Default::default()
         };
         assert_eq!(suspended.delete_state("archive/object"), (false, true));

crates/ecstore/src/bucket/versioning_sys.rs

@@ -17,7 +17,7 @@ use super::metadata::unreadable_config_refusal;
 use super::{metadata_sys::get_bucket_metadata_sys, versioning::VersioningApi};
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::Result;
-use s3s::dto::VersioningConfiguration;
+use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
 use tracing::{error, warn};
 
 pub struct BucketVersioningSys {}
@@ -163,7 +163,7 @@ mod tests {
 
     fn enabled() -> VersioningConfiguration {
         VersioningConfiguration {
-            status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
+            status: Some("Enabled".to_owned()),
             ..Default::default()
         }
     }

crates/ecstore/src/cache_value/metacache_set.rs

@@ -224,8 +224,6 @@ pub struct ListPathRawOptions {
     pub recursive: bool,
     pub incl_deleted: bool,
     pub skip_hidden_prefix_check: bool,
-    pub hidden_entry_budget: usize,
-    pub pending_prefix: Option<String>,
     pub filter_prefix: Option<String>,
     pub forward_to: Option<String>,
     pub min_disks: usize,
@@ -262,8 +260,6 @@ impl Clone for ListPathRawOptions {
             recursive: self.recursive,
             incl_deleted: self.incl_deleted,
             skip_hidden_prefix_check: self.skip_hidden_prefix_check,
-            hidden_entry_budget: self.hidden_entry_budget,
-            pending_prefix: self.pending_prefix.clone(),
             filter_prefix: self.filter_prefix.clone(),
             forward_to: self.forward_to.clone(),
             min_disks: self.min_disks,
@@ -293,8 +289,6 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
         recursive: opts.recursive,
         incl_deleted: opts.incl_deleted,
         skip_hidden_prefix_check: opts.skip_hidden_prefix_check,
-        hidden_entry_budget: opts.hidden_entry_budget,
-        pending_prefix: opts.pending_prefix.clone(),
         report_notfound: opts.report_not_found,
         filter_prefix: opts.filter_prefix.clone(),
         forward_to: opts.forward_to.clone(),
@@ -307,15 +301,13 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
 }
 
 /// How a merged raw listing stream ended.
-#[derive(Clone, Debug, Default, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
 pub struct ListPathRawOutcome {
     /// At least one producer stopped at its per-disk limit instead of at the
     /// end of its namespace, so the closed stream is not proof of exhaustion.
     /// Carried in-band by the metacache stream trailer, so remote producers
     /// report it as well.
     pub limit_reached: bool,
-    /// Inclusive sorted boundary proved by every reader before a bounded stop.
-    pub scan_cut: Option<String>,
 }
 
 pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
@@ -730,8 +722,6 @@ async fn list_path_raw_inner(
         // Sticky across rounds: once any reader's trailer says its producer
         // stopped at the per-disk limit, the merged stream may hide more keys.
         let mut limit_reached = false;
-        let mut scan_cut = None;
-        let mut last_scanned_key: Option<String> = None;
 
         loop {
             let mut current = MetaCacheEntry::default();
@@ -743,7 +733,7 @@ async fn list_path_raw_inner(
             // );
 
             if revjob_rx.is_cancelled() {
-                return Ok(ListPathRawOutcome { limit_reached, scan_cut });
+                return Ok(ListPathRawOutcome { limit_reached });
             }
 
             let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
@@ -757,16 +747,6 @@ async fn list_path_raw_inner(
             // Start every missing head read in the same round so one stalled
             // disk cannot multiply the wait budget by the erasure-set width.
             // Outcomes are still consumed below in stable disk-index order.
-            if !revjob_rx.is_cancelled() {
-                for (i, reader) in readers.iter_mut().enumerate() {
-                    if errs[i].is_none()
-                        && pending_entries[i].is_none()
-                        && let Some(Ok(Some(entry))) = reader.try_peek_buffered()
-                    {
-                        pending_entries[i] = Some(entry);
-                    }
-                }
-            }
             let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
                 if errs[i].is_some() || pending_entries[i].is_some() {
                     return None;
@@ -803,12 +783,7 @@ async fn list_path_raw_inner(
                                     continue;
                                 }
                                 // eof
-                                if r.limit_reached() == Some(true) {
-                                    limit_reached = true;
-                                    scan_cut = Some(last_scanned_key.clone().ok_or_else(|| {
-                                        DiskError::other("bounded listing reader ended without a scan boundary")
-                                    })?);
-                                }
+                                limit_reached |= r.limit_reached() == Some(true);
                                 at_eof += 1;
                                 // warn!("list_path_raw: peek eof, disk: {}", i);
                                 continue;
@@ -882,7 +857,7 @@ async fn list_path_raw_inner(
                             *r = MetacacheReader::new(detached_rd);
                             continue;
                         }
-                        PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
+                        PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
                     }
                 };
 
@@ -903,7 +878,7 @@ async fn list_path_raw_inner(
                 }
                 if entry.name == current.name {
                     // If exact match, we agree.
-                    if current.matches_strict_borrowed(&entry) {
+                    if current.matches_strict(&entry) {
                         top_entries[i] = Some(entry);
                         agree += 1;
                     } else {
@@ -941,7 +916,7 @@ async fn list_path_raw_inner(
                 if let Some(finished_fn) = opts.finished.as_ref() {
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
                         _ = finished_fn(&errs) => {}
                     }
                 }
@@ -991,14 +966,6 @@ async fn list_path_raw_inner(
                 return Err(classify_listing_quorum_failure(&failures));
             }
 
-            // The next head from a bounded reader is unknown. The previous
-            // globally consumed group is a safe cut; consuming another reader's
-            // later head would turn this absence of evidence into a skipped key.
-            // Check quorum failures above before accepting the bounded result.
-            if limit_reached {
-                break;
-            }
-
             // Break if all at EOF or error.
             if at_eof + has_err == readers.len() {
                 if has_err > 0
@@ -1006,7 +973,7 @@ async fn list_path_raw_inner(
                 {
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
                         _ = finished_fn(&errs) => {}
                     }
                 }
@@ -1017,19 +984,16 @@ async fn list_path_raw_inner(
                 break;
             }
 
-            last_scanned_key = Some(current.name.clone());
             if agree == readers.len() && !opts.preserve_replica_metadata {
                 for r in readers.iter_mut() {
-                    if !r.skip_current() {
-                        let _ = r.skip(1).await;
-                    }
+                    let _ = r.skip(1).await;
                 }
 
                 if let Some(agreed_fn) = opts.agreed.as_ref() {
                     // warn!("list_path_raw: agreed_fn start, current: {:?}", &current.name);
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
                         result = agreed_fn(current) => result?
                     }
                     // warn!("list_path_raw: agreed_fn done");
@@ -1045,7 +1009,7 @@ async fn list_path_raw_inner(
             }
 
             for (i, r) in readers.iter_mut().enumerate() {
-                if top_entries[i].is_some() && !r.skip_current() {
+                if top_entries[i].is_some() {
                     let _ = r.skip(1).await;
                 }
             }
@@ -1053,12 +1017,12 @@ async fn list_path_raw_inner(
             if let Some(partial_fn) = opts.partial.as_ref() {
                 tokio::select! {
                     biased;
-                    _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
+                    _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
                     _ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
                 }
             }
         }
-        Ok(ListPathRawOutcome { limit_reached, scan_cut })
+        Ok(ListPathRawOutcome { limit_reached })
     });
 
     let merge_started = std::time::Instant::now();
@@ -1223,24 +1187,6 @@ mod tests {
         assert!(!options.skip_total_timeout);
     }
 
-    #[test]
-    fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
-        let request = ListPathRawOptions {
-            bucket: "versioned".to_owned(),
-            hidden_entry_budget: 100,
-            pending_prefix: Some("pending/".to_owned()),
-            forward_to: Some("pending/0099".to_owned()),
-            ..Default::default()
-        };
-        // Primary and fallback readers both construct their walk from the
-        // per-job clone, rather than directly from the request options.
-        let job = request.clone();
-        let walk = walk_dir_options(&job);
-        assert_eq!(walk.hidden_entry_budget, 100);
-        assert_eq!(walk.pending_prefix.as_deref(), request.pending_prefix.as_deref());
-        assert_eq!(walk.forward_to.as_deref(), Some("pending/0099"));
-    }
-
     #[tokio::test]
     async fn list_path_raw_empty_disks_returns_read_quorum() {
         let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
@@ -1358,7 +1304,6 @@ mod tests {
             outcome.limit_reached,
             "a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
         );
-        assert_eq!(outcome.scan_cut.as_deref(), Some("bucket/object"));
     }
 
     #[tokio::test]

crates/ecstore/src/cluster/rpc/background_pipe.rs

@@ -1,682 +0,0 @@
-// Copyright 2026 RustFS Team
-// Licensed under the Apache License, Version 2.0.
-
-use crate::disk::background_memory::BackgroundBuffer;
-use std::io::{self, IoSlice};
-use std::pin::Pin;
-use std::sync::{Arc, Mutex};
-use std::task::{Context, Poll, Waker, ready};
-use tokio::io::{AsyncRead, AsyncWrite, DuplexStream, ReadBuf};
-
-pub struct BackgroundPipeReader {
-    inner: PipeEnd,
-}
-
-pub struct BackgroundPipeWriter {
-    inner: PipeEnd,
-}
-
-impl BackgroundPipeWriter {
-    /// Capture only the payload owner, without retaining node or physical units.
-    pub fn memory_budget(&self) -> Option<Arc<crate::disk::BackgroundMemoryBudget>> {
-        match &self.inner {
-            PipeEnd::Foreground(_) => None,
-            PipeEnd::Background(state) => state
-                .lock()
-                .unwrap_or_else(std::sync::PoisonError::into_inner)
-                .buffer
-                .memory_budget(),
-        }
-    }
-}
-
-enum PipeEnd {
-    Foreground(DuplexStream),
-    Background(Arc<Mutex<PipeState>>),
-}
-
-// The pipe lock may reserve memory. The memory budget never locks a pipe.
-struct PipeState {
-    buffer: BackgroundBuffer,
-    read_offset: usize,
-    readable: usize,
-    capacity: usize,
-    reader_closed: bool,
-    writer_closed: bool,
-    admission_failure: Option<io::ErrorKind>,
-    read_waker: Option<Waker>,
-    write_waker: Option<Waker>,
-}
-
-/// Capture only memory ownership while the request is admitted. Payload storage
-/// is reserved and initialized once, before the first nonempty background write.
-pub fn background_pipe(capacity: usize) -> (BackgroundPipeReader, BackgroundPipeWriter) {
-    let capacity = capacity.max(1);
-    let memory = crate::disk::current_resource_unit().and_then(|resources| resources.memory_budget());
-    match memory {
-        Some(memory) => {
-            let state = Arc::new(Mutex::new(PipeState {
-                buffer: BackgroundBuffer::new(Some(memory)),
-                read_offset: 0,
-                readable: 0,
-                capacity,
-                reader_closed: false,
-                writer_closed: false,
-                admission_failure: None,
-                read_waker: None,
-                write_waker: None,
-            }));
-            (
-                BackgroundPipeReader {
-                    inner: PipeEnd::Background(state.clone()),
-                },
-                BackgroundPipeWriter {
-                    inner: PipeEnd::Background(state),
-                },
-            )
-        }
-        None => {
-            let (reader, writer) = tokio::io::duplex(capacity);
-            (
-                BackgroundPipeReader {
-                    inner: PipeEnd::Foreground(reader),
-                },
-                BackgroundPipeWriter {
-                    inner: PipeEnd::Foreground(writer),
-                },
-            )
-        }
-    }
-}
-
-fn advance_ring(offset: usize, count: usize, capacity: usize) -> usize {
-    let to_end = capacity - offset;
-    if count < to_end { offset + count } else { count - to_end }
-}
-
-fn poll_background_read(state: &Mutex<PipeState>, cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
-    if output.remaining() == 0 {
-        return Poll::Ready(Ok(()));
-    }
-    let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
-    if state.readable == 0 {
-        if let Some(kind) = state.admission_failure.take() {
-            drop(state);
-            return Poll::Ready(Err(io::Error::new(kind, "background pipe storage admission failed")));
-        }
-        if state.writer_closed {
-            return Poll::Ready(Ok(()));
-        }
-        let replaced = state.read_waker.replace(cx.waker().clone());
-        drop(state);
-        drop(replaced);
-        return Poll::Pending;
-    }
-    let length = state.readable.min(output.remaining());
-    let first = length.min(state.capacity - state.read_offset);
-    output.put_slice(&state.buffer.as_slice()[state.read_offset..state.read_offset + first]);
-    output.put_slice(&state.buffer.as_slice()[..length - first]);
-    state.read_offset = advance_ring(state.read_offset, length, state.capacity);
-    state.readable -= length;
-    let wake = state.write_waker.take();
-    drop(state);
-    if let Some(wake) = wake {
-        wake.wake();
-    }
-    Poll::Ready(Ok(()))
-}
-
-fn poll_background_write(state: &Mutex<PipeState>, cx: &mut Context<'_>, input: &[IoSlice<'_>]) -> Poll<io::Result<usize>> {
-    let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
-    if state.reader_closed || state.writer_closed {
-        return Poll::Ready(Err(io::ErrorKind::BrokenPipe.into()));
-    }
-    if input.iter().all(|part| part.is_empty()) {
-        return Poll::Ready(Ok(0));
-    }
-    let available = state.capacity - state.readable;
-    if available == 0 {
-        let replaced = state.write_waker.replace(cx.waker().clone());
-        drop(state);
-        drop(replaced);
-        return Poll::Pending;
-    }
-    let capacity = state.capacity;
-    if let Err(error) = state.buffer.ensure_len(capacity) {
-        // Legacy walk completion may suppress ordinary producer errors. A new
-        // pipe admission failure must also reach the reader instead of EOF.
-        state.admission_failure = Some(error.kind());
-        state.writer_closed = true;
-        let wake = state.read_waker.take();
-        drop(state);
-        if let Some(wake) = wake {
-            wake.wake();
-        }
-        return Poll::Ready(Err(error));
-    }
-    let mut offset = advance_ring(state.read_offset, state.readable, capacity);
-    let mut written = 0;
-    for part in input {
-        let length = part.len().min(available - written);
-        let first = length.min(capacity - offset);
-        state.buffer.as_mut_slice()[offset..offset + first].copy_from_slice(&part[..first]);
-        state.buffer.as_mut_slice()[..length - first].copy_from_slice(&part[first..length]);
-        offset = advance_ring(offset, length, capacity);
-        written += length;
-        if written == available {
-            break;
-        }
-    }
-    state.readable += written;
-    let wake = state.read_waker.take();
-    drop(state);
-    if let Some(wake) = wake {
-        wake.wake();
-    }
-    Poll::Ready(Ok(written))
-}
-
-impl AsyncRead for BackgroundPipeReader {
-    fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
-        match &mut self.get_mut().inner {
-            PipeEnd::Foreground(reader) => Pin::new(reader).poll_read(cx, output),
-            PipeEnd::Background(state) => {
-                let coop = ready!(tokio::task::coop::poll_proceed(cx));
-                let result = poll_background_read(state, cx, output);
-                if result.is_ready() {
-                    coop.made_progress();
-                }
-                result
-            }
-        }
-    }
-}
-
-impl AsyncWrite for BackgroundPipeWriter {
-    fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, input: &[u8]) -> Poll<io::Result<usize>> {
-        match &mut self.get_mut().inner {
-            PipeEnd::Foreground(writer) => Pin::new(writer).poll_write(cx, input),
-            PipeEnd::Background(state) => {
-                let coop = ready!(tokio::task::coop::poll_proceed(cx));
-                let result = poll_background_write(state, cx, &[IoSlice::new(input)]);
-                if result.is_ready() {
-                    coop.made_progress();
-                }
-                result
-            }
-        }
-    }
-
-    fn poll_write_vectored(self: Pin<&mut Self>, cx: &mut Context<'_>, input: &[IoSlice<'_>]) -> Poll<io::Result<usize>> {
-        match &mut self.get_mut().inner {
-            PipeEnd::Foreground(writer) => Pin::new(writer).poll_write_vectored(cx, input),
-            PipeEnd::Background(state) => {
-                let coop = ready!(tokio::task::coop::poll_proceed(cx));
-                let result = poll_background_write(state, cx, input);
-                if result.is_ready() {
-                    coop.made_progress();
-                }
-                result
-            }
-        }
-    }
-
-    fn is_write_vectored(&self) -> bool {
-        match &self.inner {
-            PipeEnd::Foreground(writer) => writer.is_write_vectored(),
-            PipeEnd::Background(_) => true,
-        }
-    }
-
-    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
-        match &mut self.get_mut().inner {
-            PipeEnd::Foreground(writer) => Pin::new(writer).poll_flush(cx),
-            PipeEnd::Background(_) => Poll::Ready(Ok(())),
-        }
-    }
-
-    fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
-        match &mut self.get_mut().inner {
-            PipeEnd::Foreground(writer) => Pin::new(writer).poll_shutdown(cx),
-            PipeEnd::Background(state) => {
-                let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
-                state.writer_closed = true;
-                let wake = state.read_waker.take();
-                drop(state);
-                if let Some(wake) = wake {
-                    wake.wake();
-                }
-                Poll::Ready(Ok(()))
-            }
-        }
-    }
-}
-
-impl Drop for BackgroundPipeReader {
-    fn drop(&mut self) {
-        if let PipeEnd::Background(state) = &self.inner {
-            let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
-            state.reader_closed = true;
-            let wake = state.write_waker.take();
-            let cancelled = state.read_waker.take();
-            drop(state);
-            drop(cancelled);
-            if let Some(wake) = wake {
-                wake.wake();
-            }
-        }
-    }
-}
-
-impl Drop for BackgroundPipeWriter {
-    fn drop(&mut self) {
-        if let PipeEnd::Background(state) = &self.inner {
-            let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
-            state.writer_closed = true;
-            let wake = state.read_waker.take();
-            let cancelled = state.write_waker.take();
-            drop(state);
-            drop(cancelled);
-            if let Some(wake) = wake {
-                wake.wake();
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::disk::BackgroundMemoryBudget;
-    use std::sync::atomic::{AtomicUsize, Ordering};
-    use std::time::Duration;
-    use tokio::io::{AsyncReadExt, AsyncWriteExt};
-
-    pub(super) fn budget(capacity: usize) -> Arc<BackgroundMemoryBudget> {
-        let budget = BackgroundMemoryBudget::new();
-        budget.set_sampled_limit(u64::try_from(capacity).expect("test capacity fits u64"), Duration::from_secs(30));
-        budget
-    }
-
-    pub(super) fn local_pipe(
-        memory: &Arc<BackgroundMemoryBudget>,
-        capacity: usize,
-    ) -> (BackgroundPipeReader, BackgroundPipeWriter) {
-        let state = Arc::new(Mutex::new(PipeState {
-            buffer: BackgroundBuffer::new(Some(memory.clone())),
-            read_offset: 0,
-            readable: 0,
-            capacity,
-            reader_closed: false,
-            writer_closed: false,
-            admission_failure: None,
-            read_waker: None,
-            write_waker: None,
-        }));
-        (
-            BackgroundPipeReader {
-                inner: PipeEnd::Background(state.clone()),
-            },
-            BackgroundPipeWriter {
-                inner: PipeEnd::Background(state),
-            },
-        )
-    }
-
-    pub(super) fn write_now(writer: &mut (impl AsyncWrite + Unpin), bytes: &[u8]) -> Poll<io::Result<usize>> {
-        Pin::new(writer).poll_write(&mut Context::from_waker(Waker::noop()), bytes)
-    }
-
-    pub(super) fn read_now(reader: &mut (impl AsyncRead + Unpin), bytes: &mut [u8]) -> Poll<io::Result<usize>> {
-        let mut output = ReadBuf::new(bytes);
-        match Pin::new(reader).poll_read(&mut Context::from_waker(Waker::noop()), &mut output) {
-            Poll::Ready(Ok(())) => Poll::Ready(Ok(output.filled().len())),
-            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
-            Poll::Pending => Poll::Pending,
-        }
-    }
-    #[test]
-    fn fixed_ring_allocates_lazily_once_and_keeps_last_owner_charged() {
-        for capacity in [1, 7, 64 * 1024, 1024 * 1024] {
-            let memory = budget(capacity);
-            let (mut reader, mut writer) = local_pipe(&memory, capacity);
-            assert_eq!(memory.snapshot().used, 0);
-            assert!(matches!(write_now(&mut writer, &[]), Poll::Ready(Ok(0))));
-            assert_eq!(memory.snapshot().used, 0);
-            assert!(matches!(write_now(&mut writer, b"x"), Poll::Ready(Ok(1))));
-            assert_eq!(memory.snapshot().used, capacity as u64);
-            memory.set_sampled_limit(0, Duration::ZERO);
-            let mut output = [0];
-            assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(1))));
-            assert_eq!(output, *b"x");
-            assert!(matches!(write_now(&mut writer, b"y"), Poll::Ready(Ok(1))));
-            drop(writer);
-            assert_eq!(memory.snapshot().used, capacity as u64, "reader still owns the real ring");
-            assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(1))));
-            assert_eq!(output, *b"y");
-            assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(0))));
-            assert_eq!(memory.snapshot().used, capacity as u64, "draining does not destroy allocation ownership");
-            drop(reader);
-            assert_eq!(memory.snapshot().used, 0);
-        }
-    }
-
-    #[test]
-    fn admission_error_reaches_writer_and_reader_once_without_payload() {
-        for sample in [None, Some((6, Duration::from_secs(30))), Some((7, Duration::ZERO))] {
-            let memory = BackgroundMemoryBudget::new();
-            if let Some((limit, validity)) = sample {
-                memory.set_sampled_limit(limit, validity);
-            }
-            let (mut reader, mut writer) = local_pipe(&memory, 7);
-            let error = match write_now(&mut writer, b"x") {
-                Poll::Ready(Err(e)) => e,
-                _ => panic!("reject before copying first payload"),
-            };
-            assert!(
-                crate::disk::background_memory::is_memory_pressure(&error),
-                "writer retains the original budget marker"
-            );
-            assert_eq!(memory.snapshot().used, 0);
-            let error = match read_now(&mut reader, &mut [0]) {
-                Poll::Ready(Err(e)) => e,
-                _ => panic!("reader must see a pipe admission failure"),
-            };
-            assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
-            assert!(matches!(read_now(&mut reader, &mut [0]), Poll::Ready(Ok(0))), "failure is delivered once");
-            assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
-            drop((reader, writer));
-            assert_eq!(memory.snapshot().used, 0);
-        }
-    }
-
-    #[test]
-    fn allocation_overflow_is_a_reader_failure_without_a_leaked_reservation() {
-        let memory = BackgroundMemoryBudget::new();
-        memory.set_sampled_limit(u64::MAX, Duration::from_secs(30));
-        let (mut reader, mut writer) = local_pipe(&memory, usize::MAX);
-        assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::OutOfMemory));
-        assert_eq!(memory.snapshot().used, 0);
-        assert!(matches!(read_now(&mut reader,&mut[0]),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::OutOfMemory));
-    }
-
-    #[test]
-    fn scalar_and_vectored_wrap_match_a_reference_queue() {
-        use std::collections::VecDeque;
-        for capacity in [1, 3, 7, 31] {
-            let memory = budget(capacity);
-            let (mut reader, mut writer) = local_pipe(&memory, capacity);
-            let mut expected = VecDeque::new();
-            let mut next = 0u8;
-            for turn in 0..300 {
-                let input: Vec<_> = (0..(turn % 11 + 1))
-                    .map(|_| {
-                        let byte = next;
-                        next = next.wrapping_add(1);
-                        byte
-                    })
-                    .collect();
-                let available = capacity - expected.len();
-                let written = if turn % 2 == 0 {
-                    write_now(&mut writer, &input)
-                } else {
-                    let split = input.len() / 2;
-                    Pin::new(&mut writer).poll_write_vectored(
-                        &mut Context::from_waker(Waker::noop()),
-                        &[
-                            io::IoSlice::new(&[]),
-                            io::IoSlice::new(&input[..split]),
-                            io::IoSlice::new(&input[split..]),
-                        ],
-                    )
-                };
-                if available == 0 {
-                    assert!(written.is_pending());
-                } else {
-                    let accepted = input.len().min(available);
-                    assert!(matches!(written,Poll::Ready(Ok(n)) if n==accepted));
-                    expected.extend(&input[..accepted]);
-                }
-                let mut output = vec![0; turn % 5 + 1];
-                let count = output.len().min(expected.len());
-                assert!(matches!(read_now(&mut reader,&mut output),Poll::Ready(Ok(n)) if n==count));
-                for byte in &output[..count] {
-                    assert_eq!(Some(*byte), expected.pop_front());
-                }
-                assert_eq!(memory.snapshot().used, capacity as u64);
-            }
-            drop(writer);
-            let mut output = vec![0; capacity];
-            let count = expected.len();
-            assert!(matches!(read_now(&mut reader,&mut output),Poll::Ready(Ok(n)) if n==count));
-            assert_eq!(&output[..count], expected.into_iter().collect::<Vec<_>>().as_slice());
-            assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(0))));
-            drop(reader);
-            assert_eq!(memory.snapshot().used, 0);
-        }
-    }
-
-    #[tokio::test]
-    async fn reader_drop_wakes_blocked_writer_and_writer_drop_wakes_empty_reader() {
-        let memory = budget(7);
-        let (reader, mut writer) = local_pipe(&memory, 7);
-        writer.write_all(b"1234567").await.expect("fill the ring");
-        let task = tokio::spawn(async move { writer.write_all(b"blocked").await });
-        tokio::task::yield_now().await;
-        assert!(!task.is_finished());
-        drop(reader);
-        let error = tokio::time::timeout(Duration::from_secs(1), task)
-            .await
-            .expect("reader drop wakes writer")
-            .expect("writer task joins")
-            .expect_err("closed reader is BrokenPipe");
-        assert_eq!(error.kind(), io::ErrorKind::BrokenPipe);
-        assert_eq!(memory.snapshot().used, 0);
-        let (mut reader, writer) = local_pipe(&memory, 7);
-        let task = tokio::spawn(async move { reader.read(&mut [0]).await });
-        tokio::task::yield_now().await;
-        assert!(!task.is_finished());
-        drop(writer);
-        assert_eq!(
-            tokio::time::timeout(Duration::from_secs(1), task)
-                .await
-                .expect("writer drop wakes reader")
-                .expect("reader task joins")
-                .expect("clean EOF"),
-            0
-        );
-    }
-
-    #[tokio::test]
-    async fn read_wakes_a_full_writer_without_changing_byte_limit() {
-        let memory = budget(7);
-        let (mut reader, mut writer) = local_pipe(&memory, 7);
-        writer.write_all(b"1234567").await.expect("fill initial byte limit");
-        let task = tokio::spawn(async move {
-            writer.write_all(b"x").await.expect("resume after one byte is read");
-            writer
-        });
-        tokio::task::yield_now().await;
-        assert!(!task.is_finished());
-        let mut first = [0];
-        reader.read_exact(&mut first).await.expect("read one byte");
-        let writer = tokio::time::timeout(Duration::from_secs(1), task)
-            .await
-            .expect("read wakes writer")
-            .expect("writer joins");
-        assert_eq!(&first, b"1");
-        drop(writer);
-        let mut all = Vec::new();
-        reader.read_to_end(&mut all).await.expect("drain wrapped bytes");
-        assert_eq!(all, b"234567x");
-        drop(reader);
-        assert_eq!(memory.snapshot().used, 0);
-    }
-
-    #[tokio::test]
-    async fn shutdown_drains_then_eof_and_cancelled_writes_accept_nothing() {
-        let memory = budget(7);
-        let (mut reader, mut writer) = local_pipe(&memory, 7);
-        writer
-            .write_all(b"1234567")
-            .await
-            .expect("fill before cancelling a blocked write");
-        {
-            let blocked = writer.write(b"lost");
-            tokio::pin!(blocked);
-            assert!(futures::poll!(&mut blocked).is_pending());
-        }
-        writer.shutdown().await.expect("shutdown does not discard buffered bytes");
-        assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
-        let mut out = Vec::new();
-        reader.read_to_end(&mut out).await.expect("drain after shutdown");
-        assert_eq!(out, b"1234567");
-        drop(reader);
-        assert_eq!(memory.snapshot().used, 7, "writer still holds the allocation");
-        drop(writer);
-        assert_eq!(memory.snapshot().used, 0);
-    }
-
-    #[test]
-    fn foreground_keeps_native_duplex_semantics() {
-        let (reader, mut writer) = background_pipe(7);
-        assert!(matches!(write_now(&mut writer, b"12345678"), Poll::Ready(Ok(7))));
-        assert!(write_now(&mut writer, b"x").is_pending());
-        drop(reader);
-        assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
-        let (mut reader, writer) = background_pipe(7);
-        drop(writer);
-        assert!(matches!(read_now(&mut reader, &mut [0]), Poll::Ready(Ok(0))));
-    }
-
-    #[test]
-    fn cancelled_reader_leaves_the_real_ring_charged_until_writer_drop() {
-        let memory = budget(7);
-        let (reader, mut writer) = local_pipe(&memory, 7);
-        assert!(matches!(write_now(&mut writer, b"x"), Poll::Ready(Ok(1))));
-        drop(reader);
-        assert_eq!(memory.snapshot().used, 7, "the surviving writer still owns the ring allocation");
-        assert!(matches!(write_now(&mut writer,b"y"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
-        drop(writer);
-        assert_eq!(memory.snapshot().used, 0);
-    }
-
-    #[test]
-    fn runtime_shutdown_releases_queued_writer_but_not_the_surviving_reader() {
-        let runtime = tokio::runtime::Builder::new_current_thread()
-            .enable_all()
-            .build()
-            .expect("build isolated runtime");
-        let memory = budget(7);
-        let (reader, mut writer) = local_pipe(&memory, 7);
-        runtime.block_on(async {
-            writer.write_all(b"x").await.expect("allocate ring before runtime shutdown");
-            tokio::spawn(async move {
-                let _writer = writer;
-                std::future::pending::<()>().await;
-            });
-            tokio::task::yield_now().await;
-        });
-        drop(runtime);
-        assert_eq!(memory.snapshot().used, 7, "reader holds the actual allocation after runtime cancellation");
-        drop(reader);
-        assert_eq!(memory.snapshot().used, 0);
-    }
-
-    #[tokio::test]
-    async fn ready_operations_keep_tokio_cooperative_yields() {
-        let memory = budget(7);
-        let (_reader, mut writer) = local_pipe(&memory, 7);
-        let observed = Arc::new(AtomicUsize::new(0));
-        let flag = observed.clone();
-        let task = tokio::spawn(async move {
-            flag.store(1, Ordering::SeqCst);
-        });
-        for _ in 0..1000 {
-            assert_eq!(writer.write(&[]).await.expect("empty operation remains successful"), 0);
-        }
-        assert_eq!(
-            observed.load(Ordering::SeqCst),
-            1,
-            "ready pipe operations must not starve other ready tasks"
-        );
-        task.await.expect("observer task joins");
-    }
-}
-
-#[cfg(test)]
-mod waker_tests {
-    use super::*;
-    use std::sync::{
-        Weak,
-        atomic::{AtomicUsize, Ordering},
-    };
-    use std::task::Wake;
-
-    struct LockCheck {
-        state: Weak<Mutex<PipeState>>,
-        wakes: Arc<AtomicUsize>,
-        drops: Arc<AtomicUsize>,
-    }
-    impl LockCheck {
-        fn unlocked(&self) {
-            if let Some(state) = self.state.upgrade() {
-                assert!(state.try_lock().is_ok(), "waker code must run outside the pipe lock");
-            }
-        }
-    }
-    impl Wake for LockCheck {
-        fn wake(self: Arc<Self>) {
-            self.unlocked();
-            self.wakes.fetch_add(1, Ordering::SeqCst);
-        }
-    }
-    impl Drop for LockCheck {
-        fn drop(&mut self) {
-            self.unlocked();
-            self.drops.fetch_add(1, Ordering::SeqCst);
-        }
-    }
-
-    #[test]
-    fn replaced_and_woken_reader_and_writer_wakers_run_outside_the_lock() {
-        for waiting_reader in [true, false] {
-            let memory = super::tests::budget(7);
-            let (mut reader, mut writer) = super::tests::local_pipe(&memory, 7);
-            let PipeEnd::Background(state) = &reader.inner else { panic!("background fixture") };
-            let weak = Arc::downgrade(state);
-            let wakes = Arc::new(AtomicUsize::new(0));
-            let drops = Arc::new(AtomicUsize::new(0));
-            if !waiting_reader {
-                assert!(matches!(super::tests::write_now(&mut writer, b"1234567"), Poll::Ready(Ok(7))));
-            }
-            for _ in 0..2 {
-                let waker = Waker::from(Arc::new(LockCheck {
-                    state: weak.clone(),
-                    wakes: wakes.clone(),
-                    drops: drops.clone(),
-                }));
-                let mut cx = Context::from_waker(&waker);
-                if waiting_reader {
-                    let mut bytes = [0];
-                    assert!(
-                        Pin::new(&mut reader)
-                            .poll_read(&mut cx, &mut ReadBuf::new(&mut bytes))
-                            .is_pending()
-                    );
-                } else {
-                    assert!(Pin::new(&mut writer).poll_write(&mut cx, b"x").is_pending());
-                }
-                drop(waker);
-            }
-            assert_eq!(drops.load(Ordering::SeqCst), 1, "the replaced waker has been dropped");
-            if waiting_reader {
-                assert!(matches!(super::tests::write_now(&mut writer, b"x"), Poll::Ready(Ok(1))));
-            } else {
-                assert!(matches!(super::tests::read_now(&mut reader, &mut [0]), Poll::Ready(Ok(1))));
-            }
-            assert_eq!(wakes.load(Ordering::SeqCst), 1);
-            assert_eq!(drops.load(Ordering::SeqCst), 2);
-        }
-    }
-}

crates/ecstore/src/cluster/rpc/background_reader_stream.rs

@@ -1,586 +0,0 @@
-// Copyright 2026 RustFS Team
-// Licensed under the Apache License, Version 2.0.
-
-use crate::disk::background_memory::{BackgroundMemoryBudget, Reservation};
-use bytes::{Bytes, BytesMut};
-use futures::Stream;
-use std::io;
-use std::pin::Pin;
-use std::sync::Arc;
-use std::task::{Context, Poll};
-use tokio::io::{AsyncRead, ReadBuf};
-use tokio_util::io::ReaderStream;
-
-struct ReservedReadBytes {
-    // Free each payload view before releasing its shared allocation receipt.
-    bytes: Bytes,
-    _reservation: Arc<Reservation>,
-}
-
-impl AsRef<[u8]> for ReservedReadBytes {
-    fn as_ref(&self) -> &[u8] {
-        &self.bytes
-    }
-}
-
-struct ReaderBuffer {
-    // These initialized bytes are spare storage, not unconsumed source data.
-    bytes: BytesMut,
-    reservation: Option<Arc<Reservation>>,
-    budget: Arc<BackgroundMemoryBudget>,
-    capacity: usize,
-}
-
-impl ReaderBuffer {
-    fn new(budget: Arc<BackgroundMemoryBudget>, capacity: usize) -> Self {
-        Self {
-            bytes: BytesMut::new(),
-            reservation: None,
-            budget,
-            capacity: capacity.max(1),
-        }
-    }
-
-    fn clear(&mut self) {
-        self.bytes = BytesMut::new();
-        self.reservation = None;
-    }
-
-    fn ensure_readable(&mut self) -> io::Result<()> {
-        if !self.bytes.is_empty() {
-            return Ok(());
-        }
-        if self.reservation.is_some() {
-            if self.bytes.try_reclaim(self.capacity) {
-                self.bytes.resize(self.capacity, 0);
-                return Ok(());
-            }
-            // A live output owns this allocation. Drop our empty view before
-            // its receipt, then reserve independent replacement storage.
-            self.clear();
-        }
-        let reservation = self.budget.reserve(self.capacity)?;
-        let mut bytes = Vec::new();
-        bytes
-            .try_reserve_exact(self.capacity)
-            .map_err(|_| io::ErrorKind::OutOfMemory)?;
-        bytes.resize(self.capacity, 0);
-        // Fresh ordinary Bytes is unique and spans the complete allocation.
-        // Unlike from_owner output, this conversion cannot require a copy.
-        let bytes = Bytes::from(bytes.into_boxed_slice())
-            .try_into_mut()
-            .map_err(|_| io::Error::other("fresh reader storage was not unique"))?;
-        self.bytes = bytes;
-        self.reservation = Some(Arc::new(reservation));
-        Ok(())
-    }
-
-    fn take_bytes(&mut self, length: usize) -> io::Result<Bytes> {
-        let reservation = self
-            .reservation
-            .as_ref()
-            .cloned()
-            .ok_or_else(|| io::Error::other("reader storage has no memory reservation"))?;
-        Ok(Bytes::from_owner(ReservedReadBytes {
-            bytes: self.bytes.split_to(length).freeze(),
-            _reservation: reservation,
-        }))
-    }
-}
-
-/// RPC response chunks retain their background memory reservation until the
-/// last clone or slice is dropped. Foreground readers use the existing stream.
-pub struct BackgroundReaderStream<R> {
-    inner: ReaderState<R>,
-}
-
-enum ReaderState<R> {
-    Foreground(ReaderStream<R>),
-    Background { reader: Option<R>, buffer: ReaderBuffer },
-}
-
-impl<R: AsyncRead> BackgroundReaderStream<R> {
-    /// Capture memory ownership inside the admitted scope, before moving the
-    /// response body out of it. This stream retains no node or device capacity.
-    pub fn with_capacity(reader: R, capacity: usize) -> Self {
-        let memory = crate::disk::current_resource_unit().and_then(|resources| resources.memory_budget());
-        let inner = match memory {
-            Some(memory) => ReaderState::Background {
-                reader: Some(reader),
-                buffer: ReaderBuffer::new(memory, capacity),
-            },
-            None => ReaderState::Foreground(ReaderStream::with_capacity(reader, capacity)),
-        };
-        Self { inner }
-    }
-}
-
-impl<R: AsyncRead + Unpin> Stream for BackgroundReaderStream<R> {
-    type Item = io::Result<Bytes>;
-
-    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        let this = self.get_mut();
-        match &mut this.inner {
-            ReaderState::Foreground(reader) => Pin::new(reader).poll_next(cx),
-            ReaderState::Background { reader, buffer } => {
-                let Some(inner) = reader.as_mut() else {
-                    return Poll::Ready(None);
-                };
-                if let Err(error) = buffer.ensure_readable() {
-                    *reader = None;
-                    return Poll::Ready(Some(Err(error)));
-                }
-                let mut output = ReadBuf::new(&mut buffer.bytes);
-                match std::task::ready!(Pin::new(inner).poll_read(cx, &mut output)) {
-                    Ok(()) => {
-                        let length = output.filled().len();
-                        if length == 0 {
-                            buffer.clear();
-                            *reader = None;
-                            Poll::Ready(None)
-                        } else {
-                            let bytes = buffer.take_bytes(length);
-                            if bytes.is_err() {
-                                buffer.clear();
-                                *reader = None;
-                            }
-                            Poll::Ready(Some(bytes))
-                        }
-                    }
-                    Err(error) => {
-                        buffer.clear();
-                        *reader = None;
-                        Poll::Ready(Some(Err(error)))
-                    }
-                }
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
-    use futures::StreamExt;
-    use std::sync::Arc;
-    use std::sync::atomic::{AtomicUsize, Ordering};
-    use std::time::Duration;
-    use tokio::io::AsyncWriteExt;
-
-    struct ObservedReader {
-        polls: Arc<AtomicUsize>,
-        fail: bool,
-    }
-
-    impl AsyncRead for ObservedReader {
-        fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
-            self.polls.fetch_add(1, Ordering::SeqCst);
-            if self.fail {
-                Poll::Ready(Err(io::Error::from(io::ErrorKind::PermissionDenied)))
-            } else {
-                output.put_slice(b"data");
-                Poll::Ready(Ok(()))
-            }
-        }
-    }
-
-    #[tokio::test]
-    async fn background_frames_keep_capacity_until_the_last_slice() {
-        let memory = BackgroundMemoryBudget::new();
-        memory.set_sampled_limit(16, Duration::from_secs(30));
-        let (resources, gates) = test_resource_units(2);
-        let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
-            BackgroundReaderStream::with_capacity(io::Cursor::new(b"data"), 16)
-        })
-        .await;
-        assert!(
-            gates.iter().all(|gate| gate.snapshot().active == 0),
-            "the stream captures only memory ownership"
-        );
-        assert_eq!(memory.snapshot().used, 0, "construction does not allocate payload storage");
-        let bytes = stream.next().await.expect("first frame").expect("read frame");
-        assert_eq!(bytes.as_ref(), b"data");
-        assert_eq!(memory.snapshot().used, 16, "charge full capacity rather than visible length");
-        let slice = bytes.slice(1..3);
-        memory.set_sampled_limit(0, Duration::ZERO);
-        drop(bytes);
-        drop(stream);
-        assert_eq!(slice.as_ref(), b"at");
-        assert_eq!(memory.snapshot().used, 16);
-        drop(slice);
-        assert_eq!(memory.snapshot().used, 0);
-    }
-
-    #[tokio::test]
-    async fn background_denial_happens_before_reading_and_never_falls_back() {
-        for sample in [None, Some((7, Duration::from_secs(30))), Some((8, Duration::ZERO))] {
-            let memory = BackgroundMemoryBudget::new();
-            if let Some((limit, validity)) = sample {
-                memory.set_sampled_limit(limit, validity);
-            }
-            let polls = Arc::new(AtomicUsize::new(0));
-            let (resources, _) = test_resource_units(1);
-            let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
-                BackgroundReaderStream::with_capacity(
-                    ObservedReader {
-                        polls: polls.clone(),
-                        fail: false,
-                    },
-                    8,
-                )
-            })
-            .await;
-            let error = stream
-                .next()
-                .await
-                .expect("allocation failure")
-                .expect_err("background allocation denied");
-            assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
-            assert_eq!(polls.load(Ordering::SeqCst), 0, "denial cannot consume source bytes");
-            assert!(stream.next().await.is_none());
-            assert_eq!(memory.snapshot().used, 0);
-        }
-    }
-
-    #[tokio::test]
-    async fn background_pending_read_keeps_storage_through_shrink_or_cancellation() {
-        for cancelled in [false, true] {
-            let memory = BackgroundMemoryBudget::new();
-            memory.set_sampled_limit(16, Duration::from_secs(30));
-            let (resources, _) = test_resource_units(1);
-            let (mut sender, reader) = tokio::io::duplex(4);
-            let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
-                BackgroundReaderStream::with_capacity(reader, 16)
-            })
-            .await;
-            assert!(futures::poll!(stream.next()).is_pending());
-            memory.set_sampled_limit(0, Duration::ZERO);
-            assert_eq!(memory.snapshot().used, 16);
-            let bytes = if cancelled {
-                None
-            } else {
-                sender.write_all(b"data").await.expect("release pending read");
-                Some(
-                    stream
-                        .next()
-                        .await
-                        .expect("pending frame")
-                        .expect("existing allocation survives shrink"),
-                )
-            };
-            drop(stream);
-            if let Some(bytes) = bytes {
-                assert_eq!(bytes.as_ref(), b"data");
-                assert_eq!(memory.snapshot().used, 16);
-                drop(bytes);
-            }
-            assert_eq!(memory.snapshot().used, 0);
-        }
-    }
-
-    #[tokio::test]
-    async fn background_read_error_and_eof_release_storage() {
-        for fail in [false, true] {
-            let memory = BackgroundMemoryBudget::new();
-            memory.set_sampled_limit(8, Duration::from_secs(30));
-            let (resources, _) = test_resource_units(1);
-            let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
-                let reader: Box<dyn AsyncRead + Unpin> = if fail {
-                    Box::new(ObservedReader {
-                        polls: Arc::new(AtomicUsize::new(0)),
-                        fail: true,
-                    })
-                } else {
-                    Box::new(tokio::io::empty())
-                };
-                BackgroundReaderStream::with_capacity(reader, 8)
-            })
-            .await;
-            if fail {
-                let error = stream.next().await.expect("read failure").expect_err("preserve reader error");
-                assert_eq!(error.kind(), io::ErrorKind::PermissionDenied);
-            }
-            assert!(stream.next().await.is_none());
-            assert_eq!(memory.snapshot().used, 0);
-        }
-    }
-
-    #[tokio::test]
-    async fn retained_frame_blocks_new_storage_until_released() {
-        let memory = BackgroundMemoryBudget::new();
-        memory.set_sampled_limit(8, Duration::from_secs(30));
-        let (resources, _) = test_resource_units(1);
-        let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
-            BackgroundReaderStream::with_capacity(io::Cursor::new(b"firstonefollowup"), 8)
-        })
-        .await;
-        let first = stream.next().await.expect("first frame").expect("fits exact budget");
-        assert_eq!(first.as_ref(), b"firstone");
-        let error = stream
-            .next()
-            .await
-            .expect("next allocation")
-            .expect_err("held frame exhausts budget");
-        assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
-        assert_eq!(memory.snapshot().used, 8);
-        drop(stream);
-        drop(first);
-        assert_eq!(memory.snapshot().used, 0);
-    }
-
-    #[tokio::test]
-    async fn consumed_frames_keep_reusable_capacity_until_eof() {
-        let memory = BackgroundMemoryBudget::new();
-        memory.set_sampled_limit(8, Duration::from_secs(30));
-        let (resources, _) = test_resource_units(1);
-        let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
-            BackgroundReaderStream::with_capacity(io::Cursor::new(b"firstonefollowup"), 8)
-        })
-        .await;
-        for expected in [b"firstone", b"followup"] {
-            let bytes = stream.next().await.expect("next frame").expect("released previous capacity");
-            assert_eq!(bytes.as_ref(), expected);
-            assert_eq!(memory.snapshot().used, 8);
-            drop(bytes);
-            assert_eq!(memory.snapshot().used, 8, "the reusable allocation remains owned by the stream");
-        }
-        assert!(stream.next().await.is_none());
-        assert_eq!(memory.snapshot().used, 0);
-    }
-
-    #[tokio::test]
-    async fn foreground_stream_and_unpolled_cancellation_preserve_existing_behavior() {
-        let mut stream = BackgroundReaderStream::with_capacity(io::Cursor::new(b"foreground"), 4);
-        assert!(matches!(stream.inner, ReaderState::Foreground(_)));
-        let mut bytes = Vec::new();
-        while let Some(chunk) = stream.next().await {
-            bytes.extend_from_slice(&chunk.expect("foreground chunk"));
-        }
-        assert_eq!(bytes, b"foreground");
-
-        let memory = BackgroundMemoryBudget::new();
-        let (resources, _) = test_resource_units(1);
-        let stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
-            BackgroundReaderStream::with_capacity(tokio::io::empty(), 8)
-        })
-        .await;
-        drop(stream);
-        assert_eq!(memory.snapshot().used, 0);
-    }
-}
-
-#[cfg(test)]
-mod reuse_tests {
-    use super::*;
-    use bytes::Bytes;
-    use futures::StreamExt;
-    use std::sync::Arc;
-    use std::time::Duration;
-    use std::{
-        io,
-        pin::Pin,
-        task::{Context, Poll},
-    };
-    use tokio::io::{AsyncRead, ReadBuf};
-
-    static DATA: [u8; 1024] = [0x37; 1024];
-    struct ShortReader {
-        offset: usize,
-        length: usize,
-        chunk: usize,
-        fail_at: Option<usize>,
-    }
-    impl AsyncRead for ShortReader {
-        fn poll_read(mut self: Pin<&mut Self>, _: &mut Context<'_>, out: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
-            if self.fail_at == Some(self.offset) {
-                return Poll::Ready(Err(io::ErrorKind::PermissionDenied.into()));
-            }
-            let n = (self.length - self.offset).min(self.chunk).min(out.remaining());
-            out.put_slice(&DATA[self.offset..self.offset + n]);
-            self.offset += n;
-            Poll::Ready(Ok(()))
-        }
-    }
-    fn budget(limit: u64) -> Arc<crate::disk::BackgroundMemoryBudget> {
-        let b = crate::disk::BackgroundMemoryBudget::new();
-        b.set_sampled_limit(limit, Duration::from_secs(30));
-        b
-    }
-    async fn stream<R: AsyncRead>(
-        budget: &Arc<crate::disk::BackgroundMemoryBudget>,
-        reader: R,
-        capacity: usize,
-    ) -> BackgroundReaderStream<R> {
-        let (resources, _) = crate::disk::test_resource_units(1);
-        crate::disk::with_optional_resource_unit(Some(resources.with_memory_budget(budget.clone())), async {
-            BackgroundReaderStream::with_capacity(reader, capacity)
-        })
-        .await
-    }
-    #[tokio::test]
-    async fn complete_consumed_frames_reclaim_the_same_storage_after_expiry() {
-        let b = budget(64);
-        let mut s = stream(&b, io::Cursor::new(&DATA[..]), 64).await;
-        let mut pointer = None;
-        for _ in 0..16 {
-            let frame = s.next().await.expect("next full frame").expect("reclaimed backing");
-            assert_eq!(frame.len(), 64);
-            assert_eq!(frame.as_ref(), &DATA[..64]);
-            if let Some(p) = pointer {
-                assert_eq!(frame.as_ptr(), p);
-            } else {
-                pointer = Some(frame.as_ptr());
-            }
-            b.set_sampled_limit(0, Duration::ZERO);
-            drop(frame);
-            assert_eq!(b.snapshot().used, 64);
-        }
-        assert!(s.next().await.is_none());
-        assert_eq!(b.snapshot().used, 0);
-    }
-    #[tokio::test]
-    async fn short_live_frames_use_disjoint_tail_then_reserve_separate_storage() {
-        let b = budget(32);
-        let mut s = stream(
-            &b,
-            ShortReader {
-                offset: 0,
-                length: 32,
-                chunk: 4,
-                fail_at: None,
-            },
-            16,
-        )
-        .await;
-        let mut frames: Vec<Bytes> = Vec::new();
-        for i in 0..4 {
-            let frame = s.next().await.expect("short frame").expect("remaining initialized tail");
-            if i != 0 {
-                assert_eq!(frame.as_ptr(), frames[0].as_ptr().wrapping_add(i * 4));
-            }
-            assert_eq!(frame.as_ref(), &DATA[..4]);
-            frames.push(frame);
-            assert_eq!(b.snapshot().used, 16);
-        }
-        let next = s.next().await.expect("independent frame").expect("second backing fits");
-        assert_ne!(next.as_ptr(), frames[0].as_ptr());
-        assert_eq!(b.snapshot().used, 32);
-        drop(frames);
-        assert_eq!(b.snapshot().used, 16);
-        drop(s);
-        assert_eq!(b.snapshot().used, 16);
-        drop(next);
-        assert_eq!(b.snapshot().used, 0);
-    }
-    #[tokio::test]
-    async fn final_slice_prevents_reclaim_and_keeps_old_plus_new_charged() {
-        let b = budget(32);
-        let mut s = stream(&b, io::Cursor::new(&DATA[..32]), 16).await;
-        let first = s.next().await.expect("old frame").expect("old backing");
-        let last = first.slice(15..);
-        drop(first);
-        let next = s.next().await.expect("new frame").expect("second backing fits");
-        assert_eq!(last.as_ref(), &DATA[..1]);
-        assert_ne!(next.as_ptr(), last.as_ptr());
-        assert_eq!(b.snapshot().used, 32);
-        drop(s);
-        assert_eq!(b.snapshot().used, 32);
-        drop(next);
-        assert_eq!(b.snapshot().used, 16);
-        drop(last);
-        assert_eq!(b.snapshot().used, 0);
-    }
-    #[tokio::test]
-    async fn expiry_uses_tail_but_rejects_new_storage_before_consuming_source() {
-        let b = budget(16);
-        let mut s = stream(
-            &b,
-            ShortReader {
-                offset: 0,
-                length: 32,
-                chunk: 4,
-                fail_at: None,
-            },
-            16,
-        )
-        .await;
-        let first = s.next().await.expect("first frame").expect("initial backing");
-        b.set_sampled_limit(0, Duration::ZERO);
-        for _ in 0..3 {
-            drop(s.next().await.expect("tail frame").expect("reuse charged storage"));
-        }
-        let error = s
-            .next()
-            .await
-            .expect("new backing denied")
-            .expect_err("live frame prevents reclaim");
-        assert!(crate::disk::background_memory::is_memory_pressure(&error));
-        assert!(s.next().await.is_none());
-        assert_eq!(b.snapshot().used, 16);
-        drop(first);
-        assert_eq!(b.snapshot().used, 0);
-    }
-    #[tokio::test]
-    async fn eof_and_error_release_spare_but_keep_live_output_charged() {
-        for fail_at in [None, Some(4)] {
-            let b = budget(16);
-            let mut s = stream(
-                &b,
-                ShortReader {
-                    offset: 0,
-                    length: 4,
-                    chunk: 4,
-                    fail_at,
-                },
-                16,
-            )
-            .await;
-            let frame = s.next().await.expect("first short frame").expect("payload");
-            match fail_at {
-                None => assert!(s.next().await.is_none()),
-                Some(_) => assert_eq!(
-                    s.next()
-                        .await
-                        .expect("read error")
-                        .expect_err("preserve source failure")
-                        .kind(),
-                    io::ErrorKind::PermissionDenied
-                ),
-            }
-            assert!(s.next().await.is_none());
-            assert_eq!(b.snapshot().used, 16);
-            assert_eq!(frame.as_ref(), &DATA[..4]);
-            drop(frame);
-            assert_eq!(b.snapshot().used, 0);
-        }
-    }
-    #[test]
-    fn runtime_shutdown_drops_a_pending_stream_without_refunding_early() {
-        for abort in [false, true] {
-            let b = budget(16);
-            let rt = tokio::runtime::Builder::new_current_thread()
-                .enable_all()
-                .build()
-                .expect("runtime");
-            let (_tx, reader) = tokio::io::duplex(4);
-            let (began, wait) = tokio::sync::oneshot::channel();
-            let memory = b.clone();
-            let handle = rt.spawn(async move {
-                let mut s = stream(&memory, reader, 16).await;
-                assert!(futures::poll!(s.next()).is_pending());
-                assert_eq!(memory.snapshot().used, 16);
-                began.send(()).expect("notify pending allocation");
-                let _ = s.next().await;
-            });
-            rt.block_on(wait).expect("stream pending");
-            assert_eq!(b.snapshot().used, 16);
-            if abort {
-                handle.abort();
-                let _ = rt.block_on(handle);
-            }
-            drop(rt);
-            assert_eq!(b.snapshot().used, 0);
-        }
-    }
-}

crates/ecstore/src/cluster/rpc/client.rs

@@ -17,10 +17,10 @@ use crate::cluster::rpc::http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER;
 use crate::cluster::rpc::http_auth::{
     AuthenticatedPeerReplayCapabilities, RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER,
     RPC_CONTENT_SHA256_HEADER, RPC_REPLAY_CACHE_CAPABILITY_HEADER, RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER,
-    RollingMutationBodyDigest, TIMESTAMP_HEADER, gen_tonic_replay_scope_headers_with_admission, internode_rpc_body_digest_strict,
-    rpc_background_admission, verify_tonic_peer_replay_capabilities_response,
+    RollingMutationBodyDigest, TIMESTAMP_HEADER, internode_rpc_body_digest_strict,
+    verify_tonic_peer_replay_capabilities_response,
 };
-use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
+use crate::cluster::rpc::{gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience};
 #[cfg(test)]
 use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers};
 use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
@@ -415,16 +415,7 @@ where
                     .get(RPC_CONTENT_SHA256_HEADER)
                     .and_then(|value| value.to_str().ok()),
             ) {
-                match rpc_background_admission(request.headers()).and_then(|background| {
-                    gen_tonic_replay_scope_headers_with_admission(
-                        audience,
-                        request.uri().path(),
-                        timestamp,
-                        content_sha256,
-                        boot_epoch,
-                        background,
-                    )
-                }) {
+                match gen_tonic_replay_scope_headers(audience, request.uri().path(), timestamp, content_sha256, boot_epoch) {
                     Ok(headers) => request.headers_mut().extend(headers),
                     Err(error) => debug!(error = %error, "could not attach replay-scoped RPC signature"),
                 }

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -32,6 +32,7 @@ use crate::storage_api_contracts::internode::{
     PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
 };
 
+use hmac::{Hmac, KeyInit, Mac};
 use http::uri::Authority;
 use http::{HeaderMap, HeaderValue, Method, Uri};
 #[cfg(test)]
@@ -46,7 +47,6 @@ use rustfs_io_metrics::internode_metrics::{
     global_internode_metrics,
 };
 use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
-use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
 use rustfs_utils::get_env_bool;
 use sha2::Digest as _;
 use sha2::Sha256;
@@ -58,10 +58,9 @@ use time::OffsetDateTime;
 use tracing::{error, info, warn};
 use uuid::Uuid;
 
+type HmacSha256 = Hmac<Sha256>;
+
 const SIGNATURE_HEADER: &str = "x-rustfs-signature";
-pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
-const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
-const RPC_BACKGROUND_ADMISSION_DOMAIN: &[u8] = b"rustfs-rpc-background-admission-v1\0";
 pub(crate) const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
 pub(crate) const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
 const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
@@ -78,46 +77,6 @@ pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_HEADER: &str = "x-rustfs-rpc-replay
 pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER: &str = "x-rustfs-rpc-replay-cache-capability-proof";
 const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
 const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
-
-tokio::task_local! {
-    static RPC_BACKGROUND_REQUEST: bool;
-}
-
-pub(crate) fn current_rpc_background_request() -> bool {
-    RPC_BACKGROUND_REQUEST
-        .try_with(|background| *background)
-        .unwrap_or_else(|_| crate::disk::has_resource_io_scope())
-}
-
-/// Preserve the authenticated wire class while polling locally admitted work.
-/// This scope carries only the class, never node, disk, or memory ownership.
-pub async fn with_rpc_background_request<F: std::future::Future>(background: bool, operation: F) -> F::Output {
-    RPC_BACKGROUND_REQUEST.scope(background, operation).await
-}
-
-/// Parse the request class only; callers must verify the signature before using it.
-pub fn rpc_background_admission(headers: &HeaderMap) -> std::io::Result<bool> {
-    let mut values = headers.get_all(RPC_BACKGROUND_ADMISSION_HEADER).iter();
-    match (values.next(), values.next()) {
-        (None, None) => Ok(false),
-        (Some(value), None) if value == RPC_BACKGROUND_ADMISSION_V1 => Ok(true),
-        _ => Err(std::io::Error::other("Invalid RPC background admission version")),
-    }
-}
-
-fn bind_background_admission(mac: &mut HmacSha256, background: bool) {
-    if background {
-        // Every auth lane binds the class, so stripping the marker or falling
-        // back to an older signature cannot turn admitted work into foreground I/O.
-        mac.update(RPC_BACKGROUND_ADMISSION_DOMAIN);
-    }
-}
-
-fn insert_background_admission(headers: &mut HeaderMap, background: bool) {
-    if background {
-        headers.insert(RPC_BACKGROUND_ADMISSION_HEADER, HeaderValue::from_static(RPC_BACKGROUND_ADMISSION_V1));
-    }
-}
 const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
 const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
 const RPC_REPLAY_CACHE_CAPABILITY_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-replay-cache-capability-proof-v1\0";
@@ -560,23 +519,21 @@ fn redacted_rpc_path(url: &str) -> String {
 }
 
 /// Generate HMAC-SHA256 signature for the given data
-fn generate_signature(secret: &str, url: &str, method: &Method, timestamp: i64, background: bool) -> String {
+fn generate_signature(secret: &str, url: &str, method: &Method, timestamp: i64) -> String {
     let data = signature_payload(url, method, timestamp);
     let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
-    bind_background_admission(&mut mac, background);
     mac.update(data.as_bytes());
     let result = mac.finalize();
     base64_simd::STANDARD.encode_to_string(result.into_bytes())
 }
 
-fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, signature: &str, background: bool) -> bool {
+fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, signature: &str) -> bool {
     let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else {
         return false;
     };
 
     let data = signature_payload(url, method, timestamp);
     let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
-    bind_background_admission(&mut mac, background);
     mac.update(data.as_bytes());
     mac.verify_slice(&signature).is_ok()
 }
@@ -814,7 +771,6 @@ pub fn verify_ns_scanner_capability_with_checkpoint_coverage(
 
 #[derive(Clone, Copy)]
 struct SignatureV2Scope<'a> {
-    background: bool,
     audience: &'a str,
     service: &'a str,
     rpc_method: &'a str,
@@ -824,7 +780,6 @@ struct SignatureV2Scope<'a> {
 }
 
 fn update_signature_v2(mac: &mut HmacSha256, scope: SignatureV2Scope<'_>) {
-    bind_background_admission(mac, scope.background);
     for part in [
         b"rustfs-rpc-auth-v2|".as_slice(),
         scope.audience.as_bytes(),
@@ -863,7 +818,6 @@ fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &st
 
 #[derive(Clone, Copy)]
 struct ReplayScope<'a> {
-    background: bool,
     audience: &'a str,
     path: &'a str,
     timestamp: &'a str,
@@ -873,7 +827,6 @@ struct ReplayScope<'a> {
 }
 
 fn update_replay_scope(mac: &mut HmacSha256, scope: ReplayScope<'_>) {
-    bind_background_admission(mac, scope.background);
     mac.update(RPC_REPLAY_SCOPE_DOMAIN);
     for part in [
         scope.audience.as_bytes(),
@@ -1017,24 +970,6 @@ pub fn gen_tonic_replay_scope_headers(
     timestamp: &str,
     content_sha256: &str,
     boot_epoch: Uuid,
-) -> std::io::Result<HeaderMap> {
-    gen_tonic_replay_scope_headers_with_admission(
-        audience,
-        path,
-        timestamp,
-        content_sha256,
-        boot_epoch,
-        current_rpc_background_request(),
-    )
-}
-
-pub(crate) fn gen_tonic_replay_scope_headers_with_admission(
-    audience: &str,
-    path: &str,
-    timestamp: &str,
-    content_sha256: &str,
-    boot_epoch: Uuid,
-    background: bool,
 ) -> std::io::Result<HeaderMap> {
     if audience.is_empty() || !path.starts_with('/') || !valid_content_sha256(content_sha256) || boot_epoch.is_nil() {
         return Err(std::io::Error::other("Invalid replay-scoped RPC signing scope"));
@@ -1048,7 +983,6 @@ pub(crate) fn gen_tonic_replay_scope_headers_with_admission(
     let signature = generate_replay_scope_signature(
         &get_shared_secret()?,
         ReplayScope {
-            background,
             audience,
             path,
             timestamp,
@@ -1058,7 +992,6 @@ pub(crate) fn gen_tonic_replay_scope_headers_with_admission(
         },
     )?;
     let mut headers = HeaderMap::new();
-    insert_background_admission(&mut headers, background);
     headers.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
     headers.insert(
         RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
@@ -1293,12 +1226,10 @@ pub fn build_auth_headers(url: &str, method: &Method, headers: &mut HeaderMap) -
 pub fn gen_signature_headers(url: &str, method: &Method) -> std::io::Result<HeaderMap> {
     let secret = get_shared_secret()?;
     let timestamp = OffsetDateTime::now_utc().unix_timestamp();
-    let background = current_rpc_background_request();
 
-    let signature = generate_signature(&secret, url, method, timestamp, background);
+    let signature = generate_signature(&secret, url, method, timestamp);
 
     let mut headers = HeaderMap::new();
-    insert_background_admission(&mut headers, background);
     headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).expect("Invalid header value"));
     headers.insert(
         TIMESTAMP_HEADER,
@@ -1333,12 +1264,10 @@ pub fn gen_tonic_signature_headers(
     let timestamp_header = timestamp.to_string();
     let body_nonce = (content_sha256 != UNSIGNED_PAYLOAD).then(|| Uuid::new_v4().to_string());
     let nonce = body_nonce.as_deref().unwrap_or(UNSIGNED_PAYLOAD_NONCE);
-    let background = current_rpc_background_request();
-    let legacy_signature = generate_signature(&secret, TONIC_RPC_PREFIX, &Method::GET, timestamp, background);
+    let legacy_signature = generate_signature(&secret, TONIC_RPC_PREFIX, &Method::GET, timestamp);
     let signature_v2 = generate_signature_v2(
         &secret,
         SignatureV2Scope {
-            background,
             audience,
             service,
             rpc_method,
@@ -1349,7 +1278,6 @@ pub fn gen_tonic_signature_headers(
     )?;
 
     let mut headers = HeaderMap::new();
-    insert_background_admission(&mut headers, background);
     headers.insert(SIGNATURE_HEADER, header_value(&legacy_signature, SIGNATURE_HEADER)?);
     headers.insert(TIMESTAMP_HEADER, header_value(&timestamp_header, TIMESTAMP_HEADER)?);
     headers.insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
@@ -1573,7 +1501,6 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
     if !verify_replay_scope_signature(
         &secret,
         ReplayScope {
-            background: rpc_background_admission(headers)?,
             audience,
             path,
             timestamp,
@@ -1588,13 +1515,7 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
     if boot_epoch != tonic_rpc_boot_epoch() {
         return Err(std::io::Error::other("RPC boot epoch is stale"));
     }
-    check_and_record_tonic_nonce(nonce, signed_at, path)?;
-    // A body-bound request also carries a usable v2 signature. Consume both
-    // nonces so stripping the replay-scope headers cannot authorize it again.
-    if content_sha256 != UNSIGNED_PAYLOAD {
-        verify_tonic_rpc_signature_with_strictness(audience, path, headers, true)?;
-    }
-    Ok(())
+    check_and_record_tonic_nonce(nonce, signed_at, path)
 }
 
 /// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
@@ -1704,9 +1625,6 @@ fn verify_tonic_rpc_signature_with_strictness(
     strict: bool,
 ) -> std::io::Result<()> {
     if !has_v2_auth_headers(headers) {
-        if rpc_background_admission(headers)? {
-            return Err(std::io::Error::other("RPC background admission requires v2 authentication"));
-        }
         // RUSTFS_COMPAT_TODO(heal-rpc-auth-v2): accept old peers during rolling upgrades. Remove after the minimum
         // supported RustFS peer version sends v2 authentication on every internode gRPC request.
         if strict {
@@ -1778,7 +1696,6 @@ fn verify_tonic_rpc_signature_with_strictness(
     if !verify_signature_v2(
         &secret,
         SignatureV2Scope {
-            background: rpc_background_admission(headers)?,
             audience,
             service,
             rpc_method,
@@ -1819,7 +1736,7 @@ pub fn verify_rpc_signature(url: &str, method: &Method, headers: &HeaderMap) ->
     // Verify signature with constant-time HMAC comparison.
     let secret = get_shared_secret()?;
 
-    if !verify_signature(&secret, url, method, timestamp, signature, rpc_background_admission(headers)?) {
+    if !verify_signature(&secret, url, method, timestamp, signature) {
         let rpc_path = redacted_rpc_path(url);
         error!(
             rpc_path = %rpc_path,
@@ -1911,98 +1828,6 @@ mod tests {
         runtime_sources::ensure_test_rpc_secret();
     }
 
-    #[tokio::test]
-    async fn background_admission_is_bound_in_http_and_both_tonic_signature_lanes() {
-        ensure_test_rpc_secret();
-        let secret = get_shared_secret().expect("RPC secret");
-        let path = "/node_service.NodeService/ReadAll";
-        for background in [false, true] {
-            let (http, tonic) = with_rpc_background_request(background, async {
-                (
-                    gen_signature_headers("/rustfs/rpc/read?disk=a", &Method::GET).expect("HTTP signature"),
-                    gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "ReadAll", None)
-                        .expect("gRPC signature"),
-                )
-            })
-            .await;
-            assert_eq!(rpc_background_admission(&http).expect("HTTP class"), background);
-            assert_eq!(rpc_background_admission(&tonic).expect("gRPC class"), background);
-            verify_rpc_signature("/rustfs/rpc/read?disk=a", &Method::GET, &http).expect("HTTP authentication");
-            verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &tonic, true).expect("v2 authentication");
-
-            let timestamp = tonic[TIMESTAMP_HEADER].to_str().expect("timestamp");
-            let content = tonic[RPC_CONTENT_SHA256_HEADER].to_str().expect("content digest");
-            assert_eq!(
-                verify_signature_v2(
-                    &secret,
-                    SignatureV2Scope {
-                        background: false,
-                        audience: "node-a:9000",
-                        service: "node_service.NodeService",
-                        rpc_method: "ReadAll",
-                        timestamp,
-                        nonce: tonic[RPC_NONCE_HEADER].to_str().expect("nonce"),
-                        content_sha256: content,
-                    },
-                    tonic[RPC_SIGNATURE_V2_HEADER].to_str().expect("signature"),
-                ),
-                !background,
-                "an old peer cannot authenticate a background request as foreground"
-            );
-
-            let mut replay = tonic.clone();
-            // The channel can run outside the caller's task-local scope.
-            replay.extend(
-                gen_tonic_replay_scope_headers_with_admission(
-                    "node-a:9000",
-                    path,
-                    timestamp,
-                    content,
-                    tonic_rpc_boot_epoch(),
-                    background,
-                )
-                .expect("v3 signature"),
-            );
-            for mutation in ["strip-or-add", "unknown", "duplicate"] {
-                let alter = |headers: &HeaderMap| {
-                    let mut changed = headers.clone();
-                    match mutation {
-                        "strip-or-add" if background => {
-                            changed.remove(RPC_BACKGROUND_ADMISSION_HEADER);
-                        }
-                        "strip-or-add" => insert_background_admission(&mut changed, true),
-                        "unknown" => {
-                            changed.insert(RPC_BACKGROUND_ADMISSION_HEADER, HeaderValue::from_static("v2"));
-                        }
-                        _ => {
-                            insert_background_admission(&mut changed, true);
-                            changed.append(RPC_BACKGROUND_ADMISSION_HEADER, HeaderValue::from_static("v1"));
-                        }
-                    }
-                    changed
-                };
-                assert!(verify_rpc_signature("/rustfs/rpc/read?disk=a", &Method::GET, &alter(&http)).is_err());
-                assert!(verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &alter(&tonic), true).is_err());
-                assert!(verify_tonic_replay_scope_signature("node-a:9000", path, &alter(&replay)).is_err());
-            }
-            verify_tonic_replay_scope_signature("node-a:9000", path, &replay).expect("v3 authentication");
-            assert!(
-                verify_tonic_replay_scope_signature("node-a:9000", path, &replay).is_err(),
-                "replay rejected"
-            );
-
-            if background {
-                let mut downgraded = HeaderMap::new();
-                for name in [SIGNATURE_HEADER, TIMESTAMP_HEADER, RPC_BACKGROUND_ADMISSION_HEADER] {
-                    downgraded.insert(name, tonic[name].clone());
-                }
-                assert!(verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &downgraded, false).is_err());
-                downgraded.remove(RPC_BACKGROUND_ADMISSION_HEADER);
-                assert!(verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &downgraded, false).is_err());
-            }
-        }
-    }
-
     #[test]
     fn namespace_scanner_capability_proof_binds_challenge_and_server_epoch() {
         let secret = "test-scanner-capability-secret";
@@ -2140,8 +1965,8 @@ mod tests {
         let method = Method::GET;
         let timestamp = 1640995200; // Fixed timestamp
 
-        let signature1 = generate_signature(secret, url, &method, timestamp, false);
-        let signature2 = generate_signature(secret, url, &method, timestamp, false);
+        let signature1 = generate_signature(secret, url, &method, timestamp);
+        let signature2 = generate_signature(secret, url, &method, timestamp);
 
         assert_eq!(signature1, signature2, "Same inputs should produce same signature");
         assert!(!signature1.is_empty(), "Signature should not be empty");
@@ -2154,10 +1979,10 @@ mod tests {
         let method = Method::GET;
         let timestamp = 1640995200;
 
-        let signature1 = generate_signature(secret, url, &method, timestamp, false);
-        let signature2 = generate_signature(secret, "http://different.com/api/test2", &method, timestamp, false);
-        let signature3 = generate_signature(secret, url, &Method::POST, timestamp, false);
-        let signature4 = generate_signature(secret, url, &method, timestamp + 1, false);
+        let signature1 = generate_signature(secret, url, &method, timestamp);
+        let signature2 = generate_signature(secret, "http://different.com/api/test2", &method, timestamp);
+        let signature3 = generate_signature(secret, url, &Method::POST, timestamp);
+        let signature4 = generate_signature(secret, url, &method, timestamp + 1);
 
         assert_ne!(signature1, signature2, "Different URLs should produce different signatures");
         assert_ne!(signature1, signature3, "Different methods should produce different signatures");
@@ -2263,14 +2088,14 @@ mod tests {
         let url = "http://example.com/api/test";
         let method = Method::GET;
         let timestamp = 1640995200;
-        let signature = generate_signature(secret, url, &method, timestamp, false);
+        let signature = generate_signature(secret, url, &method, timestamp);
         let mut tampered = base64_simd::STANDARD.decode_to_vec(&signature).unwrap();
         tampered[0] ^= 1;
         let tampered_signature = base64_simd::STANDARD.encode_to_string(tampered);
 
-        assert!(verify_signature(secret, url, &method, timestamp, &signature, false));
-        assert!(!verify_signature(secret, url, &method, timestamp, &tampered_signature, false));
-        assert!(!verify_signature(secret, url, &method, timestamp, "invalid-signature", false));
+        assert!(verify_signature(secret, url, &method, timestamp, &signature));
+        assert!(!verify_signature(secret, url, &method, timestamp, &tampered_signature));
+        assert!(!verify_signature(secret, url, &method, timestamp, "invalid-signature"));
     }
 
     #[test]
@@ -2287,11 +2112,11 @@ mod tests {
         );
         let method = Method::GET;
         let timestamp = 1_640_995_200;
-        let signature = generate_signature(secret, signed_url, &method, timestamp, false);
+        let signature = generate_signature(secret, signed_url, &method, timestamp);
 
-        assert!(verify_signature(secret, signed_url, &method, timestamp, &signature, false));
-        assert!(!verify_signature(secret, downgraded_url, &method, timestamp, &signature, false));
-        assert!(!verify_signature(secret, tampered_body_digest, &method, timestamp, &signature, false));
+        assert!(verify_signature(secret, signed_url, &method, timestamp, &signature));
+        assert!(!verify_signature(secret, downgraded_url, &method, timestamp, &signature));
+        assert!(!verify_signature(secret, tampered_body_digest, &method, timestamp, &signature));
     }
 
     #[test]
@@ -2301,7 +2126,7 @@ mod tests {
         let method = Method::GET;
         let timestamp = OffsetDateTime::now_utc().unix_timestamp();
         let secret = get_shared_secret().expect("test RPC secret should resolve");
-        let expected_signature = generate_signature(&secret, url, &method, timestamp, false);
+        let expected_signature = generate_signature(&secret, url, &method, timestamp);
         let invalid_signature = "invalid-signature";
         let logs = CapturedLogs::default();
         let subscriber = tracing_subscriber::fmt()
@@ -2339,7 +2164,7 @@ mod tests {
         // Set expired timestamp (older than SIGNATURE_VALID_DURATION)
         let expired_timestamp = OffsetDateTime::now_utc().unix_timestamp() - SIGNATURE_VALID_DURATION - 10;
         let secret = get_shared_secret().expect("test RPC secret should resolve");
-        let signature = generate_signature(&secret, url, &method, expired_timestamp, false);
+        let signature = generate_signature(&secret, url, &method, expired_timestamp);
 
         headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
         headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&expired_timestamp.to_string()).unwrap());
@@ -2361,7 +2186,7 @@ mod tests {
 
         let future_timestamp = OffsetDateTime::now_utc().unix_timestamp() + SIGNATURE_VALID_DURATION + 10;
         let secret = get_shared_secret().expect("test RPC secret should resolve");
-        let signature = generate_signature(&secret, url, &method, future_timestamp, false);
+        let signature = generate_signature(&secret, url, &method, future_timestamp);
 
         headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
         headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&future_timestamp.to_string()).unwrap());
@@ -2475,7 +2300,7 @@ mod tests {
         // Test timestamp just within valid duration
         let valid_timestamp = current_time - SIGNATURE_VALID_DURATION + 1;
 
-        let signature = generate_signature(&secret, url, &method, valid_timestamp, false);
+        let signature = generate_signature(&secret, url, &method, valid_timestamp);
 
         headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
         headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&valid_timestamp.to_string()).unwrap());
@@ -2486,7 +2311,7 @@ mod tests {
         // Test timestamp just outside valid duration
         let mut headers = HeaderMap::new();
         let invalid_timestamp = current_time - SIGNATURE_VALID_DURATION - 15;
-        let signature = generate_signature(&secret, url, &method, invalid_timestamp, false);
+        let signature = generate_signature(&secret, url, &method, invalid_timestamp);
 
         headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
         headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&invalid_timestamp.to_string()).unwrap());
@@ -2585,43 +2410,6 @@ mod tests {
         assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
     }
 
-    #[test]
-    fn replay_scope_consumes_the_body_bound_v2_nonce() {
-        ensure_test_rpc_secret();
-        let audience = "node-a:9000";
-        let path = "/node_service.NodeService/RecoverDeleteTransaction";
-        let digest = "a".repeat(64);
-        for v3_first in [true, false] {
-            let mut headers =
-                gen_tonic_signature_headers(audience, "node_service.NodeService", "RecoverDeleteTransaction", Some(&digest))
-                    .expect("body-bound v2 headers should build");
-            let timestamp = headers[TIMESTAMP_HEADER].to_str().expect("timestamp").to_string();
-            headers.extend(
-                gen_tonic_replay_scope_headers(audience, path, &timestamp, &digest, tonic_rpc_boot_epoch())
-                    .expect("replay-scope headers should build"),
-            );
-            let mut downgraded = headers.clone();
-            for header in [
-                RPC_REPLAY_SCOPE_VERSION_HEADER,
-                RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
-                RPC_REPLAY_SCOPE_NONCE_HEADER,
-                RPC_BOOT_EPOCH_HEADER,
-            ] {
-                downgraded.remove(header);
-            }
-            let (first, replay) = if v3_first {
-                (&headers, &downgraded)
-            } else {
-                (&downgraded, &headers)
-            };
-            verify_tonic_rpc_signature_with_policy(audience, path, first, true, false, false)
-                .expect("the first delivery must be accepted");
-            let error = verify_tonic_rpc_signature_with_policy(audience, path, replay, true, false, false)
-                .expect_err("a mutation cannot replay through the other signature generation");
-            assert_eq!(error.to_string(), "RPC request replay detected");
-        }
-    }
-
     #[test]
     fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
         ensure_test_rpc_secret();

crates/ecstore/src/cluster/rpc/internode_data_transport.rs

@@ -33,9 +33,7 @@ use rustfs_config::{
     DEFAULT_INTERNODE_DATA_TRANSPORT, ENV_RUSTFS_INTERNODE_DATA_TRANSPORT, INTERNODE_DATA_TRANSPORT_TCP,
     KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS,
 };
-use rustfs_rio::{
-    ChunkReaderBox, HttpChunkReader, HttpReader, HttpWriter, HttpWriterMemoryReservation, HttpWriterMemoryReserver,
-};
+use rustfs_rio::{ChunkReaderBox, HttpChunkReader, HttpReader, HttpWriter};
 use sha2::{Digest, Sha256};
 use std::collections::HashMap;
 use std::future::Future;
@@ -68,19 +66,6 @@ const PUT_FILE_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
 const CONTENT_TYPE_JSON: &str = "application/json";
 const CONTENT_TYPE_MSGPACK: &str = "application/msgpack";
 
-fn http_writer_memory_reserver() -> Option<HttpWriterMemoryReserver> {
-    crate::disk::current_resource_unit()
-        .and_then(|resources| resources.memory_budget())
-        .map(|budget| {
-            // Payloads can outlive the operation; never retain its node or disk units.
-            Arc::new(move |capacity| {
-                budget
-                    .reserve(capacity)
-                    .map(|reservation| Box::new(reservation) as HttpWriterMemoryReservation)
-            }) as HttpWriterMemoryReserver
-        })
-}
-
 fn unsupported_transport_message(transport: &str) -> String {
     format!(
         "invalid {ENV_RUSTFS_INTERNODE_DATA_TRANSPORT}={transport:?}; supported values: {}",
@@ -361,17 +346,13 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
     }
 
     async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
-        let memory_reserver = http_writer_memory_reserver();
         let server_epoch = self.put_file_auth_capability(&request.endpoint).await?;
         let auth_scope = server_epoch.map(|server_epoch| (Uuid::new_v4(), server_epoch));
         let url = build_put_file_stream_url(&request, auth_scope);
         let endpoint = request.endpoint;
         let mut headers = json_headers();
         build_auth_headers(&url, &Method::PUT, &mut headers)?;
-        let writer = match memory_reserver {
-            Some(reserve) => HttpWriter::new_with_memory_reserver(url.clone(), Method::PUT, headers, reserve).await?,
-            None => HttpWriter::new(url.clone(), Method::PUT, headers).await?,
-        };
+        let writer = HttpWriter::new(url.clone(), Method::PUT, headers).await?;
         match auth_scope {
             Some((nonce, server_epoch)) => Ok(Box::new(PutFileAuthWriter::new(writer, url, nonce, endpoint, server_epoch))),
             None => Ok(Box::new(writer)),
@@ -967,193 +948,6 @@ mod tests {
     use tokio::io::AsyncWriteExt;
     use tokio::sync::{Barrier, Notify};
 
-    fn memory_test_write_request(authenticated: bool) -> WriteStreamRequest {
-        crate::runtime::sources::ensure_test_rpc_secret();
-        let endpoint = format!("http://writer-memory-{}.invalid", Uuid::new_v4());
-        put_file_capability_cache_entry(&endpoint).write().cached = Some(if authenticated {
-            PutFileCapabilityState::V1 {
-                server_epoch: Uuid::new_v4(),
-                revalidate_after: Instant::now() + PUT_FILE_V1_CAPABILITY_TTL,
-            }
-        } else {
-            PutFileCapabilityState::LegacyUntil(Instant::now() + PUT_FILE_LEGACY_CAPABILITY_TTL)
-        });
-        WriteStreamRequest {
-            endpoint,
-            disk: "disk-a".to_owned(),
-            volume: "bucket".to_owned(),
-            path: "part.1".to_owned(),
-            append: false,
-            size: 4,
-        }
-    }
-
-    #[tokio::test]
-    async fn http_write_memory_reserver_outlives_only_the_memory_scope() {
-        use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
-
-        assert!(http_writer_memory_reserver().is_none(), "foreground must use the existing writer");
-        let memory = BackgroundMemoryBudget::new();
-        memory.set_sampled_limit(2 * 1024 * 1024, Duration::from_secs(30));
-        let (resources, gates) = test_resource_units(2);
-        let request = memory_test_write_request(true);
-        let endpoint = request.endpoint.clone();
-        let writer = with_optional_resource_unit(
-            Some(resources.with_memory_budget(memory.clone())),
-            TcpHttpInternodeDataTransport.open_write(request),
-        )
-        .await;
-        PUT_FILE_CAPABILITY_CACHE.write().remove(&endpoint);
-        let mut writer = writer.expect("admitted transport writer");
-        assert!(
-            gates.iter().all(|gate| gate.snapshot().active == 0),
-            "the callback must not retain disk units"
-        );
-        assert_eq!(memory.snapshot().used, 1024 * 1024);
-        writer.write_all(b"data").await.expect("buffered bytes");
-        writer.flush().await.expect("transfer charged allocation to queue");
-        memory.set_sampled_limit(2 * 1024 * 1024, Duration::ZERO);
-        writer
-            .write_all(&vec![0x5a; 1024 * 1024 - 4])
-            .await
-            .expect("reuse the already charged tail after observation expiry");
-        assert_eq!(memory.snapshot().used, 1024 * 1024, "reuse must not allocate separate storage");
-        let error = writer
-            .write(b"rejected")
-            .await
-            .expect_err("expired observation must reject new storage");
-        assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
-        assert!(
-            crate::disk::background_memory::is_memory_pressure(&error),
-            "preserve the typed memory-pressure marker"
-        );
-        assert_eq!(memory.snapshot().used, 1024 * 1024, "queued storage remains charged after expiry");
-        drop(writer);
-        tokio::task::yield_now().await;
-        assert_eq!(memory.snapshot().used, 0);
-    }
-
-    #[tokio::test]
-    async fn http_write_memory_denial_preserves_the_budget_error() {
-        use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
-
-        for (authenticated, sample) in [false, true].into_iter().flat_map(|authenticated| {
-            [
-                None,
-                Some((1024 * 1024 - 1, Duration::from_secs(30))),
-                Some((1024 * 1024, Duration::ZERO)),
-            ]
-            .into_iter()
-            .map(move |sample| (authenticated, sample))
-        }) {
-            let memory = BackgroundMemoryBudget::new();
-            if let Some((limit, valid_for)) = sample {
-                memory.set_sampled_limit(limit, valid_for);
-            }
-            let (resources, _) = test_resource_units(1);
-            let request = memory_test_write_request(authenticated);
-            let endpoint = request.endpoint.clone();
-            let result = with_optional_resource_unit(
-                Some(resources.with_memory_budget(memory.clone())),
-                TcpHttpInternodeDataTransport.open_write(request),
-            )
-            .await;
-            PUT_FILE_CAPABILITY_CACHE.write().remove(&endpoint);
-            let error = match result {
-                Err(Error::Io(error)) => error,
-                Err(error) => panic!("unexpected transport error: {error}"),
-                Ok(_) => panic!("unavailable memory must reject construction"),
-            };
-            assert!(crate::disk::background_memory::is_memory_pressure(&error));
-            assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
-            assert_eq!(memory.snapshot().used, 0);
-        }
-    }
-
-    #[tokio::test]
-    async fn admitted_remote_reads_bind_receiver_admission_to_the_http_signature() {
-        use crate::disk::with_background_resource_unit;
-        use rustfs_concurrency::adaptive::AdaptiveAdmission;
-        use std::num::NonZeroU16;
-
-        crate::runtime::sources::ensure_test_rpc_secret();
-        let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
-            .await
-            .expect("remote admission listener");
-        let address = listener.local_addr().expect("remote admission address");
-        let server = tokio::spawn(async move {
-            let (mut socket, _) = listener.accept().await.expect("remote read connection");
-            let mut bytes = Vec::new();
-            while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
-                let mut chunk = [0; 1024];
-                let read = socket.read(&mut chunk).await.expect("remote read headers");
-                assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete request headers");
-                bytes.extend_from_slice(&chunk[..read]);
-            }
-            let request = std::str::from_utf8(&bytes).expect("HTTP request");
-            let mut lines = request.split("\r\n");
-            let target = lines
-                .next()
-                .expect("request line")
-                .split_whitespace()
-                .nth(1)
-                .expect("request target")
-                .to_owned();
-            let mut headers = HeaderMap::new();
-            for line in lines.take_while(|line| !line.is_empty()) {
-                let (name, value) = line.split_once(':').expect("header field");
-                headers.append(
-                    http::header::HeaderName::from_bytes(name.as_bytes()).expect("header name"),
-                    HeaderValue::from_str(value.trim()).expect("header value"),
-                );
-            }
-            socket
-                .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 5\r\nconnection: close\r\n\r\nhello")
-                .await
-                .expect("remote read response");
-            (target, headers)
-        });
-        let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
-        let resource = gate.try_acquire().expect("sender device admission");
-        let mut reader = tokio::time::timeout(
-            Duration::from_secs(5),
-            with_background_resource_unit(
-                resource,
-                TcpHttpInternodeDataTransport.open_read(ReadStreamRequest {
-                    endpoint: format!("http://{address}"),
-                    disk: "disk-a".to_owned(),
-                    volume: "bucket".to_owned(),
-                    path: "part.1".to_owned(),
-                    offset: 0,
-                    length: 5,
-                    stall_timeout: Some(Duration::from_secs(5)),
-                }),
-            ),
-        )
-        .await
-        .expect("bounded remote request")
-        .expect("remote reader");
-        let mut body = Vec::new();
-        reader.read_to_end(&mut body).await.expect("complete remote body");
-        assert_eq!(body, b"hello");
-        let (target, headers) = server.await.expect("remote request capture");
-        assert_eq!(gate.snapshot().active, 0, "remote body does not retain the sender's device");
-        assert_eq!(
-            headers
-                .get("x-rustfs-rpc-background-admission")
-                .and_then(|value| value.to_str().ok()),
-            Some("v1"),
-            "resource-controlled remote work must request receiver-side admission"
-        );
-        crate::cluster::rpc::verify_rpc_signature(&target, &Method::GET, &headers).expect("bound request signature");
-        let mut stripped = headers;
-        stripped.remove("x-rustfs-rpc-background-admission");
-        assert!(
-            crate::cluster::rpc::verify_rpc_signature(&target, &Method::GET, &stripped).is_err(),
-            "removing background admission must invalidate the signature"
-        );
-    }
-
     async fn wait_for_capability_flight_waiters(entry: &PutFileCapabilityCacheEntry, waiters: usize) {
         tokio::time::timeout(Duration::from_secs(5), async {
             loop {

crates/ecstore/src/cluster/rpc/mod.rs

@@ -13,8 +13,6 @@
 // limitations under the License.
 
 pub(crate) mod background_monitor;
-pub(crate) mod background_pipe;
-pub(crate) mod background_reader_stream;
 pub(crate) mod client;
 pub(crate) mod context_propagation;
 pub(crate) mod http_auth;
@@ -35,17 +33,16 @@ pub use client::{
 // Re-exported through `api::rpc`; not every item is consumed inside this crate.
 #[allow(unused_imports)]
 pub use http_auth::{
-    RPC_BACKGROUND_ADMISSION_HEADER, TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer,
-    check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
-    normalize_tonic_rpc_audience, rpc_background_admission, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest,
-    set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest, sign_ns_scanner_capability,
-    sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_tier_registry_generation,
-    sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
-    tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
-    verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability,
-    verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
-    verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
-    verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
+    TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers,
+    gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
+    set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest,
+    sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
+    sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
+    tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
+    verify_ns_scanner_capability_with_checkpoint_coverage, verify_ns_scanner_capability_with_tier_registry_generation,
+    verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
+    verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
+    verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
 };
 #[cfg(test)]
 pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -1965,6 +1965,15 @@ impl PeerRestClient {
         Ok((self.topology_member.clone(), epoch))
     }
 
+    pub async fn probe_restore_copyback_capability(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
+        let probe = rustfs_protos::restore_copyback_capability_probe(Uuid::new_v4().as_bytes());
+        let result = self
+            .heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
+            .await?;
+        let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
+        Ok((self.topology_member.clone(), epoch))
+    }
+
     pub async fn probe_cross_pool_fence(&self, topology_fingerprint: String) -> Result<(String, u32, Uuid)> {
         let mut probe = rustfs_protos::CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX.to_vec();
         probe.extend_from_slice(Uuid::new_v4().as_bytes());
@@ -2014,21 +2023,9 @@ impl PeerRestClient {
             }
             result
         })
-        .await;
-        self.settle_bucket_metadata_reload(bucket, result).await
-    }
-
-    async fn settle_bucket_metadata_reload(
-        &self,
-        bucket: &str,
-        result: std::result::Result<Result<()>, tokio::time::error::Elapsed>,
-    ) -> Result<()> {
-        match result {
-            Ok(result) => self.finalize_result(result).await,
-            // A local operation deadline does not establish that the peer is
-            // unreachable. In particular, it must not gate IAM revocations.
-            Err(_) => Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))),
-        }
+        .await
+        .unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
+        self.finalize_result(result).await
     }
 
     async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
@@ -4765,31 +4762,6 @@ mod tests {
         assert!(client.offline.load(Ordering::Acquire));
     }
 
-    #[tokio::test]
-    async fn bucket_metadata_deadline_does_not_gate_healthy_peer_notifications() {
-        let client = test_peer_client();
-        let elapsed = tokio::time::timeout(std::time::Duration::ZERO, std::future::pending::<Result<()>>()).await;
-        let error = client
-            .settle_bucket_metadata_reload("slow-policy", elapsed)
-            .await
-            .expect_err("local budget expires");
-        assert!(error.to_string().contains("timed out"));
-        assert!(
-            !client.offline.load(Ordering::Acquire),
-            "local work must not suppress subsequent IAM RPCs"
-        );
-        client
-            .settle_bucket_metadata_reload("healthy", Ok(Ok(())))
-            .await
-            .expect("peer reply");
-        let error = client
-            .settle_bucket_metadata_reload("unreachable", Ok(Err(Error::from(tonic::Status::unavailable("connection refused")))))
-            .await
-            .expect_err("real transport failure");
-        assert!(error.to_string().contains("connection refused"));
-        assert!(client.offline.load(Ordering::Acquire), "transport failures must still enter recovery");
-    }
-
     #[tokio::test]
     async fn peer_rest_client_finalize_result_keeps_online_for_business_errors() {
         let client = test_peer_client();

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -17,7 +17,6 @@ use crate::cluster::rpc::client::{
     node_service_delete_recovery_client, node_service_time_out_client, node_service_time_out_client_for_class,
     node_service_time_out_client_no_auth,
 };
-use crate::cluster::rpc::http_auth::{current_rpc_background_request, with_rpc_background_request};
 use crate::cluster::rpc::internode_data_transport::{
     InternodeDataTransport, NsScannerCapabilityRequest, NsScannerStreamRequest, ReadStreamRequest, WalkDirStreamRequest,
     WriteStreamRequest,
@@ -571,7 +570,6 @@ struct RetryingRemoteReader {
     reader: Option<FileReader>,
     transport: Arc<dyn InternodeDataTransport>,
     request: ReadStreamRequest,
-    background: bool,
     emitted: usize,
     retried: bool,
     initial_read_timeout: Option<Duration>,
@@ -592,7 +590,6 @@ impl RetryingRemoteReader {
             reader: Some(reader),
             transport,
             request,
-            background: current_rpc_background_request(),
             emitted: 0,
             retried: false,
             initial_read_timeout,
@@ -610,10 +607,9 @@ impl RetryingRemoteReader {
         let request = resumed_read_request(&self.request, self.emitted)?;
         let recovery_timeout = self.recovery_timeout;
         let transport = Arc::clone(&self.transport);
-        self.resume = Some(AbortOnDropTask::new(tokio::spawn(with_rpc_background_request(
-            self.background,
-            async move { with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_fresh(request)).await },
-        ))));
+        self.resume = Some(AbortOnDropTask::new(tokio::spawn(async move {
+            with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_fresh(request)).await
+        })));
         Ok(())
     }
 
@@ -721,7 +717,6 @@ struct RetryingRemoteChunkReader {
     reader: Option<rustfs_rio::ChunkReaderBox>,
     transport: Arc<dyn InternodeDataTransport>,
     request: ReadStreamRequest,
-    background: bool,
     emitted: usize,
     retried: bool,
     initial_read_timeout: Option<Duration>,
@@ -742,7 +737,6 @@ impl RetryingRemoteChunkReader {
             reader: Some(reader),
             transport,
             request,
-            background: current_rpc_background_request(),
             emitted: 0,
             retried: false,
             initial_read_timeout,
@@ -760,10 +754,9 @@ impl RetryingRemoteChunkReader {
         let request = resumed_read_request(&self.request, self.emitted)?;
         let recovery_timeout = self.recovery_timeout;
         let transport = Arc::clone(&self.transport);
-        self.resume = Some(AbortOnDropTask::new(tokio::spawn(with_rpc_background_request(
-            self.background,
-            async move { with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_chunks_fresh(request)).await },
-        ))));
+        self.resume = Some(AbortOnDropTask::new(tokio::spawn(async move {
+            with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_chunks_fresh(request)).await
+        })));
         Ok(())
     }
 
@@ -1082,10 +1075,7 @@ impl RemoteDisk {
         if payload.len() > operation.max_wire_payload_bytes() {
             return Err(Error::other("delete recovery operation exceeds payload limit"));
         }
-        let settle_timeout = if matches!(
-            &operation,
-            DeleteRecoveryRpcOperation::Settle { .. } | DeleteRecoveryRpcOperation::SettleBounded(_)
-        ) {
+        let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) {
             Some(crate::disk::get_delete_recovery_timeout()?)
         } else {
             None
@@ -1640,7 +1630,6 @@ impl RemoteDisk {
         let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
         if initial_probe_ok {
             health.record_operation_success(&endpoint, "connectivity_probe_success");
-            rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
         } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
             warn!(
                 event = EVENT_REMOTE_DISK_HEALTH,
@@ -1712,7 +1701,6 @@ impl RemoteDisk {
                     // Perform basic connectivity check
                     if Self::perform_connectivity_check(&addr).await.is_ok() {
                         health.record_operation_success(&endpoint, "connectivity_probe_success");
-                        rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
                     } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
                         warn!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -1783,7 +1771,6 @@ impl RemoteDisk {
                 }
                 _ = interval.tick() => {
                     if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
-                        rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
                         let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
                         debug!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -2015,7 +2002,6 @@ impl RemoteDisk {
             let operation_result = operation().await;
             if operation_result.is_ok() {
                 self.health.record_operation_success(&self.endpoint, "operation_success");
-                rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
             }
             self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                 .await;
@@ -2030,7 +2016,6 @@ impl RemoteDisk {
                 // Log success; the waiting guard balances every exit path.
                 if operation_result.is_ok() {
                     self.health.record_operation_success(&self.endpoint, "operation_success");
-                    rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
                 }
                 self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                     .await;
@@ -2719,54 +2704,6 @@ impl RemoteDisk {
 
 #[async_trait::async_trait]
 impl DiskAPI for RemoteDisk {
-    async fn list_multipart_index_roots(
-        &self,
-        bucket: &str,
-        marker: Option<&str>,
-    ) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
-        use crate::disk::multipart_index::{MULTIPART_INDEX_RPC_MAX_BYTES, MultipartIndexRpcReply, MultipartIndexRpcRequest};
-        let expected_disk = self
-            .id
-            .lock()
-            .await
-            .filter(|id| !id.is_nil())
-            .ok_or(DiskError::DiskNotFound)?;
-        let query = MultipartIndexRpcRequest::new(bucket, marker);
-        let endpoint = self.endpoint.to_string();
-        let canonical = query.canonical_request(&endpoint)?;
-        let opts = serde_json::to_string(&query)?;
-        self.execute_with_timeout(
-            || async {
-                let mut client = self.get_client().await?;
-                let mut request = Request::new(DiskInfoRequest {
-                    disk: endpoint.clone(),
-                    opts: opts.clone(),
-                });
-                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
-                let response = client.disk_info(request).await?.into_inner();
-                if !response.success {
-                    return Err(response.error.unwrap_or_default().into());
-                }
-                if response.disk_info.len() > MULTIPART_INDEX_RPC_MAX_BYTES {
-                    return Err(DiskError::FileCorrupt);
-                }
-                // An old server ignores the additive query and returns ordinary DiskInfo.
-                // It cannot satisfy the dedicated reply schema or the fresh challenge.
-                let reply: MultipartIndexRpcReply =
-                    serde_json::from_str(&response.disk_info).map_err(|_| DiskError::MethodNotAllowed)?;
-                reply.validate(&query, expected_disk)?;
-                crate::cluster::rpc::verify_tonic_rpc_response_proof(
-                    &query.canonical_response(&endpoint, &reply.page)?,
-                    &reply.proof,
-                )
-                .map_err(Error::other)?;
-                Ok(reply.page)
-            },
-            get_drive_disk_info_timeout(),
-        )
-        .await
-    }
-
     async fn supports_guarded_delete_versions(&self) -> Result<()> {
         let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
         if !response.errors.is_empty() || !response.item_errors.is_empty() {
@@ -4350,35 +4287,6 @@ impl DiskAPI for RemoteDisk {
         }
     }
 
-    async fn discover_delete_transactions_bounded(
-        &self,
-        volume: &str,
-        object: &str,
-        max_entries: u32,
-    ) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
-        // Older peers reject this variant. Do not retry through the unbounded protocol.
-        match self
-            .recover_delete_transaction(
-                volume,
-                DeleteRecoveryRpcOperation::DiscoverBounded {
-                    object: object.to_owned(),
-                    max_entries,
-                },
-            )
-            .await
-        {
-            Ok(DeleteRecoveryRpcReply::DiscoveredBounded(page)) => Ok(page),
-            Ok(_) => Err(Error::FileCorrupt),
-            Err(error)
-                if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
-                    || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
-                    .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
-            {
-                Err(Error::DeleteRecoveryWorkLimit)
-            }
-            Err(error) => Err(error),
-        }
-    }
     async fn discover_delete_transactions(
         &self,
         volume: &str,
@@ -4414,58 +4322,6 @@ impl DiskAPI for RemoteDisk {
             _ => Err(Error::other("unexpected delete discovery reply")),
         }
     }
-    async fn settle_delete_transaction_bounded(
-        &self,
-        volume: &str,
-        request: crate::disk::BoundedDeleteRecoverySettlement,
-        _namespace_owner: Option<Arc<dyn Send + Sync>>,
-    ) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
-        match self
-            .recover_delete_transaction(volume, DeleteRecoveryRpcOperation::SettleBounded(request))
-            .await
-        {
-            Ok(DeleteRecoveryRpcReply::SettledBounded(reply)) => Ok(reply),
-            Ok(_) => Err(Error::FileCorrupt),
-            Err(error)
-                if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
-                    || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
-                    .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
-            {
-                Err(Error::DeleteRecoveryWorkLimit)
-            }
-            Err(error) => Err(error),
-        }
-    }
-    async fn inspect_delete_transaction_bounded(
-        &self,
-        volume: &str,
-        object: &str,
-        transaction: Uuid,
-        max_entries: u32,
-    ) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
-        match self
-            .recover_delete_transaction(
-                volume,
-                DeleteRecoveryRpcOperation::InspectBounded {
-                    object: object.to_owned(),
-                    transaction,
-                    max_entries,
-                },
-            )
-            .await
-        {
-            Ok(DeleteRecoveryRpcReply::InspectedBounded(page)) => Ok(page),
-            Ok(_) => Err(Error::FileCorrupt),
-            Err(error)
-                if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
-                    || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
-                    .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
-            {
-                Err(Error::DeleteRecoveryWorkLimit)
-            }
-            Err(error) => Err(error),
-        }
-    }
     async fn inspect_delete_transaction(
         &self,
         volume: &str,
@@ -4986,113 +4842,6 @@ mod tests {
         }
     }
 
-    #[tokio::test]
-    async fn bounded_delete_discovery_lost_remote_reply_leaves_no_cursor() {
-        runtime_sources::ensure_test_rpc_secret();
-        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
-            .await
-            .expect("test peer must bind");
-        let directory = tempfile::tempdir().expect("temporary disk");
-        let endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("endpoint");
-        let disk = Arc::new(
-            crate::disk::local::LocalDisk::new(&endpoint, false)
-                .await
-                .expect("local disk"),
-        );
-        disk.make_volume("bucket").await.expect("bucket");
-        let prefix = directory.path().join("bucket/prefix");
-        std::fs::create_dir(&prefix).expect("prefix");
-        for index in 0..64 {
-            std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
-        }
-        let fixture = Arc::new(TestBoundedDiscovery {
-            disk: Arc::clone(&disk),
-            response_gate: TestFormatReadGate::default(),
-        });
-        *peer.peer.bounded_discovery.lock().expect("bounded fixture") = Some(Arc::clone(&fixture));
-        let remote = remote_disk_for_addr(&peer.addr).await;
-        let task = tokio::spawn(async move { remote.discover_delete_transactions_bounded("bucket", "prefix", 8).await });
-        tokio::time::timeout(Duration::from_secs(5), fixture.response_gate.entered.notified())
-            .await
-            .expect("disk completed bounded scan");
-        task.abort();
-        assert!(task.await.expect_err("caller cancelled").is_cancelled());
-        assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
-        fixture.response_gate.release.cancel();
-        peer.stop().await;
-        assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
-        assert!(prefix.join("child-63").is_dir());
-    }
-
-    #[tokio::test]
-    async fn multipart_index_old_disk_info_handler_is_not_a_ready_empty_index() {
-        runtime_sources::ensure_test_rpc_secret();
-        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
-            .await
-            .expect("old peer must bind");
-        let remote = remote_disk_for_addr(&peer.addr).await;
-        remote
-            .set_disk_id(Some(Uuid::new_v4()))
-            .await
-            .expect("expected disk identity");
-        assert!(matches!(
-            remote.list_multipart_index_roots("bucket", None).await,
-            Err(DiskError::MethodNotAllowed)
-        ));
-        assert_eq!(
-            peer.peer.disk_info_calls.load(Ordering::Acquire),
-            1,
-            "exercise the actual legacy JSON handler"
-        );
-        peer.stop().await;
-    }
-
-    #[tokio::test]
-    async fn bounded_delete_discovery_old_peer_rejects_without_paginated_fallback() {
-        runtime_sources::ensure_test_rpc_secret();
-        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
-            .await
-            .expect("old peer must bind");
-        let remote = remote_disk_for_addr(&peer.addr).await;
-        assert_eq!(
-            remote
-                .discover_delete_transactions_bounded("bucket", "prefix", 4096)
-                .await
-                .expect_err("old peer cannot prove a bounded inventory"),
-            DiskError::DeleteRecoveryWorkLimit
-        );
-        let transaction = Uuid::new_v4();
-        assert_eq!(
-            remote
-                .inspect_delete_transaction_bounded("bucket", "prefix", transaction, 65536)
-                .await
-                .expect_err("old peer cannot inspect with a budget"),
-            DiskError::DeleteRecoveryWorkLimit
-        );
-        assert_eq!(
-            remote
-                .settle_delete_transaction_bounded(
-                    "bucket",
-                    crate::disk::BoundedDeleteRecoverySettlement {
-                        object: "prefix".to_owned(),
-                        transaction,
-                        expected_fingerprint: [0; 32],
-                        action: DeleteRecoveryAction::PublishCommit,
-                        max_entries: 65536,
-                    },
-                    None
-                )
-                .await
-                .expect_err("old peer cannot settle with a budget"),
-            DiskError::DeleteRecoveryWorkLimit
-        );
-        assert_eq!(
-            *peer.peer.unsupported_methods.lock().expect("observed methods"),
-            vec!["/node_service.NodeService/RecoverDeleteTransaction"; 3]
-        );
-        peer.stop().await;
-    }
-
     #[tokio::test]
     async fn multipart_source_recovery_old_peer_rejects_without_generic_fallback() {
         runtime_sources::ensure_test_rpc_secret();
@@ -5985,16 +5734,9 @@ mod tests {
         format_data: Bytes,
         read_all_data: Bytes,
         next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
-        bounded_discovery: Arc<StdMutex<Option<Arc<TestBoundedDiscovery>>>>,
         unsupported_methods: Arc<StdMutex<Vec<String>>>,
     }
 
-    #[derive(Debug)]
-    struct TestBoundedDiscovery {
-        disk: Arc<crate::disk::local::LocalDisk>,
-        response_gate: TestFormatReadGate,
-    }
-
     #[derive(Debug, Default)]
     struct TestFormatReadGate {
         entered: tokio::sync::Notify,
@@ -6011,7 +5753,6 @@ mod tests {
                 format_data,
                 read_all_data,
                 next_format_read_gate: Arc::default(),
-                bounded_discovery: Arc::default(),
                 unsupported_methods: Arc::default(),
             }
         }
@@ -6111,56 +5852,6 @@ mod tests {
                         Ok(grpc.unary(method, request).await)
                     })
                 }
-                "/node_service.NodeService/RecoverDeleteTransaction"
-                    if self.peer.bounded_discovery.lock().expect("bounded fixture").is_some() =>
-                {
-                    #[derive(Clone)]
-                    struct BoundedSvc(Arc<TestBoundedDiscovery>);
-                    impl tonic::server::UnaryService<RecoverDeleteTransactionRequest> for BoundedSvc {
-                        type Response = rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse;
-                        type Future = Pin<Box<dyn Future<Output = std::result::Result<Response<Self::Response>, Status>> + Send>>;
-                        fn call(&mut self, request: Request<RecoverDeleteTransactionRequest>) -> Self::Future {
-                            let fixture = Arc::clone(&self.0);
-                            Box::pin(async move {
-                                let request = request.into_inner();
-                                let operation: DeleteRecoveryRpcOperation =
-                                    rustfs_protos::decode_delete_recovery_payload(&request.operation)
-                                        .map_err(|_| Status::invalid_argument("invalid bounded request"))?;
-                                let DeleteRecoveryRpcOperation::DiscoverBounded { object, max_entries } = operation else {
-                                    return Err(Status::invalid_argument("unexpected fallback"));
-                                };
-                                let page = fixture
-                                    .disk
-                                    .discover_delete_transactions_bounded(&request.volume, &object, max_entries)
-                                    .await
-                                    .map_err(|_| Status::internal("bounded discovery failed"))?;
-                                // Lose the response only after the real disk operation has completed.
-                                fixture.response_gate.entered.notify_one();
-                                fixture.response_gate.release.cancelled().await;
-                                let result = rmp_serde::to_vec_named(&DeleteRecoveryRpcReply::DiscoveredBounded(page))
-                                    .map_err(|_| Status::internal("encode bounded reply"))?;
-                                Ok(Response::new(rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse {
-                                    success: true,
-                                    protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
-                                    result: result.into(),
-                                    error: None,
-                                }))
-                            })
-                        }
-                    }
-                    let fixture = self
-                        .peer
-                        .bounded_discovery
-                        .lock()
-                        .expect("bounded fixture")
-                        .clone()
-                        .expect("configured fixture");
-                    Box::pin(async move {
-                        let codec = tonic_prost::ProstCodec::default();
-                        let mut grpc = tonic::server::Grpc::new(codec);
-                        Ok(grpc.unary(BoundedSvc(fixture), request).await)
-                    })
-                }
                 "/node_service.NodeService/ReadAll" => {
                     #[derive(Clone)]
                     struct ReadAllSvc(AuthenticatedReadPeer);
@@ -8078,98 +7769,6 @@ mod tests {
         .await;
     }
 
-    #[tokio::test]
-    async fn remote_read_retries_preserve_signed_admission_after_sender_scope_ends() {
-        use crate::cluster::rpc::{rpc_background_admission, verify_rpc_signature};
-        use crate::disk::with_background_resource_unit;
-        use rustfs_concurrency::adaptive::AdaptiveAdmission;
-        use std::num::NonZeroU16;
-
-        crate::runtime::sources::ensure_test_rpc_secret();
-        for background in [false, true] {
-            for chunked in [false, true] {
-                let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("retry listener");
-                let address = listener.local_addr().expect("retry address");
-                let server = tokio::spawn(async move {
-                    let (mut socket, _) = listener.accept().await.expect("fresh connection");
-                    let mut bytes = Vec::new();
-                    while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
-                        let mut chunk = [0; 1024];
-                        let read = socket.read(&mut chunk).await.expect("retry request");
-                        assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete headers");
-                        bytes.extend_from_slice(&chunk[..read]);
-                    }
-                    let request = std::str::from_utf8(&bytes).expect("HTTP request");
-                    let mut lines = request.split("\r\n");
-                    let target = lines
-                        .next()
-                        .expect("request line")
-                        .split_whitespace()
-                        .nth(1)
-                        .expect("target")
-                        .to_owned();
-                    let mut headers = http::HeaderMap::new();
-                    for line in lines.take_while(|line| !line.is_empty()) {
-                        let (name, value) = line.split_once(':').expect("header");
-                        headers.append(
-                            http::header::HeaderName::from_bytes(name.as_bytes()).expect("header name"),
-                            http::HeaderValue::from_str(value.trim()).expect("header value"),
-                        );
-                    }
-                    socket
-                        .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 6\r\nconnection: close\r\n\r\n456789")
-                        .await
-                        .expect("retry response");
-                    (target, headers)
-                });
-                let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
-                let request = ReadStreamRequest {
-                    endpoint: format!("http://{address}"),
-                    ..resume_request(10)
-                };
-                let create = async {
-                    let transport = Arc::new(TcpHttpInternodeDataTransport);
-                    if chunked {
-                        Box::new(RetryingRemoteChunkReader::new_with_timeouts(
-                            resume_step_chunk_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec())),
-                            transport,
-                            request,
-                            None,
-                            None,
-                        )) as FileReader
-                    } else {
-                        Box::new(RetryingRemoteReader::new_with_timeouts(
-                            resume_step_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec())),
-                            transport,
-                            request,
-                            None,
-                            None,
-                        )) as FileReader
-                    }
-                };
-                let mut reader = if background {
-                    with_background_resource_unit(gate.try_acquire().expect("sender admission"), create).await
-                } else {
-                    create.await
-                };
-                assert_eq!(gate.snapshot().active, 0, "retry reader cannot retain sender disk capacity");
-                let mut output = Vec::new();
-                tokio::time::timeout(
-                    Duration::from_secs(5),
-                    with_rpc_background_request(!background, reader.read_to_end(&mut output)),
-                )
-                .await
-                .expect("bounded retry")
-                .expect("resumed body");
-                assert_eq!(output, b"0123456789", "chunked={chunked}, background={background}");
-                let (target, headers) = server.await.expect("captured retry");
-                assert!(target.contains("offset=11") && target.contains("length=6"), "{target}");
-                assert_eq!(rpc_background_admission(&headers).expect("admission marker"), background);
-                verify_rpc_signature(&target, &http::Method::GET, &headers).expect("signed retry class");
-            }
-        }
-    }
-
     #[tokio::test]
     async fn remote_reader_resumes_from_emitted_bytes_without_duplicates() {
         let transport = Arc::new(ResumeTransport::with_read_steps(vec![ResumeReadStep::Data(b"456789".to_vec())]));

crates/ecstore/src/cluster/rpc/remote_locker.rs

@@ -445,10 +445,6 @@ impl RemoteClient {
                 }
             }
         });
-        // A successful lock RPC over a cached or fresh channel proves the peer is
-        // reachable; reconcile the internode peer-health gauge so it does not stay
-        // stale when no fresh dial fires (issue #9119).
-        rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
     }
 
     fn admit_late_release(&self) -> bool {
@@ -676,7 +672,6 @@ impl RemoteClient {
                     with_lock_peer_health(&addr, |health| {
                         health.last_success = Some(Instant::now());
                     });
-                    rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
                     if let Some(late) = late {
                         late(response).await;

crates/ecstore/src/config/com.rs

@@ -3506,7 +3506,6 @@ mod tests {
                 replication_decision: String::new(),
                 checksum: None,
                 multipart_completion_replayed: false,
-                backfilled_old_current_size: None,
             }
         }
     }

crates/ecstore/src/core/pool_meta_native_integration_tests.rs

@@ -18,7 +18,6 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
 use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
 use std::path::PathBuf;
 use tokio::fs;
-use uuid::Uuid;
 
 struct PhysicalFixture {
     _directories: Vec<tempfile::TempDir>,
@@ -629,559 +628,3 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
         );
     }
 }
-
-async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
-    let fixture = fixture(0).await;
-    for pool in disks(&fixture.store).await {
-        for disk in pool {
-            let object = object_path(&disk);
-            let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
-            while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
-                if entry.file_type().await.expect("fixture entry type").is_dir() {
-                    fs::remove_dir_all(entry.path())
-                        .await
-                        .expect("remove task-owned fixture rollback directory");
-                }
-            }
-        }
-    }
-    fixture
-}
-
-async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
-    let mut all = Vec::new();
-    for pool in &store.pools {
-        let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
-        let members = set
-            .disks
-            .read()
-            .await
-            .iter()
-            .cloned()
-            .map(|disk| disk.expect("identity fixture disk must be online"))
-            .collect::<Vec<_>>();
-        let mut heads = Vec::new();
-        for disk in members {
-            match disk
-                .pool_meta_native_recovery(Request::InspectIdentity {})
-                .await
-                .expect("physical identity inspection")
-            {
-                Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
-                _ => panic!("identity inspection must return its physical head"),
-            }
-        }
-        all.push(heads);
-    }
-    all
-}
-
-async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
-    let fixture = owner_repair_no_backups_fixture().await;
-    let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
-        PoolMetaReplica::Valid {
-            meta, committed: true, ..
-        } => meta,
-        _ => panic!("fixture must provide valid committed pool metadata"),
-    };
-    meta.version = version;
-    let raw = meta
-        .encode_config_data_for_v2_gate(version == POOL_META_VERSION)
-        .expect("legacy fixture should encode in its original version");
-    assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
-    let all = disks(&fixture.store).await;
-    let now = OffsetDateTime::now_utc();
-    for pool in &all {
-        let heads = replicas(&raw, now).await;
-        for (index, disk) in pool.iter().enumerate() {
-            fs::write(object_path(disk).join("xl.meta"), &heads[index])
-                .await
-                .expect("install exact legacy head");
-        }
-    }
-    if stale {
-        let old = replicas(&raw, now - Duration::seconds(10)).await;
-        fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
-            .await
-            .expect("install stale same-raw legacy head");
-    }
-    (fixture, raw)
-}
-
-#[tokio::test]
-#[serial_test::serial(pool_meta_version_env)]
-async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
-    temp_env::async_with_vars(
-        [
-            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
-        ],
-        async {
-            let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
-            let guard = owner(&fixture.store).await;
-            let mut state = fixture.store.pool_meta_save_gate.lock().await;
-            let result = tokio::time::timeout(
-                std::time::Duration::from_secs(30),
-                fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
-            )
-            .await
-            .expect("legacy native heal must terminate");
-            assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
-            drop(state);
-            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-            for read in &reads {
-                assert!(
-                    matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
-                    if bytes == &raw && revision.version == POOL_META_V1_VERSION),
-                    "native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
-                );
-            }
-        },
-    )
-    .await;
-}
-
-#[tokio::test]
-#[serial_test::serial(pool_meta_version_env)]
-async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
-    temp_env::async_with_vars(
-        [
-            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
-        ],
-        async {
-            for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
-                let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
-                let mut source_raw = raw.clone();
-                // A second MessagePack value is readable by legacy decoding but
-                // must not be copied as part of an unverified repair payload.
-                source_raw.push(0xc0);
-                assert!(
-                    matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
-                    "ordinary legacy reads must remain compatible with trailing bytes"
-                );
-                let all_disks = disks(&fixture.store).await;
-                let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
-                for (index, disk) in all_disks[0].iter().enumerate() {
-                    fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
-                        .await
-                        .expect("install readable legacy source with trailing bytes");
-                }
-                let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
-                    .await
-                    .expect("legacy semantic selection must remain readable");
-                assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
-                let before = snapshots(&fixture.store).await;
-                let identities = owner_repair_identity_heads(&fixture.store).await;
-                let guard = owner(&fixture.store).await;
-                let mut state = fixture.store.pool_meta_save_gate.lock().await;
-                let result = tokio::time::timeout(
-                    std::time::Duration::from_secs(30),
-                    fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
-                )
-                .await
-                .expect("legacy native heal must terminate");
-                drop(state);
-                assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
-                assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
-                assert_eq!(
-                    owner_repair_identity_heads(&fixture.store).await,
-                    identities,
-                    "version {version}: no identity shard may change"
-                );
-                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-                assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
-                assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
-            }
-        },
-    )
-    .await;
-}
-
-#[tokio::test]
-#[serial_test::serial(pool_meta_version_env)]
-async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
-    temp_env::async_with_vars(
-        [
-            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
-        ],
-        async {
-            for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
-                for target_pool in [0, 1] {
-                    let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
-                    let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
-                        PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
-                        _ => panic!("initial V1 metadata must decode"),
-                    };
-                    meta.version = POOL_META_VERSION;
-                    let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
-                    assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
-                    assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
-                    let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
-                        PoolMetaReplica::Valid { canonical, .. } => canonical,
-                        _ => panic!("V1 payload must remain valid"),
-                    };
-                    let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
-                        PoolMetaReplica::Valid { canonical, .. } => canonical,
-                        _ => panic!("V2 payload must remain valid"),
-                    };
-                    assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
-                    let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
-                    // Pool two follows pool zero; pools zero and one cover both
-                    // orderings and both target formats in separate runs.
-                    let versions = [first_version, other_version, first_version];
-                    let physical = disks(&fixture.store).await;
-                    let timestamp = OffsetDateTime::now_utc();
-                    for (pool_index, pool) in physical.iter().enumerate() {
-                        let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
-                        let heads = replicas(raw, timestamp).await;
-                        for (index, disk) in pool.iter().enumerate() {
-                            fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
-                        }
-                    }
-                    let before = snapshots(&fixture.store).await;
-                    let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
-                        .await.expect("equivalent legacy pools should be selectable");
-                    assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
-                    assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
-                    let guard = owner(&fixture.store).await;
-                    let mut state = fixture.store.pool_meta_save_gate.lock().await;
-                    tokio::time::timeout(std::time::Duration::from_secs(30),
-                        fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
-                        .await.expect("mixed legacy heal must terminate")
-                        .expect("equivalent legacy metadata should heal without downgrading the format floor");
-                    drop(state);
-                    let after = snapshots(&fixture.store).await;
-                    let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-                    for (pool_index, read) in reads.iter().enumerate() {
-                        let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
-                        let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
-                        assert!(matches!(&read.replica,
-                            PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
-                                if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
-                        ), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
-                        for (index, snapshot) in after[pool_index].iter().enumerate() {
-                            codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
-                                .await.expect("every physical shard must match its selected legacy payload");
-                        }
-                        if pool_index != target_pool {
-                            assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
-                        }
-                    }
-                }
-            }
-        },
-    ).await;
-}
-
-#[tokio::test]
-#[serial_test::serial(pool_meta_version_env)]
-async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
-    temp_env::async_with_vars(
-        [
-            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
-        ],
-        async {
-            for prepare_pool in [0, 1] {
-                let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
-                let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
-                    PoolMetaReplica::Valid {
-                        meta,
-                        canonical,
-                        revision,
-                        committed: true,
-                        ..
-                    } if revision.version == POOL_META_V1_VERSION => (meta, canonical),
-                    _ => panic!("baseline V1 metadata must be committed and readable"),
-                };
-                let previous = PoolMetaCommittedCandidate {
-                    canonical: legacy_canonical,
-                    meta: legacy_meta.clone(),
-                    revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
-                };
-                let mut next = legacy_meta;
-                next.version = POOL_META_GENERATION_VERSION;
-                next.pools[0].last_update += Duration::seconds(1);
-                let revision = PoolMetaRevision {
-                    version: POOL_META_GENERATION_VERSION,
-                    cluster_id: fixture.revision.cluster_id,
-                    epoch: fixture.revision.epoch,
-                    generation: 1,
-                    transaction_id: Some(Uuid::new_v4()),
-                };
-                let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
-                    .expect("V3 PREPARE with V1 predecessor must encode");
-                assert!(matches!(decode_pool_meta_replica(pending.clone()),
-                    PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
-                let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
-                let physical = disks(&fixture.store).await;
-                for (index, disk) in physical[prepare_pool].iter().enumerate() {
-                    fs::write(object_path(disk).join("xl.meta"), &heads[index])
-                        .await
-                        .expect("install task-owned V3 PREPARE evidence");
-                }
-                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-                assert!(matches!(&reads[prepare_pool].replica,
-                    PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
-                        if raw == &pending && *actual == revision));
-                let other_pool = 1 - prepare_pool;
-                assert!(matches!(&reads[other_pool].replica,
-                    PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
-                        if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
-                let before = snapshots(&fixture.store).await;
-                let identities = owner_repair_identity_heads(&fixture.store).await;
-                let guard = owner(&fixture.store).await;
-                let mut state = fixture.store.pool_meta_save_gate.lock().await;
-                let result = tokio::time::timeout(
-                    std::time::Duration::from_secs(30),
-                    fixture
-                        .store
-                        .heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
-                )
-                .await
-                .expect("mixed V1/V3 PREPARE heal must terminate");
-                drop(state);
-                assert!(
-                    result.is_err(),
-                    "pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
-                );
-                assert_eq!(
-                    snapshots(&fixture.store).await,
-                    before,
-                    "pool {prepare_pool}: rejection must retain all physical head and backup evidence"
-                );
-                assert_eq!(
-                    owner_repair_identity_heads(&fixture.store).await,
-                    identities,
-                    "pool {prepare_pool}: rejection must not mutate identity evidence"
-                );
-            }
-        },
-    )
-    .await;
-}
-
-#[tokio::test]
-#[serial_test::serial(pool_meta_version_env)]
-async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
-    temp_env::async_with_vars(
-        [
-            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
-        ],
-        async {
-            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
-            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
-                PoolMetaReplica::Valid {
-                    meta, committed: true, ..
-                } => meta,
-                _ => panic!("baseline V1 metadata must decode"),
-            };
-            let statuses = meta
-                .pools
-                .iter()
-                .map(|pool| {
-                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
-                    LegacyPoolStatus {
-                        id: pool.id,
-                        cmd_line: pool.cmd_line.clone(),
-                        last_update: pool.last_update,
-                        decommission: None,
-                    }
-                })
-                .collect();
-            let legacy = LegacyPoolMeta {
-                version: POOL_META_V1_VERSION,
-                pools: statuses,
-                dont_save: true,
-            };
-            let mut dangerous_raw = Vec::new();
-            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
-            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
-            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
-            let old_reader: LegacyPoolMeta =
-                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
-            assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
-            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
-            let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
-                PoolMetaReplica::Valid {
-                    canonical,
-                    revision,
-                    committed: true,
-                    ..
-                } => {
-                    assert_eq!(revision.version, POOL_META_V1_VERSION);
-                    canonical
-                }
-                _ => panic!("current reader must recognize the legacy source"),
-            };
-            assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
-                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
-                    if canonical == source && revision.version == POOL_META_V1_VERSION));
-            let physical = disks(&fixture.store).await;
-            let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
-            for (index, disk) in physical[0].iter().enumerate() {
-                fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
-                    .await
-                    .expect("install legacy source on pool zero");
-            }
-            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
-            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
-            let before = snapshots(&fixture.store).await;
-            let identities = owner_repair_identity_heads(&fixture.store).await;
-            let guard = owner(&fixture.store).await;
-            let mut state = fixture.store.pool_meta_save_gate.lock().await;
-            let result = tokio::time::timeout(
-                std::time::Duration::from_secs(30),
-                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
-            )
-            .await
-            .expect("unsafe legacy heal must terminate");
-            drop(state);
-            let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
-            assert!(
-                error.to_string().contains("dont_save"),
-                "rejection must identify the unsafe legacy field: {error}"
-            );
-            assert_eq!(
-                snapshots(&fixture.store).await,
-                before,
-                "rejection must preserve every original head and backup"
-            );
-            assert_eq!(
-                owner_repair_identity_heads(&fixture.store).await,
-                identities,
-                "rejection must not write identity shards"
-            );
-            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
-            assert!(
-                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
-                "the dangerous source bytes must not be copied into the target pool"
-            );
-        },
-    )
-    .await;
-}
-
-#[tokio::test]
-#[serial_test::serial(pool_meta_version_env)]
-async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
-    temp_env::async_with_vars(
-        [
-            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
-        ],
-        async {
-            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
-            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
-                PoolMetaReplica::Valid {
-                    meta, committed: true, ..
-                } => meta,
-                _ => panic!("baseline V1 metadata must decode"),
-            };
-            let statuses = meta
-                .pools
-                .iter()
-                .map(|pool| {
-                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
-                    LegacyPoolStatus {
-                        id: pool.id,
-                        cmd_line: pool.cmd_line.clone(),
-                        last_update: pool.last_update,
-                        decommission: None,
-                    }
-                })
-                .collect();
-            let legacy = LegacyPoolMeta {
-                version: POOL_META_V1_VERSION,
-                pools: statuses,
-                dont_save: true,
-            };
-            let mut dangerous_raw = Vec::new();
-            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
-            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
-            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
-            let old_reader: LegacyPoolMeta =
-                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
-            assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
-            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
-            let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
-                PoolMetaReplica::Valid {
-                    canonical,
-                    revision,
-                    committed: true,
-                    ..
-                } => {
-                    assert_eq!(revision.version, POOL_META_V1_VERSION);
-                    canonical
-                }
-                _ => panic!("safe pool-zero source must decode"),
-            };
-            assert!(
-                matches!(decode_pool_meta_replica(dangerous_raw.clone()),
-                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
-                    if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
-                "the dangerous target must remain semantically equivalent to the safe source"
-            );
-            let physical = disks(&fixture.store).await;
-            let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
-            for (index, disk) in physical[1].iter().enumerate() {
-                fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
-                    .await
-                    .expect("install legacy dangerous target head");
-            }
-            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
-            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
-            let before = snapshots(&fixture.store).await;
-            let identities = owner_repair_identity_heads(&fixture.store).await;
-            let guard = owner(&fixture.store).await;
-            let mut state = fixture.store.pool_meta_save_gate.lock().await;
-            let result = tokio::time::timeout(
-                std::time::Duration::from_secs(30),
-                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
-            )
-            .await
-            .expect("unsafe legacy target heal must terminate");
-            drop(state);
-            let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
-            assert!(
-                error.to_string().contains("dont_save"),
-                "rejection must identify the unsafe legacy field: {error}"
-            );
-            assert_eq!(
-                snapshots(&fixture.store).await,
-                before,
-                "rejection must preserve all physical heads and backups"
-            );
-            assert_eq!(
-                owner_repair_identity_heads(&fixture.store).await,
-                identities,
-                "rejection must not write identity shards"
-            );
-            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
-            assert!(
-                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
-                "unsafe target evidence must not be overwritten by the safe source"
-            );
-        },
-    )
-    .await;
-}