fix(s3): await PUT rename completion before responding (#9229)

Preserve cached lock channels for authenticated stale boot-epoch rejections.

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
-sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
+sha256-linux=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
+sha256-darwin=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=67251cb7d9978d38e899268cac4d7898297ff7e5b5c48c9c1546dc760b93568d
-sha256-linux=673da76c2e5710c4622fe2fa32d2b297730895769487fcf1152ee1c2366f32f4
+sha256-darwin=219ac11e720b3f54dfee4d5d5e95171785ba159b2fe01db3037a8f5061c1ad32
+sha256-linux=2d34698cc8ec52d5b6c37ac612919a1d21f36cd56a1af539a7522a4eea518e26

.config/e2e-nightly-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=3814cdde8cacf4187e18188d342ed70931cc32e268e42e5bb438d565bb4d0f81
-sha256-linux=663025a80273579f95ae3f49af02c0afcbfce94df09701c360a13e607b7637db
+sha256-darwin=e84e04208f19dc0596b10d4d1b799ce43d6cc431415902f9c5458aa8232bc2d8
+sha256-linux=1f9c7add72c44d1c12ef4ac050bb7917431be657c0c90b29f70c3df56f8bbae3

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=e9dc896774122a304e2b8b90f28c82d4d4e0af8ded36a82b147b30945ec994e3
+sha256=5bdd4c2eadbc41625ae3184fff34062e000d328108436259b2b7c4fe2531ecaf

.config/make/tests.mak

@@ -28,6 +28,7 @@ script-tests: ## Run shell script tests
 	./scripts/test_build_rustfs_options.sh
 	./scripts/test_docker_runtime_timezone.sh
 	./scripts/test_entrypoint_credentials.sh
+	./scripts/test_entrypoint_directories.sh
 	./scripts/test_internode_grpc_ab_bench.sh
 	./scripts/test_object_batch_bench_enhanced.sh
 	./scripts/test_package_service_scripts.sh
@@ -70,6 +71,7 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_repro_rejoin_readiness_docker.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_validation_registry.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_validation_runner.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py

.config/nextest.toml

@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
 # processes cannot exhaust those budgets. Keep deadlines and assertions intact.
 # The short HTTP fixtures are tracked in rustfs/backlog#2714.
 [[profile.default.overrides]]
-filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
+filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
 threads-required = "num-test-threads"
 
 # This cancellation probe hashes the serving executable before acquiring its
@@ -548,7 +548,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
 test-group = 'embedded-test-ports'
 
 [[profile.ci.overrides]]
-filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
+filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
 threads-required = "num-test-threads"
 
 # Preserve the cancellation probe's startup budget and assertions in CI too.
@@ -701,7 +701,7 @@ test-group = 'ecstore-serial-flaky'
 [profile.e2e-smoke]
 default-filter = """
     package(e2e_test) & (
-        test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
+        test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
         | test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
         | test(/^reliant::lifecycle::/)
         | test(/^conditional_delete_objects_test::/)
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
 # ---------------------------------------------------------------------------
 # e2e-distributed profile — 4-node 4-disk Actions suite
 # ---------------------------------------------------------------------------
-# Storage-sensitive PR / nightly / dispatch lane owned by
+# Nightly and manual-dispatch lane owned by
 # .github/workflows/e2e-distributed.yml.
 # Each case starts four rustfs processes (and for site replication, two
 # clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
@@ -976,6 +976,12 @@ test-group = 'e2e-vault'
 filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
 threads-required = "num-test-threads"
 
+# The mc fixture uploads hundreds of objects concurrently before measuring listing.
+# Reserve the runner so other server fixtures do not starve its disk operations.
+[[profile.e2e-full.overrides]]
+filter = 'package(e2e_test) & test(=mc_mirror_small_bucket_test::test_mc_mirror_small_bucket_completes_without_list_timeout)'
+threads-required = "num-test-threads"
+
 # Fixed, small contract for every executable server change. These existing
 # scenarios exercise write/read, anonymous denial and retained bytes on restart.
 [profile.core-contract]

.config/release-validation-plan.json

@@ -0,0 +1,77 @@
+{
+  "schema_version": 1,
+  "plan_id": "rustfs-release-validation",
+  "description": "Versioned certification plan for every public RustFS release exit (rustfs/backlog#2662, R01). preview, prerelease and stable all require this explicit plan. Scripts/release/release_gate.py fail-closes publication on any missing, duplicate, stale, substituted or non-PASS required item.",
+  "release_types": ["preview", "prerelease", "stable"],
+  "targets": [
+    {"os": "linux", "arch": "x86_64", "abi": "musl"},
+    {"os": "linux", "arch": "aarch64", "abi": "musl"},
+    {"os": "linux", "arch": "x86_64", "abi": "gnu"},
+    {"os": "linux", "arch": "aarch64", "abi": "gnu"},
+    {"os": "macos", "arch": "aarch64", "abi": "none"},
+    {"os": "windows", "arch": "x86_64", "abi": "msvc"}
+  ],
+  "targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
+  "features": ["pyroscope"],
+  "trusted_workflows": [
+    {"repository": "rustfs/s3chaos", "workflow_id": "release-gate.yml"}
+  ],
+  "trusted_workflows_basis": "The shared evidence envelope producer lives in rustfs/s3chaos (G01, fault/release_gate.rs) and its plan/live split workflow is .github/workflows/release-gate.yml there. Only results from these identities may enter a release certification; PR-uploaded JSON never gains release authority. R02 must extend this list when additional verifier workflows land, never widen it to untrusted sources.",
+  "lanes": [
+    {
+      "lane": "identity",
+      "harness_repository": "rustfs/s3chaos",
+      "harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
+      "harness_basis": "Fixed G01 evidence-envelope producer revision (rustfs/s3chaos PR #102 head recorded in rustfs/backlog#2658). R02 re-pins this when the live verifier ships.",
+      "timeout_seconds": 1800,
+      "environment_capabilities": ["process-exec"],
+      "cases": ["artifact-identity", "startup"],
+      "applies_to": {
+        "release_types": ["preview", "prerelease", "stable"],
+        "platforms": "all"
+      },
+      "applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
+    },
+    {
+      "lane": "upgrade",
+      "harness_repository": "rustfs/s3chaos",
+      "harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
+      "harness_basis": "Same fixed envelope producer as the identity lane; the workflow SHA and the product build SHA are bound separately and need not be equal.",
+      "timeout_seconds": 5400,
+      "environment_capabilities": ["process-exec", "previous-release-artifact"],
+      "cases": [
+        "direct-upgrade",
+        "mixed-version-upgrade",
+        "bucket-config-upgrade",
+        "bucket-config-rollback",
+        "multipart-layout-upgrade",
+        "multipart-layout-baseline"
+      ],
+      "cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
+      "applies_to": {
+        "release_types": ["prerelease", "stable"],
+        "platforms": [{"os": "linux", "arch": "x86_64", "abi": "gnu"}]
+      },
+      "applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
+    }
+  ],
+  "upgrade_sources": {
+    "basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
+    "sources": [
+      {
+        "version": "1.0.1",
+        "role": "N-1",
+        "artifact_name": "rustfs-linux-x86_64-gnu-v1.0.1.zip",
+        "artifact_sha256": "9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5"
+      },
+      {
+        "version": "1.0.0-rc.5",
+        "role": "N-2",
+        "artifact_name": "rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip",
+        "artifact_sha256": "3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d"
+      }
+    ],
+    "rollback_supported": false,
+    "rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
+  }
+}

.docker/README.md

@@ -22,9 +22,9 @@ The following files in the project root are essential for Docker operations:
 | File | Description | Usage |
 | :--- | :--- | :--- |
 | **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
-| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
-| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
-| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |
+| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads verified RustFS and `rc` binaries from GitHub Releases and bundles `/etc/rustfs/bootstrap`. | `docker build -t rustfs:latest .` |
+| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Includes the bundled bootstrap configuration. | `docker build -f Dockerfile.glibc .` |
+| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code and bundles `rc` and the default bootstrap configuration. | `docker build -f Dockerfile.source .` |
 
 ### Docker Compose Configurations
 
@@ -110,6 +110,36 @@ docker compose up -d
 
 ---
 
+## Docker Compose environment values
+
+Use map syntax when quoting environment values in YAML:
+
+```yaml
+services:
+  rustfs:
+    image: rustfs/rustfs:latest
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    volumes:
+      - ./data:/data
+      - ./logs:/var/log/rustfs
+    environment:
+      RUSTFS_ADDRESS: ":9000"
+      RUSTFS_CONSOLE_ADDRESS: ":9001"
+      RUSTFS_CONSOLE_ENABLE: "true"
+      RUSTFS_OBS_LOGGER_LEVEL: error
+      RUSTFS_OBS_LOG_DIRECTORY: "/var/log/rustfs/"
+      RUSTFS_ACCESS_KEY: "${RUSTFS_ACCESS_KEY:?Set RUSTFS_ACCESS_KEY}"
+      RUSTFS_SECRET_KEY: "${RUSTFS_SECRET_KEY:?Set RUSTFS_SECRET_KEY}"
+```
+
+Create the host directories and grant the container user write access before starting this example; official images run as UID/GID `10001:10001`. Set both credential variables to your own values, then run `docker compose config` to inspect the resolved configuration locally before `docker compose up -d`. The resolved output includes credentials, so do not share it publicly.
+
+With list syntax, use `- RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/` or quote the whole item as `- "RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/"`. Writing `- RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"` passes the inner quotes literally, making this a relative path beginning with `"`, which can fail with `mkdir: cannot create directory '"': Permission denied`. This also applies to addresses and credentials: fix every affected list item, rather than only the log directory. The entrypoint preserves configured values, including literal quotes in valid paths and secrets.
+
+---
+
 ## 🔧 Common Operations
 
 ### Cleaning Up

.docker/observability/otel-collector-config.yaml

@@ -28,6 +28,15 @@ processors:
     check_interval: 1s
     limit_mib: 1024
     spike_limit_mib: 256
+  transform/metrics:
+    error_mode: ignore
+    metric_statements:
+      # RustFS process metrics already expose process_pid as a metric label.
+      # Remove only the duplicate datapoint label and keep the shared resource
+      # process.pid available to sibling metrics and lock histograms.
+      - context: datapoint
+        statements:
+          - delete_key(attributes, "process_pid") where IsMatch(metric.name, "^rustfs_system_process_")
   transform/logs:
     error_mode: ignore
     log_statements:
@@ -112,7 +121,7 @@ service:
       exporters: [ otlp/tempo, otlp/jaeger ]
     metrics:
       receivers: [ otlp ]
-      processors: [ batch ]
+      processors: [ transform/metrics, memory_limiter, batch ]
       exporters: [ prometheus ]
     logs:
       receivers: [ otlp ]

.docker/rejoin-readiness.Dockerfile

@@ -0,0 +1,40 @@
+# syntax=docker/dockerfile:1.6
+# Copyright 2026 RustFS Team
+# Licensed under the Apache License, Version 2.0.
+
+# A compatible native runtime with Python 3.11+, getent and stat is required by
+# the isolated fixture's staging/provenance checks. This is a test image.
+ARG RUNTIME_BASE=python:3.13-slim-trixie
+FROM ${RUNTIME_BASE}
+
+ARG SOURCE_REVISION
+ARG BINARY_SHA256
+ARG BUILD_PROFILE=dev
+COPY rustfs /usr/bin/rustfs
+RUN python3 - "$SOURCE_REVISION" "$BINARY_SHA256" "$BUILD_PROFILE" <<'PY'
+import hashlib
+import os
+import re
+import shutil
+import sys
+
+source, digest, profile = sys.argv[1:]
+if not re.fullmatch(r"[0-9a-f]{40}", source):
+    raise SystemExit("invalid source revision")
+if not re.fullmatch(r"[0-9a-f]{64}", digest):
+    raise SystemExit("invalid binary digest")
+if profile not in ("dev", "release"):
+    raise SystemExit("invalid build profile")
+with open("/usr/bin/rustfs", "rb") as binary:
+    if hashlib.file_digest(binary, "sha256").hexdigest() != digest:
+        raise SystemExit("binary digest mismatch")
+for tool in ("getent", "stat"):
+    if shutil.which(tool) is None:
+        raise SystemExit(f"missing runtime tool: {tool}")
+os.chmod("/usr/bin/rustfs", 0o755)
+PY
+LABEL io.rustfs.repro.source_revision="${SOURCE_REVISION}" \
+      io.rustfs.repro.binary_sha256="${BINARY_SHA256}" \
+      io.rustfs.repro.build_profile="${BUILD_PROFILE}"
+USER 10001:10001
+ENTRYPOINT ["/usr/bin/rustfs"]

.github/dependabot.yml

@@ -32,6 +32,8 @@ updates:
       - "overtrue"
       - "majinghe"
     ignore:
+      - dependency-name: "pulsar"
+        versions: [ "6.9.x" ]
       - dependency-name: "object_store"
         versions: [ "0.13.x" ]
       - dependency-name: "libunftp"

.github/scheduled-validations.json

@@ -296,6 +296,32 @@
       "origin": "baseline",
       "alert_contract": "scheduled-validation"
     },
+    {
+      "workflow": ".github/workflows/rustfs-validation-sweep.yml",
+      "name": "RustFS Validation Sweep",
+      "roles": [
+        "periodic",
+        "manual"
+      ],
+      "events": [
+        "schedule",
+        "workflow_dispatch"
+      ],
+      "cadence": "nightly",
+      "timezone": "UTC",
+      "schedule": [
+        "30 19 * * *"
+      ],
+      "owner": "@overtrue",
+      "policy": "active",
+      "monitor": "workflow",
+      "max_age_hours": 60,
+      "result_owner": "rustfs-validation-sweep.yml",
+      "disposition": "canonical validation cases executed in rotating batches",
+      "failure_owner": "workflow",
+      "origin": "baseline",
+      "alert_contract": "scheduled-validation"
+    },
     {
       "workflow": ".github/workflows/btf-prototype-reference.yml",
       "name": "BTF prototype reference",
@@ -743,12 +769,10 @@
       "workflow": ".github/workflows/e2e-distributed.yml",
       "name": "e2e-distributed",
       "roles": [
-        "pr",
         "periodic",
         "manual"
       ],
       "events": [
-        "pull_request",
         "workflow_dispatch",
         "schedule"
       ],
@@ -823,12 +847,11 @@
       "workflow": ".github/workflows/e2e-upgrade.yml",
       "name": "Upgrade Compatibility",
       "roles": [
-        "pr",
         "periodic",
-        "manual"
+        "manual",
+        "release"
       ],
       "events": [
-        "pull_request",
         "push",
         "schedule",
         "workflow_dispatch"

.github/workflows/build.yml

@@ -814,13 +814,18 @@ jobs:
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
-          # Determine upload path based on build type
+          # Determine upload path based on build type. Release and prerelease
+          # builds only stage their bytes here: the official
+          # artifacts/rustfs/release/ path is written solely by the
+          # certification-gated promote-release-artifacts job, from these same
+          # staged objects. Development builds keep the uncertified dev channel
+          # and never touch official or latest channels.
           if [[ "$BUILD_TYPE" == "development" ]]; then
             R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/dev/"
             echo "📤 Uploading development build to R2 dev directory"
           else
-            R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
-            echo "📤 Uploading release build to R2 release directory"
+            R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
+            echo "📤 Staging release build for certification-gated promotion"
           fi
 
           # Upload all rustfs zip files to R2 using S3-compatible API
@@ -869,6 +874,221 @@ jobs:
             echo "This run does not trigger Docker publishing."
           fi
 
+  # Aggregate the parent candidate batch. This job always collects results and
+  # retains its evidence, including on failure, and every publication job
+  # below requires its success. Missing, stale, substituted or non-PASS
+  # evidence fails closed here: only the trusted verification sources
+  # (rustfs/s3chaos release-gate envelopes, rustfs/backlog#2658) can satisfy
+  # it, so a PR-uploaded JSON set never gains release authority.
+  release-certification:
+    name: Release Certification
+    needs: [ build-check, build-rustfs ]
+    if: ${{ !cancelled() && needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') }}
+    runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download all build artifacts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: ./artifacts
+          pattern: rustfs-*
+          merge-multiple: true
+
+      - name: Write candidate and collect trusted validation results
+        env:
+          # Cross-repo read of the trusted verification run. Falls back to the
+          # workflow token, which cannot read rustfs/s3chaos; a missing token
+          # therefore fail-closes instead of certifying.
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          PRODUCT_COMMIT: ${{ github.sha }}
+          VERSION: ${{ needs.build-check.outputs.version }}
+          BUILD_TYPE: ${{ needs.build-check.outputs.build_type }}
+          FEATURES: ${{ needs.build-check.outputs.build_type == 'development' && '' || 'pyroscope' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          mkdir -p results release-gate-output
+          PLAN_SHA256=$(python3 -c "import hashlib,json,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('.config/release-validation-plan.json'))))")
+
+          python3 - "$PLAN_SHA256" > candidate.json <<'PY'
+          import hashlib, json, os, sys, zipfile
+
+          plan_sha256 = sys.argv[1]
+          profile = {"release": "stable", "prerelease": "prerelease", "preview": "preview"}.get(os.environ["BUILD_TYPE"])
+          if profile is None:
+              raise SystemExit("release certification only covers preview, prerelease and stable builds")
+
+          def sha256(path):
+              digest = hashlib.sha256()
+              with open(path, "rb") as handle:
+                  for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+                      digest.update(chunk)
+              return digest.hexdigest()
+
+          deliverables = []
+          for name in sorted(os.listdir("artifacts")):
+              if not name.startswith("rustfs-") or not name.endswith(".zip"):
+                  continue
+              path = os.path.join("artifacts", name)
+              members = zipfile.ZipFile(path).namelist()
+              binary_name = "rustfs.exe" if "rustfs.exe" in members else "rustfs"
+              with zipfile.ZipFile(path) as archive:
+                  binary_bytes = archive.read(binary_name)
+              import tempfile
+              with tempfile.NamedTemporaryFile() as handle:
+                  handle.write(binary_bytes)
+                  handle.flush()
+                  binary_sha = sha256(handle.name)
+              deliverables.append({
+                  "kind": "zip",
+                  "name": name,
+                  "location": "https://github.com/rustfs/rustfs/releases/download/{}/{}".format(os.environ["VERSION"], name),
+                  "sha256": sha256(path),
+                  "size": os.path.getsize(path),
+                  "binary_sha256": binary_sha,
+              })
+          if not deliverables:
+              raise SystemExit("no built deliverables found to certify")
+          candidate = {
+              "schema_version": 1,
+              "product_commit": os.environ["PRODUCT_COMMIT"],
+              "version": os.environ["VERSION"],
+              "release_profile": profile,
+              "features": [f for f in os.environ.get("FEATURES", "").split(",") if f],
+              "plan_sha256": plan_sha256,
+              "validation_batch": {"run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"])},
+              "deliverables": deliverables,
+          }
+          json.dump(candidate, sys.stdout, indent=2, sort_keys=True)
+          print()
+          PY
+
+          # Trusted verification evidence only: the release-gate envelopes of
+          # rustfs/s3chaos for this exact product commit. Absence blocks below.
+          TRUSTED_REPO="rustfs/s3chaos"
+          TRUSTED_WORKFLOW="release-gate.yml"
+          TRUSTED_RUN=$(gh api "repos/${TRUSTED_REPO}/actions/workflows/${TRUSTED_WORKFLOW}/runs?head_sha=${PRODUCT_COMMIT}&status=success&per_page=1" --jq '.workflow_runs[0].id // empty' || true)
+          if [[ -z "$TRUSTED_RUN" ]]; then
+            echo "❌ No trusted verification run of ${TRUSTED_REPO}/${TRUSTED_WORKFLOW} for ${PRODUCT_COMMIT}; refusing to certify" >&2
+          else
+            gh run download "$TRUSTED_RUN" -R "$TRUSTED_REPO" -n release-validation-evidence -D results || \
+              echo "❌ Trusted verification run ${TRUSTED_RUN} published no release-validation-evidence artifact" >&2
+          fi
+
+          python3 scripts/release/release_gate.py validate \
+            --plan .config/release-validation-plan.json \
+            --candidate candidate.json \
+            --results results \
+            --output release-gate-output
+
+      - name: Upload certification evidence
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: release-certification
+          path: |
+            candidate.json
+            results/
+            release-gate-output/
+          if-no-files-found: warn
+          retention-days: 30
+
+  # The only writer of the official R2 release path. Promotion re-checks the
+  # exact bytes about to be published against the passed certification list
+  # and copies the staged objects byte-for-byte: no rebuild-after-verify.
+  promote-release-artifacts:
+    name: Promote Release Artifacts
+    needs: [ build-check, build-rustfs, release-certification ]
+    # Target-channel lock for the official R2 release path.
+    concurrency:
+      group: release-channel-r2-release
+      cancel-in-progress: false
+    if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    env:
+      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+      R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+      R2_BUCKET: ${{ secrets.R2_BUCKET }}
+      AWS_EC2_METADATA_DISABLED: true
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download build and certification artifacts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: ./artifacts
+          pattern: rustfs-*
+          merge-multiple: true
+
+      - name: Download certification evidence
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: release-certification
+          path: ./certification
+
+      - name: Verify and promote staged release artifacts
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
+            echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
+            exit 1
+          fi
+
+          for zip_file in artifacts/rustfs-*.zip; do
+            [[ -f "$zip_file" ]] || continue
+            name="$(basename "$zip_file")"
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$zip_file" \
+              --artifact-name "$name"
+          done
+
+          if ! command -v aws >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y awscli
+          fi
+          export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
+          export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
+          export AWS_DEFAULT_REGION="auto"
+
+          STAGING_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
+          RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
+          for zip_file in artifacts/rustfs-*.zip; do
+            [[ -f "$zip_file" ]] || continue
+            name="$(basename "$zip_file")"
+            staged="$(mktemp)"
+            aws s3 cp "${STAGING_PATH}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
+            built_sha="$(sha256sum "$zip_file" | awk '{print $1}')"
+            if [[ "$staged_sha" != "$built_sha" ]]; then
+              echo "❌ Staged object ${name} does not match the verified bytes" >&2
+              exit 1
+            fi
+            aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            rm -f "$staged"
+            echo "✅ Promoted ${name} (${built_sha})"
+          done
+
   # Create GitHub Release for every valid release tag, including previews
   create-release:
     name: Create GitHub Release
@@ -1028,6 +1248,10 @@ jobs:
   update-latest-version:
     name: Update Latest Version
     needs: [ build-check, publish-release ]
+    # Target-channel lock for the latest.json version pointer.
+    concurrency:
+      group: release-channel-oss-latest
+      cancel-in-progress: false
     if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -1068,6 +1292,45 @@ jobs:
           rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
           chmod +x "$OSSUTIL_BIN"
 
+          # Candidate/pointer ordering: an older candidate must never overwrite
+          # a newer pointer, even if its promotion job finishes later.
+          CURRENT_LATEST="${RUNNER_TEMP}/current-latest.json"
+          if "$OSSUTIL_BIN" cp oss://rustfs-version/latest.json "$CURRENT_LATEST" 2>/dev/null; then
+            python3 - "$CURRENT_LATEST" "$VERSION" <<'PY'
+          import json
+          import re
+          import sys
+
+          def key(version):
+              match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$", version)
+              if not match:
+                  raise SystemExit(3)
+              core = tuple(int(part) for part in match.groups()[:3])
+              prerelease = match.group(4)
+              if prerelease is None:
+                  return core + (1,)
+              return core + (0, tuple(int(p) if p.isdigit() else p for p in re.split(r"[.-]", prerelease)))
+
+          try:
+              current = json.load(open(sys.argv[1], encoding="utf-8")).get("version", "")
+              proposed = sys.argv[2]
+              if current and key(current) > key(proposed):
+                  raise SystemExit(1)
+          except SystemExit:
+              raise
+          except (OSError, ValueError, TypeError):
+              raise SystemExit(3)
+          PY
+            guard_status=$?
+            if [[ "$guard_status" -eq 1 ]]; then
+              echo "Current latest.json points at a newer version; refusing to downgrade" >&2
+              exit 0
+            elif [[ "$guard_status" -ne 0 ]]; then
+              echo "Existing latest.json is unparsable; refusing to overwrite it blindly" >&2
+              exit 1
+            fi
+          fi
+
           # Create latest.json
           cat > latest.json << EOF
           {
@@ -1084,16 +1347,59 @@ jobs:
 
           echo "✅ Updated latest.json for ${RELEASE_TYPE} release $VERSION"
 
-  # Publish release (remove draft status)
+  # Publish release (remove draft status). Requires the release-certification
+  # aggregate: without a complete legal success there is no public write.
   publish-release:
     name: Publish Release
-    needs: [ build-check, create-release, upload-release-assets ]
+    needs: [ build-check, create-release, upload-release-assets, release-certification ]
+    # Target-channel lock: concurrent releases serialize on this channel so an
+    # older candidate cannot interleave its publish with a newer one.
+    concurrency:
+      group: release-channel-github-release
+      cancel-in-progress: false
     if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
     runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
     timeout-minutes: 30
     permissions:
       contents: write
+      actions: read
     steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download certification evidence
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: release-certification
+          path: ./certification
+
+      - name: Verify release assets against certification
+        env:
+          GH_TOKEN: ${{ github.token }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          TAG="${{ needs.build-check.outputs.version }}"
+          mkdir -p "$RUNNER_TEMP/promotion-assets"
+          gh release download "$TAG" --dir "$RUNNER_TEMP/promotion-assets" --pattern '*.zip'
+          found=0
+          for zip_file in "$RUNNER_TEMP"/promotion-assets/*.zip; do
+            [[ -f "$zip_file" ]] || continue
+            found=1
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$zip_file" \
+              --artifact-name "$(basename "$zip_file")"
+          done
+          if [[ "$found" -ne 1 ]]; then
+            echo "No release zip assets to verify against the certification" >&2
+            exit 1
+          fi
+
       - name: Publish release
         env:
           GH_TOKEN: ${{ github.token }}

.github/workflows/cache-warm.yml

@@ -96,7 +96,7 @@ env:
   RUST_BACKTRACE: 1
 
 jobs:
-  # Readers: test-and-lint (workspace and ILM), build-rustfs-debug-binary,
+  # Readers: workspace-lint-and-build, workspace-tests (workspace and ILM), build-rustfs-debug-binary,
   # e2e-tests, e2e-full.
   warm-ci-dev:
     name: Warm ci-dev
@@ -146,7 +146,7 @@ jobs:
           if-no-files-found: error
 
       # --all-targets covers the test binaries nextest builds, including
-      # e2e_test, which test-and-lint's own run excludes. The second build adds
+      # e2e_test, which the workspace run excludes. The second build adds
       # the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
       # and that no lint lane enables.
       - name: Build ci-dev superset

.github/workflows/ci.yml

@@ -113,7 +113,7 @@ jobs:
             printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
             echo "Base CI policy unavailable; running the full matrix."
           fi
-          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^rehearse_shards=' "$GITHUB_OUTPUT"; then
+          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
             printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
             echo "Base policy does not cover this workflow contract; retaining the full matrix."
           fi
@@ -160,7 +160,7 @@ jobs:
       - name: Run shared quick checks
         uses: ./.github/actions/quick-checks
 
-  test-and-lint:
+  workspace-lint-and-build:
     name: Workspace Lint and Build
     if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
     needs: [ quick-checks, classify-changes ]
@@ -193,13 +193,13 @@ jobs:
 
       - name: Prepare test evidence
         run: |
-          mkdir -p artifacts/test-and-lint
+          mkdir -p artifacts/workspace
           {
             echo "run_id=${GITHUB_RUN_ID}"
             echo "job=${GITHUB_JOB}"
             echo "runner=${RUNNER_NAME}"
             echo "started_at=$(date --utc --iso-8601=seconds)"
-          } > artifacts/test-and-lint/run-metadata.txt
+          } > artifacts/workspace/run-metadata.txt
 
       # Clippy gates every event, pull requests included. When it ran only after
       # merge, a lint regression merged freely and then failed every main run
@@ -215,7 +215,8 @@ jobs:
 
       - name: Archive workspace test binaries
         env:
-          CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}
+          # Workspace test compilation has high per-process memory use.
+          CARGO_BUILD_JOBS: "2"
         run: |
           ./scripts/ci/resource_sampler.sh start workspace-archive
           trap './scripts/ci/resource_sampler.sh stop' EXIT
@@ -233,11 +234,11 @@ jobs:
 
       - name: Run documentation tests
         run: |
-          mkdir -p artifacts/test-and-lint
+          mkdir -p artifacts/workspace
           set +e
           timeout --verbose --signal=TERM --kill-after=30s 15m \
             python3 scripts/ci_test_artifact.py cargo doc --configuration workspace \
-            2>&1 | tee artifacts/test-and-lint/doctest.log
+            2>&1 | tee artifacts/workspace/doctest.log
           status=${PIPESTATUS[0]}
           {
             echo "command=cargo test --all --exclude e2e_test --doc"
@@ -246,7 +247,7 @@ jobs:
             echo
             echo "Remaining test-related processes:"
             pgrep -af 'cargo|rustdoc|target/.*/deps/' || true
-          } > artifacts/test-and-lint/doctest-diagnostics.txt
+          } > artifacts/workspace/doctest-diagnostics.txt
           exit "${status}"
 
       - name: Upload test reports and diagnostics
@@ -255,7 +256,7 @@ jobs:
         with:
           name: junit-test-and-lint-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
-            artifacts/test-and-lint
+            artifacts/workspace
           retention-days: 3
           if-no-files-found: error
 
@@ -268,7 +269,7 @@ jobs:
 
   workspace-tests:
     name: Workspace tests
-    needs: [ test-and-lint, classify-changes ]
+    needs: [ workspace-lint-and-build, classify-changes ]
     # The archive retains compile-time fixture paths and the large test set
     # requires the same runner image until the hosted-runtime pilot passes.
     runs-on: sm-standard-4
@@ -293,14 +294,14 @@ jobs:
 
       - name: Extract and list workspace tests
         run: |
-          mkdir -p artifacts/test-and-lint
+          mkdir -p artifacts/workspace
           cargo nextest list --profile ci --archive-file "$RUNNER_TEMP/workspace-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json \
-            > artifacts/test-and-lint/core-test-listing.json
-          python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json
+            > artifacts/workspace/core-test-listing.json
+          python3 scripts/check_test_wiring.py --check-core artifacts/workspace/core-test-listing.json
 
       - name: Run nextest tests
         run: |
-          mkdir -p artifacts/test-and-lint
+          mkdir -p artifacts/workspace
           rm -f target/nextest/ci/junit.xml
           ./scripts/ci/resource_sampler.sh start nextest
           trap './scripts/ci/resource_sampler.sh stop' EXIT
@@ -309,7 +310,7 @@ jobs:
             cargo nextest run --profile ci --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json \
             --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
             --status-level all --final-status-level all \
-            2>&1 | tee artifacts/test-and-lint/nextest.log
+            2>&1 | tee artifacts/workspace/nextest.log
           status=${PIPESTATUS[0]}
           {
             echo "command=workspace archive execution"
@@ -321,13 +322,13 @@ jobs:
             echo
             echo "Kernel OOM / kill events:"
             dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
-          } > artifacts/test-and-lint/nextest-diagnostics.txt
+          } > artifacts/workspace/nextest-diagnostics.txt
           exit "${status}"
 
       - name: Verify workspace execution completeness
         run: |
-          python3 scripts/ci_test_artifact.py evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
-          ./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
+          python3 scripts/ci_test_artifact.py evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
+          ./scripts/check_migration_gate_count.sh evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
 
       - name: Run ignored ILM integration tests serially
         if: needs.classify-changes.outputs.mode == 'full'
@@ -358,7 +359,7 @@ jobs:
           name: workspace-test-evidence-${{ github.run_attempt }}
           path: |
             target/nextest/ci/junit.xml
-            artifacts/test-and-lint
+            artifacts/workspace
             artifacts/ilm-integration
           retention-days: 3
           if-no-files-found: error
@@ -368,7 +369,7 @@ jobs:
         if: needs.classify-changes.outputs.rehearse_shards == 'true'
         run: |
           python3 scripts/ci_test_shards.py prepare \
-            --listing artifacts/test-and-lint/core-test-listing.json \
+            --listing artifacts/workspace/core-test-listing.json \
             --junit target/nextest/ci/junit.xml --output artifacts/workspace-shard-plan
       - name: Upload same-run shard plan
         if: needs.classify-changes.outputs.rehearse_shards == 'true'
@@ -449,7 +450,7 @@ jobs:
 
   connect-feature-boundaries:
     name: Connect Feature Boundaries
-    if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
@@ -497,13 +498,72 @@ jobs:
       - name: Check offline enrollment E2E root boundary
         run: ./scripts/check_offline_enrollment_e2e.sh
 
+  fips-provider:
+    name: FIPS provider boundaries
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
+    needs: [ quick-checks, classify-changes ]
+    # This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: [default, fips]
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      CARGO_BUILD_JOBS: "2"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-fips-provider
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
+
+      - name: Install FIPS build tools
+        if: matrix.mode == 'fips'
+        run: sudo apt-get install -y cmake golang clang libclang-dev
+
+      - name: Verify provider installation
+        shell: bash
+        run: |
+          args=()
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            args+=(--features fips)
+          fi
+          cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
+          cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
+          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
+          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
+            cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
+            cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
+          fi
+          tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
+          fi
+          for test_name in "${tests[@]}"; do
+            cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
+          done
+          cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
+
   test-and-lint-rio-v2:
     name: Test and Lint (rio-v2)
-    # Alternate feature configurations run on the nightly schedule, merge
-    # groups and manual dispatch. The default workspace remains the PR gate and
-    # the per-push main baseline: repeating four feature builds on every merge
-    # bought nothing the nightly run does not report within a day.
-    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
+    # Specialized boundary, hardware and alternate-feature lanes run only on
+    # the nightly schedule and manual dispatch. They do not gate a merge: the
+    # nightly run reports a regression within a day, and repeating these builds
+    # on every PR revision and merge group bought no earlier signal worth the
+    # runner occupancy. Workspace lint and tests, the core contract, the debug
+    # build, E2E smoke and the S3 lanes remain the pull-request gate.
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
@@ -540,12 +600,12 @@ jobs:
           # --profile ci so the quarantine list (and its junit flaky markers)
           # covers this leg too; the default profile is the local no-retry
           # profile and silently ignored quarantined flakes here (rustfs#6703).
-          mkdir -p artifacts/test-and-lint
+          mkdir -p artifacts/workspace
           rm -f target/nextest/ci/junit.xml
           ./scripts/ci/resource_sampler.sh start nextest-rio-v2
           trap './scripts/ci/resource_sampler.sh stop' EXIT
           cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
-            2>&1 | tee artifacts/test-and-lint/nextest.log
+            2>&1 | tee artifacts/workspace/nextest.log
           cargo test -p rustfs --doc --features rio-v2
 
       - name: Upload rio-v2 test reports and diagnostics
@@ -555,14 +615,14 @@ jobs:
           name: junit-test-and-lint-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
             target/nextest/ci/junit.xml
-            artifacts/test-and-lint
+            artifacts/workspace
           retention-days: 3
           if-no-files-found: error
 
   test-and-lint-protocols:
     name: "Test and Lint (${{ matrix.features.name }})"
     # Same cadence as test-and-lint-rio-v2 above.
-    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
@@ -631,7 +691,7 @@ jobs:
           path: |
             target/nextest/ci/junit.xml
             artifacts/protocol-tests
-            artifacts/test-and-lint
+            artifacts/workspace
           retention-days: 3
           if-no-files-found: warn
 
@@ -783,7 +843,7 @@ jobs:
     # job had neither, so each closed/merged PR really ran the whole io_uring
     # suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
     # 30662728539) and kept the cancellation run in progress for minutes.
-    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
     needs: [ quick-checks, classify-changes ]
     # GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
     # a container, applies no seccomp filter that would block io_uring_setup — so
@@ -978,7 +1038,7 @@ jobs:
           path: |
             ${{ runner.temp }}/rustfs-e2e-smoke-logs/
             ${{ runner.temp }}/rustfs-e2e-smoke-list.json
-            artifacts/test-and-lint
+            artifacts/workspace
           if-no-files-found: warn
 
       - name: Upload e2e smoke JUnit report
@@ -1389,12 +1449,13 @@ jobs:
       - classify-changes
       - typos
       - quick-checks
-      - test-and-lint
+      - workspace-lint-and-build
       - workspace-tests
       - workspace-shards
       - workspace-shard-evidence
       - core-contract
       - connect-feature-boundaries
+      - fips-provider
       - test-and-lint-rio-v2
       - test-and-lint-protocols
       - build-rustfs-debug-binary
@@ -1422,10 +1483,11 @@ jobs:
     needs:
       - classify-changes
       - connect-feature-boundaries
+      - fips-provider
       - required-checks
       - typos
       - quick-checks
-      - test-and-lint
+      - workspace-lint-and-build
       - workspace-tests
       - core-contract
       - test-and-lint-rio-v2

.github/workflows/cla.yml

@@ -95,7 +95,7 @@ jobs:
 
       - name: Run CLA Bot
         if: github.event_name != 'merge_group'
-        uses: overtrue/cla-bot@5e5929846a60cfb42b16d0d38e135ef67d94fc88 # v0.0.9
+        uses: overtrue/cla-bot@4bd1387c19691168c42fcf5edfa17b15819b2ab3 # v0.1.1
         with:
           github-token: ${{ github.token }}
           registry-token: ${{ steps.registry-token.outputs.token }}

.github/workflows/docker.yml

@@ -329,13 +329,75 @@ jobs:
           echo "  - Create latest: $create_latest"
           echo "  - Source ref: $source_ref"
 
+  # Consume the trusted release certification for the exact version this
+  # workflow wraps and verify the certified input digests. A manual dispatch
+  # without that certification fails here and pushes no official tag; only
+  # results from the trusted verification sources carry release authority.
+  release-certification:
+    name: Release Certification
+    needs: build-check
+    if: needs.build-check.outputs.should_build == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+          ref: ${{ needs.build-check.outputs.source_ref }}
+
+      - name: Download and verify release certification
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
+          VERSION: ${{ needs.build-check.outputs.version }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p certification
+          run_id="$TRIGGERING_RUN_ID"
+          if [[ -z "$run_id" ]]; then
+            run_id="$(gh api --method GET \
+              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$VERSION" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' || true)"
+          fi
+          if [[ -z "$run_id" ]]; then
+            echo "No certified Build and Release run found for ${VERSION}; failing closed" >&2
+            exit 1
+          fi
+          if ! gh run download "$run_id" -n release-certification -D certification; then
+            echo "Build run ${run_id} published no release-certification artifact; failing closed" >&2
+            exit 1
+          fi
+          python3 scripts/release/release_gate.py verify-promotion \
+            --plan .config/release-validation-plan.json \
+            --candidate certification/candidate.json \
+            --summary certification/release-gate-output \
+            --artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
+            --artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
+
+          # Input digest binding: every zip this image wraps must be the
+          # certified byte-for-byte content, never a mutable latest pointer.
+          while IFS=$'\t' read -r name location expected; do
+            echo "Verifying certified input ${name}"
+            actual="$(curl -fsSL "$location" | sha256sum | awk '{print $1}')"
+            if [[ "$actual" != "$expected" ]]; then
+              echo "Certified input ${name} does not match its recorded digest" >&2
+              exit 1
+            fi
+          done < <(jq -r '.deliverables[] | select(.name | contains("linux")) | [.name, .location, .sha256] | @tsv' certification/candidate.json)
+
   # Build multi-arch Docker images
   # Strategy: Build images using pre-built binaries from dl.rustfs.com
   # Supports both release and dev channel binaries based on build context
   # Only runs when should_build is true (which includes workflow success check)
   build-docker:
     name: Build Docker Images
-    needs: build-check
+    needs: [ build-check, release-certification ]
     if: needs.build-check.outputs.should_build == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -443,6 +505,14 @@ jobs:
           # Output tags
           echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
 
+          # Staging identity for this run's promotion: official tags are only
+          # created after the staged image verifies against the certification,
+          # and they point at this same digest (no rebuild-after-verify).
+          STAGING_TAG="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${VARIANT_SUFFIX}"
+          STAGING_TAGS="${{ env.REGISTRY_DOCKERHUB }}:${STAGING_TAG},${{ env.REGISTRY_GHCR }}:${STAGING_TAG},${{ env.REGISTRY_QUAY }}:${STAGING_TAG}"
+          echo "staging_tags=$STAGING_TAGS" >> "$GITHUB_OUTPUT"
+          echo "staging_tag=$STAGING_TAG" >> "$GITHUB_OUTPUT"
+
           # Generate labels
           LABELS="org.opencontainers.image.title=RustFS"
           LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
@@ -467,7 +537,9 @@ jobs:
           file: ${{ matrix.file }}
           platforms: ${{ env.DOCKER_PLATFORMS }}
           push: ${{ needs.build-check.outputs.should_push == 'true' }}
-          tags: ${{ steps.meta.outputs.tags }}
+          # Push to staging tags only: official tags are created by the
+          # certification-gated promotion step below from this same digest.
+          tags: ${{ steps.meta.outputs.staging_tags }}
           labels: ${{ steps.meta.outputs.labels }}
           # No layer cache. This build compiles nothing — it downloads a
           # release zip and runs apk/apt — so the cache could only save the
@@ -493,6 +565,114 @@ jobs:
           no-cache: false
           pull: true
 
+      # A later-built wrapper: the staged image must complete this platform's
+      # own binary identity verification before any official tag is written.
+      # It may not inherit an unverified package's conclusion.
+      - name: Verify staged image against certification
+        if: needs.build-check.outputs.should_push == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
+          VERSION: ${{ needs.build-check.outputs.version }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p certification
+          run_id="$TRIGGERING_RUN_ID"
+          if [[ -z "$run_id" ]]; then
+            run_id="$(gh api --method GET \
+              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$VERSION" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' || true)"
+          fi
+          gh run download "$run_id" -n release-certification -D certification
+
+          STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
+          case "${{ matrix.variant }}" in
+            musl) libc="musl" ;;
+            glibc) libc="gnu" ;;
+          esac
+          for platform in linux/amd64 linux/arm64; do
+            case "$platform" in
+              linux/amd64) arch_sub="x86_64" ;;
+              linux/arm64) arch_sub="aarch64" ;;
+            esac
+            docker pull --platform "$platform" "$STAGING_REF"
+            container="$(docker create --platform "$platform" "$STAGING_REF")"
+            docker cp "$container:/usr/bin/rustfs" "$RUNNER_TEMP/staged-rustfs"
+            docker rm "$container" >/dev/null
+            actual="$(sha256sum "$RUNNER_TEMP/staged-rustfs" | awk '{print $1}')"
+            expected="$(jq -r --arg needle "linux-${arch_sub}-${libc}" \
+              '[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
+              certification/candidate.json)"
+            if [[ -z "$expected" || "$actual" != "$expected" ]]; then
+              echo "Staged image binary for ${platform} does not match the certified binary digest" >&2
+              exit 1
+            fi
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$RUNNER_TEMP/staged-rustfs" \
+              --artifact-role binary
+          done
+
+      - name: Promote staged image to official tags
+        if: needs.build-check.outputs.should_push == 'true'
+        shell: bash
+        run: |
+          set -euo pipefail
+          STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
+          IFS=',' read -r -a official_tags <<< "${{ steps.meta.outputs.tags }}"
+          for tag in "${official_tags[@]}"; do
+            docker buildx imagetools create -t "$tag" "$STAGING_REF"
+            echo "Promoted $tag -> $STAGING_REF"
+          done
+
+      # Downstream consumers (Helm) verify this record instead of inheriting
+      # an unverified wrapper's conclusion.
+      - name: Record image promotion
+        if: needs.build-check.outputs.should_push == 'true'
+        shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
+          VERSION: ${{ needs.build-check.outputs.version }}
+        run: |
+          set -euo pipefail
+          mkdir -p certification
+          run_id="$TRIGGERING_RUN_ID"
+          if [[ -z "$run_id" ]]; then
+            run_id="$(gh api --method GET \
+              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$VERSION" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' || true)"
+          fi
+          gh run download "$run_id" -n release-certification -D certification
+          STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
+          digest="$(docker buildx imagetools inspect "$STAGING_REF" --format '{{json .Manifest}}' | jq -r '.digest')"
+          candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
+          jq -n \
+            --arg version "$VERSION" \
+            --arg digest "$digest" \
+            --arg staging_ref "$STAGING_REF" \
+            --arg candidate_sha256 "$candidate_sha" \
+            --arg tags "${{ steps.meta.outputs.tags }}" \
+            --slurpfile candidate certification/candidate.json \
+            '{schema_version: 1, version: $version, image_digest: $digest, staging_ref: $staging_ref,
+              promoted_tags: ($tags | split(",")), candidate_sha256: $candidate_sha256,
+              binary_sha256s: ($candidate[0].deliverables | map(.binary_sha256) | unique)}' \
+            > image-promotion-record.json
+
+      - name: Upload image promotion record
+        if: needs.build-check.outputs.should_push == 'true'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: image-promotion-record${{ matrix.suffix }}
+          path: image-promotion-record.json
+          if-no-files-found: error
+          retention-days: 30
+
   # Note: Manifest creation is no longer needed as we only build one variant
   # Multi-arch manifests are automatically created by docker/build-push-action
 

.github/workflows/e2e-distributed.yml

@@ -17,8 +17,8 @@
 # Each selected test starts a real localhost cluster via
 # `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
 # case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
-# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
-# nightly runs, and manual dispatches all execute the same fail-closed suite.
+# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
+# and manual dispatches execute the same fail-closed suite.
 # Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
 #
 # Isolated pool filesystems: expand/decommission/rebalance cases require
@@ -33,32 +33,6 @@
 name: e2e-distributed
 
 on:
-  pull_request:
-    paths:
-      - "Cargo.lock"
-      - "Cargo.toml"
-      - ".config/nextest.toml"
-      - ".config/e2e-distributed-selection.txt"
-      - ".github/workflows/e2e-distributed.yml"
-      - "crates/audit/**"
-      - "crates/common/**"
-      - "crates/config/**"
-      - "crates/e2e_test/**"
-      - "crates/ecstore/**"
-      - "crates/filemeta/**"
-      - "crates/heal/**"
-      - "crates/iam/**"
-      - "crates/lock/**"
-      - "crates/madmin/**"
-      - "crates/notify/**"
-      - "crates/replication/**"
-      - "crates/s3-client/**"
-      - "crates/s3-ops/**"
-      - "crates/s3-types/**"
-      - "crates/scanner/**"
-      - "crates/storage-api/**"
-      - "crates/utils/**"
-      - "rustfs/**"
   workflow_dispatch:
     inputs:
       filter:

.github/workflows/e2e-upgrade.yml

@@ -12,30 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Single-node upgrade and rollback lane.
+#
+# The weekly Monday schedule, version tags, and workflow_dispatch run this
+# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
+
 name: Upgrade Compatibility
 
 on:
-  pull_request:
-    paths:
-      - ".github/workflows/e2e-upgrade.yml"
-      - "crates/e2e_test/src/common.rs"
-      - "crates/e2e_test/src/distributed/**"
-      - "crates/e2e_test/src/fake_s3_target/**"
-      - "crates/e2e_test/src/lib.rs"
-      - "crates/e2e_test/src/replication_extension_test.rs"
-      - "crates/e2e_test/src/rpc_fault_proxy.rs"
-      - "crates/e2e_test/src/upgrade_compatibility_test.rs"
-      - "crates/ecstore/**"
-      - "crates/filemeta/**"
-      - "crates/heal/**"
-      - "crates/kms/**"
-      - "crates/storage-api/**"
-      - "rustfs/**"
-      - "Cargo.lock"
   push:
     tags:
       - "[0-9]*.[0-9]*.[0-9]*"
   schedule:
+    # 03:17 UTC every Monday.
     - cron: "17 3 * * 1"
   workflow_dispatch:
 
@@ -44,7 +33,7 @@ permissions:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  cancel-in-progress: false
 
 env:
   CARGO_TERM_COLOR: always

.github/workflows/helm-package.yml

@@ -30,9 +30,102 @@ permissions:
   contents: read
 
 jobs:
+  # Consume the same trusted release certification as every other publication
+  # exit and verify the image promotion record of the Docker workflow. The
+  # chart wraps no binary of its own, so it may only reference an image whose
+  # platform verification already passed; it must not inherit an unverified
+  # package's conclusion. A run without that evidence fails closed here and
+  # never reaches the chart repository.
+  release-certification:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    outputs:
+      version: ${{ steps.version.outputs.raw_tag }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Normalize release version
+        id: version
+        env:
+          RAW_INPUT: ${{ github.event.inputs.version }}
+          RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
+        run: |
+          set -euo pipefail
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            RAW="$RAW_INPUT"
+          else
+            RAW="$RAW_BRANCH"
+          fi
+          case "$RAW" in
+            refs/tags/*) RAW_TAG="${RAW#refs/tags/}" ;;
+            *) RAW_TAG="$RAW" ;;
+          esac
+          echo "raw_tag=$RAW_TAG" >> "$GITHUB_OUTPUT"
+
+      - name: Download and verify release certification and image record
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
+          RAW_TAG: ${{ steps.version.outputs.raw_tag }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p certification image-record
+
+          build_run="$TRIGGERING_RUN_ID"
+          if [[ -z "$build_run" ]]; then
+            build_run="$(gh api --method GET \
+              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$RAW_TAG" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' || true)"
+          fi
+          if [[ -z "$build_run" ]]; then
+            echo "No certified Build and Release run found for ${RAW_TAG}; failing closed" >&2
+            exit 1
+          fi
+          if ! gh run download "$build_run" -n release-certification -D certification; then
+            echo "Build run ${build_run} published no release-certification artifact; failing closed" >&2
+            exit 1
+          fi
+          python3 scripts/release/release_gate.py verify-promotion \
+            --plan .config/release-validation-plan.json \
+            --candidate certification/candidate.json \
+            --summary certification/release-gate-output \
+            --artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
+            --artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
+
+          docker_run="$(gh api --method GET \
+            "repos/${GITHUB_REPOSITORY}/actions/workflows/docker.yml/runs" \
+            -f branch="$RAW_TAG" -f status=success -F per_page=1 \
+            --jq '.workflow_runs[0].id // empty' || true)"
+          if [[ -z "$docker_run" ]] || ! gh run download "$docker_run" -n image-promotion-record -D image-record; then
+            echo "No image promotion record for ${RAW_TAG}; a chart may not wrap an unverified image" >&2
+            exit 1
+          fi
+          candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
+          record_sha="$(jq -r '.candidate_sha256 // empty' image-record/image-promotion-record.json)"
+          if [[ "$record_sha" != "$candidate_sha" ]]; then
+            echo "Image promotion record does not belong to this certification" >&2
+            exit 1
+          fi
+          overlap="$(jq -r --slurpfile candidate certification/candidate.json \
+            '[.binary_sha256s[]] - ($candidate[0].deliverables | map(.binary_sha256)) | length' \
+            image-record/image-promotion-record.json)"
+          if [[ "$overlap" -ne 0 ]]; then
+            echo "Image promotion record carries binary digests outside the certification" >&2
+            exit 1
+          fi
+
   build-helm-package:
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    needs: [ release-certification ]
     if: |
       (github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
       (

.github/workflows/nightly-gnu.yml

@@ -87,18 +87,206 @@ jobs:
           install-build-packaging-tools: 'false'
           install-test-tools: 'false'
 
+      - name: Download static console assets
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          # Packaging tools are off on this lane, so setup does not install unzip.
+          if ! command -v unzip >/dev/null 2>&1; then
+            sudo apt-get update -qq
+            sudo apt-get install -y -qq unzip
+          fi
+          if ! command -v unzip >/dev/null 2>&1; then
+            echo "unzip is required to extract console assets" >&2
+            exit 1
+          fi
+
+          mkdir -p ./rustfs/static
+
+          verify_sha256() {
+            local expected="$1"
+            local file="$2"
+            local actual=""
+            local actual_line=""
+
+            if command -v sha256sum >/dev/null 2>&1; then
+              actual_line=$(sha256sum -- "$file") || return 1
+            elif command -v shasum >/dev/null 2>&1; then
+              actual_line=$(shasum -a 256 -- "$file") || return 1
+            else
+              echo "No SHA-256 verification tool found" >&2
+              return 1
+            fi
+            actual="${actual_line%%[[:space:]]*}"
+
+            if [ "$actual" = "$expected" ]; then
+              echo "SHA256 verified OK: $actual"
+              return 0
+            else
+              echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
+              return 1
+            fi
+          }
+
+          download_console_assets() {
+            local curl_bin="curl"
+            local python_bin="python3"
+            local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
+            local console_json="console-release.json"
+            local console_url
+            local console_sha256
+            local curl_auth_args=()
+            console_tag=""
+
+            chmod +w ./rustfs/static/LICENSE || true
+
+            if ! command -v "$python_bin" >/dev/null 2>&1; then
+              python_bin="python"
+            fi
+            if ! command -v "$python_bin" >/dev/null 2>&1; then
+              echo "No Python interpreter found for release metadata parsing" >&2
+              return 1
+            fi
+
+            if [[ -n "${GITHUB_TOKEN:-}" ]]; then
+              curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
+            fi
+
+            "$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
+              -o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
+
+            read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
+          import json
+          import re
+          import sys
+
+          with open(sys.argv[1], encoding="utf-8") as handle:
+              release = json.load(handle)
+
+          tag = release.get("tag_name", "") or "unknown"
+
+          for asset in release.get("assets", []):
+              name = asset.get("name", "")
+              digest = asset.get("digest", "")
+              url = asset.get("browser_download_url", "")
+              if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
+                  sha256 = digest.split(":", 1)[1]
+                  if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
+                      raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
+                  sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
+                  break
+          else:
+              raise SystemExit("no console zip asset with sha256 digest found")
+          PY
+            ) || return 1
+
+            if [[ -z "$console_url" || -z "$console_sha256" ]]; then
+              echo "Console release metadata is missing URL or SHA-256 digest" >&2
+              return 1
+            fi
+
+            echo "Console release: ${console_tag}"
+            echo "Downloading console asset: ${console_url}"
+            "$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
+            verify_sha256 "$console_sha256" console.zip || return 2
+            unzip -o console.zip -d ./rustfs/static || return 2
+          }
+
+          if download_console_assets; then
+            rm -f console.zip console-release.json
+          else
+            status=$?
+            rm -f console.zip console-release.json
+            if [[ "$status" -eq 2 ]]; then
+              echo "Console asset integrity verification failed" >&2
+            fi
+            echo "Failed to download verified console assets" >&2
+            exit 1
+          fi
+
+          if [[ ! -s ./rustfs/static/index.html ]]; then
+            echo "Console asset archive is missing static/index.html" >&2
+            exit 1
+          fi
+
+          asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
+          echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
+
       - name: Build RustFS
         # Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
         # nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
-        run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
+        shell: bash
+        run: |
+          # Force rebuild by touching build.rs so a restored cache cannot
+          # reuse a rustfs artifact compiled before these console assets existed.
+          touch rustfs/build.rs
+          cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
 
       - name: Verify glibc symbol baseline
         run: |
           python3 scripts/release/check_glibc.py 2.31 \
-            target/x86_64-unknown-linux-gnu/release/rustfs \
-            target/x86_64-unknown-linux-gnu/release/rustfs-cli
+            target/x86_64-unknown-linux-gnu/release/rustfs
           target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
 
+      - name: Verify console
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          find_free_port() {
+            python3 - <<'PY'
+          import socket
+
+          with socket.socket() as sock:
+              sock.bind(("127.0.0.1", 0))
+              print(sock.getsockname()[1])
+          PY
+          }
+
+          binary="target/x86_64-unknown-linux-gnu/release/rustfs"
+          data_dir="$(mktemp -d)"
+          log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
+          server_pid=""
+          trap '
+            if [[ -n "${server_pid:-}" ]]; then
+              kill "$server_pid" 2>/dev/null || true
+              wait "$server_pid" 2>/dev/null || true
+            fi
+            rm -rf "$data_dir"
+          ' EXIT
+
+          if [[ ! -x "$binary" ]]; then
+            echo "rustfs binary not found: ${binary}" >&2
+            exit 1
+          fi
+
+          api_port="$(find_free_port)"
+          console_port="$(find_free_port)"
+          console_url="http://127.0.0.1:${console_port}/rustfs/console/"
+
+          "$binary" server \
+            --address "127.0.0.1:${api_port}" \
+            --console-enable \
+            --console-address "127.0.0.1:${console_port}" \
+            --access-key console-smoke \
+            --secret-key console-smoke-secret \
+            "$data_dir" >"$log_file" 2>&1 &
+          server_pid=$!
+
+          response=""
+          for _ in {1..40}; do
+            response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
+            if [[ "$response" == 200\ text/html* ]]; then
+              exit 0
+            fi
+            sleep 0.25
+          done
+
+          echo "Console endpoint did not return 200 text/html: $response" >&2
+          cat "$log_file"
+          exit 1
+
       - name: Build DEB package
         id: deb
         shell: bash
@@ -346,12 +534,12 @@ jobs:
           path: ${{ steps.publish.outputs.candidate_file }}
           if-no-files-found: error
 
-      # Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
-      # engineers can download and install the nightly directly. The branch is
-      # a single-commit orphan rewritten on every build, which keeps the repo
-      # small while the latest files stay reachable at stable raw URLs.
       # Publish the deb/rpm pair as assets of the rolling `nightly` release on
       # rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
+      # The dispatch below runs only after this upload and the immutable R2
+      # candidate both succeed. It does not replace rustfs-functional-chain.yml,
+      # which still starts from this workflow's workflow_run completion.
+      # kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
       - name: Publish packages to auto-testing release assets
         env:
           ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -360,6 +548,90 @@ jobs:
           DEB_DATE: ${{ steps.deb.outputs.deb_date }}
           BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
         run: bash scripts/release/publish_nightly_assets.sh
+
+      # Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
+      # PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
+      # or fine-grained Contents write (the permission GitHub checks for
+      # POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
+      # step, and therefore this job, so the scheduled failure alert fires.
+      # GitHub allows at most 10 top-level client_payload properties, so the
+      # schema 2 candidate object is nested under `candidate` and the release
+      # fields sit beside it.
+      - name: Dispatch nightly candidate to auto-testing
+        if: ${{ steps.publish.outputs.candidate_file != '' }}
+        env:
+          DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
+          CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
+          DEB_DATE: ${{ steps.deb.outputs.deb_date }}
+          SOURCE_REPO: ${{ github.repository }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          : "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
+          : "${CANDIDATE_FILE:?nightly candidate manifest is required}"
+          : "${DEB_DATE:?DEB_DATE is required}"
+          [[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
+          [[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
+
+          payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
+            --arg event_type "nightly-candidate-ready" \
+            --arg source_repo "${SOURCE_REPO}" \
+            --arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
+            --arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
+            --arg channel "nightly" \
+            --arg deb_date "${DEB_DATE}" \
+            --arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
+            '{
+              event_type: $event_type,
+              source_repo: $source_repo,
+              latest_alias_url: $latest_alias_url,
+              auto_testing_release: $auto_testing_release,
+              channel: $channel,
+              deb_date: $deb_date,
+              nightly_workflow: $nightly_workflow,
+              candidate: $candidate[0]
+            }' > "${payload}"
+          if ! jq -e '
+            .event_type == "nightly-candidate-ready"
+            and .source_repo == "rustfs/rustfs"
+            and .channel == "nightly"
+            and .deb_date == env.DEB_DATE
+            and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
+            and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
+            and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
+            and .candidate.schema == 2
+            and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
+            and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
+          ' "${payload}" >/dev/null; then
+            echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
+            exit 1
+          fi
+
+          body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --slurpfile client_payload "${payload}" \
+            '{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
+          response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
+          status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
+            -o "${response}" -w '%{http_code}' \
+            -X POST \
+            -H "Authorization: Bearer ${DISPATCH_TOKEN}" \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            -H "Content-Type: application/json" \
+            --data-binary @"${body}" \
+            https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
+            echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
+            exit 1
+          }
+          if [[ "${status}" != "204" ]]; then
+            echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
+            if [[ -s "${response}" ]]; then
+              cat "${response}" >&2
+            fi
+            exit 1
+          fi
+          echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
   # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
   #
   # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and

.github/workflows/package.yml

@@ -225,10 +225,57 @@ jobs:
           echo "  Build run ID: $BUILD_RUN_ID"
           echo "  Build run number: $RUN_NUMBER"
 
+  # Consume the trusted release certification of the resolved build run and
+  # verify the certified input digests before any package is wrapped. A run
+  # without that certification fails closed here; packages built later must
+  # complete their own verification and cannot inherit an unverified
+  # conclusion.
+  release-certification:
+    name: Release Certification
+    needs: resolve
+    if: needs.resolve.result == 'success'
+    runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download and verify release certification
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
+          BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p certification
+          if [[ "$BUILD_TYPE" == "development" ]]; then
+            # Development artifacts are explicitly uncertified and never touch
+            # official or latest channels; only the staging/dev paths remain.
+            jq -n '{schema_version: 1, certified: false, reason: "development artifacts are explicitly uncertified"}' \
+              > certification/development-uncertified.json
+            exit 0
+          fi
+          if ! gh run download "$BUILD_RUN_ID" -n release-certification -D certification; then
+            echo "Build run ${BUILD_RUN_ID} published no release-certification artifact; failing closed" >&2
+            exit 1
+          fi
+          python3 scripts/release/release_gate.py verify-promotion \
+            --plan .config/release-validation-plan.json \
+            --candidate certification/candidate.json \
+            --summary certification/release-gate-output \
+            --artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
+            --artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
+
   # Build DEB and RPM packages for each architecture and libc variant
   package:
     name: Package (${{ matrix.arch }}-${{ matrix.libc }})
-    needs: resolve
+    needs: [ resolve, release-certification ]
     runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
     timeout-minutes: 30
     strategy:
@@ -494,6 +541,55 @@ jobs:
           echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ RPM built: $RPM_FILE"
 
+      - name: Download certification evidence
+        if: needs.resolve.outputs.build_type != 'development'
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: release-certification
+          path: ./certification
+
+      # Later-built wrapper verification: the binary inside each package must
+      # match the certified binary digest for this platform before anything is
+      # uploaded, and the package itself stays on staging until promotion.
+      - name: Verify wrapped binary against certification
+        if: needs.resolve.outputs.build_type != 'development'
+        shell: bash
+        env:
+          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
+          RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
+          ARTIFACT_NAME: ${{ matrix.artifact_name }}
+        run: |
+          set -euo pipefail
+          expected="$(jq -r --arg needle "$ARTIFACT_NAME" \
+            '[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
+            certification/candidate.json)"
+          if [[ -z "$expected" ]]; then
+            echo "No certified binary digest recorded for ${ARTIFACT_NAME}" >&2
+            exit 1
+          fi
+          if ! command -v rpm2cpio >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y rpm2cpio
+          fi
+          if ! command -v cpio >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y cpio
+          fi
+          for package_file in "$DEB_FILE" "$RPM_FILE"; do
+            case "$package_file" in
+              *.deb)
+                dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
+                ;;
+              *.rpm)
+                rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
+                ;;
+            esac
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$RUNNER_TEMP/wrapped-rustfs" \
+              --artifact-role binary
+          done
+
       - name: Upload packages to artifacts
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
@@ -531,10 +627,14 @@ jobs:
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
+          # Build-phase writes stay on staging for release types: the official
+          # packages/release and packages/latest paths are written only by the
+          # certification-gated promote-packages job, from these same staged
+          # objects. Development packages keep the uncertified dev channel.
           if [[ "$BUILD_TYPE" == "development" ]]; then
             R2_PREFIX="artifacts/rustfs/packages/dev"
           else
-            R2_PREFIX="artifacts/rustfs/packages/release"
+            R2_PREFIX="artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}"
           fi
           R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
 
@@ -549,18 +649,6 @@ jobs:
 
           echo "✅ Upload complete"
 
-          # Also upload as latest for release/prerelease
-          if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
-            LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
-            for f in "$DEB_FILE" "$RPM_FILE"; do
-              if [[ -n "$f" && -f "$f" ]]; then
-                echo "Uploading latest: $(basename "$f")"
-                aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
-              fi
-            done
-            echo "✅ Latest packages updated"
-          fi
-
       - name: Upload packages to GitHub Release
         if: needs.resolve.outputs.tag != ''
         env:
@@ -614,6 +702,114 @@ jobs:
 
           echo "✅ GitHub release assets updated"
 
+  # The only writer of the official packages/release and packages/latest R2
+  # paths. Promotion re-checks each package's wrapped binary against the
+  # passed certification list and copies the staged objects byte-for-byte:
+  # no rebuild-after-verify, no mutable latest resolution downstream.
+  promote-packages:
+    name: Promote Packages
+    needs: [ resolve, package, release-certification ]
+    # Target-channel lock for the official package paths.
+    concurrency:
+      group: release-channel-r2-packages
+      cancel-in-progress: false
+    if: needs.resolve.outputs.tag != '' && (needs.resolve.outputs.build_type == 'release' || needs.resolve.outputs.build_type == 'prerelease')
+    runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    env:
+      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+      R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+      R2_BUCKET: ${{ secrets.R2_BUCKET }}
+      AWS_EC2_METADATA_DISABLED: true
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download packages and certification
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: ./packages
+          pattern: packages-*
+          merge-multiple: true
+
+      - name: Download certification evidence
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: release-certification
+          path: ./certification
+
+      - name: Verify and promote staged packages
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
+            echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
+            exit 1
+          fi
+
+          STAGING_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
+          RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/release/"
+          LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
+
+          if ! command -v aws >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y awscli
+          fi
+          export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
+          export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
+          export AWS_DEFAULT_REGION="auto"
+
+          if ! command -v rpm2cpio >/dev/null 2>&1; then
+            sudo apt-get install -y rpm2cpio
+          fi
+          if ! command -v cpio >/dev/null 2>&1; then
+            sudo apt-get install -y cpio
+          fi
+
+          promoted=0
+          for package_file in packages/*.deb packages/*.rpm; do
+            [[ -f "$package_file" ]] || continue
+            name="$(basename "$package_file")"
+            case "$package_file" in
+              *.deb)
+                dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
+                ;;
+              *.rpm)
+                rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
+                ;;
+            esac
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$RUNNER_TEMP/wrapped-rustfs" \
+              --artifact-role binary
+
+            staged="$(mktemp)"
+            aws s3 cp "${STAGING_PREFIX}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
+            built_sha="$(sha256sum "$package_file" | awk '{print $1}')"
+            if [[ "$staged_sha" != "$built_sha" ]]; then
+              echo "❌ Staged package ${name} does not match the verified bytes" >&2
+              exit 1
+            fi
+            aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            aws s3 cp "$staged" "${LATEST_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            rm -f "$staged"
+            promoted=$((promoted + 1))
+            echo "✅ Promoted ${name} (${built_sha})"
+          done
+          if [[ "$promoted" -eq 0 ]]; then
+            echo "No staged packages found to promote" >&2
+            exit 1
+          fi
+
   # Summary
   summary:
     name: Summary

.github/workflows/rustfs-validation-sweep.yml

@@ -0,0 +1,457 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: RustFS Validation Sweep
+
+# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
+# on the shared functional fleet. Everything with a lane executor is swept in
+# rotating nightly batches so the whole set is covered roughly weekly; a
+# single case or one module can be dispatched manually. That is the
+# implemented catalog (66 today) plus the eight planned cases whose dedicated
+# executors already exist and have live PASS evidence - fresh nightly
+# evidence feeds their acceptance review. heal.* cases stay on the
+# interactive validation flow: they require reviewed native/tool collectors
+# built by separate validation-tools runs before fleet execution.
+
+on:
+  schedule:
+    # After the nightly functional chain settles (chain starts ~16:48 UTC and
+    # runs for about two hours).
+    - cron: "30 19 * * *"
+  workflow_dispatch:
+    inputs:
+      case:
+        description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
+        type: string
+        required: false
+        default: ''
+      module:
+        description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
+        type: string
+        required: false
+        default: ''
+      limit:
+        description: 'Maximum cases per sweep (0 = run everything selected)'
+        type: number
+        required: false
+        default: 12
+      build_run_id:
+        description: 'Successful nightly package run; empty resolves the latest'
+        type: string
+        required: false
+        default: ''
+      build_run_attempt:
+        description: 'Attempt of that package run'
+        type: string
+        required: false
+        default: '1'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rustfs-validation-sweep
+  cancel-in-progress: false
+
+env:
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  sweep:
+    if: github.repository == 'rustfs/rustfs'
+    runs-on: smoke-testing
+    timeout-minutes: 150
+    steps:
+      - name: Checkout chain tooling
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Resolve nightly candidate
+        id: candidate
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
+          CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
+        run: |
+          set -euo pipefail
+          # Scheduled runs carry no inputs and the resolver only derives a
+          # build from a workflow_run payload (which a schedule cannot
+          # provide), so resolve the newest successful nightly build
+          # explicitly before invoking it.
+          if [ -z "${BUILD_RUN_ID:-}" ]; then
+            LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
+            BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
+            BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
+            export BUILD_RUN_ID BUILD_RUN_ATTEMPT
+            echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
+          fi
+          python3 scripts/resolve_functional_candidate.py
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+          root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
+          root.mkdir(parents=True, exist_ok=True)
+          chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
+          manifest = chain["candidate"]["manifest"]
+          (root / "candidate.json").write_text(json.dumps({
+              "package_url": manifest["package_url"],
+              "package_sha256": manifest["package_sha256"],
+              "source_sha": manifest["source_sha"],
+          }, sort_keys=True) + "\n")
+          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+              output.write("testing_sha=" + chain["testing_sha"] + "\n")
+              output.write("package_url=" + manifest["package_url"] + "\n")
+              output.write("source_sha=" + manifest["source_sha"] + "\n")
+          PY
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.candidate.outputs.testing_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Prepare validation Python runtime
+        # The pool / ilm / replication validation runners require a managed
+        # Python, exactly like the chain's validation job provides.
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+
+      - name: Install validation Python requirements
+        # Case scripts call python3 and import boto3 at startup. Install the
+        # functional chain's client into a per-run env so that interpreter
+        # sees it without changing the runner's shared Python.
+        run: |
+          set -euo pipefail
+          validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          test ! -e "${validation_python}"
+          python3 -m venv "${validation_python}"
+          "${validation_python}/bin/python" -m pip install \
+            -r auto-testing/scripts/validation_ilm/requirements.txt
+          printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
+
+      - name: Select sweep cases
+        id: select
+        env:
+          SELECTED_CASE: ${{ inputs.case }}
+          SELECTED_MODULE: ${{ inputs.module }}
+          CASE_LIMIT: ${{ inputs.limit }}
+          SWEEP_EVENT: ${{ github.event_name }}
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import datetime
+          import json
+          import os
+          from pathlib import Path
+          sweep_root = Path(os.environ["SWEEP_ROOT"])
+
+          catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
+          def sweepable(case):
+              status = case.get("source_status")
+              if status == "implemented":
+                  return True
+              # The remaining planned non-heal cases route to the lane
+              # executors and already have live PASS evidence; sweeping them
+              # nightly produces fresh evidence for their acceptance review.
+              return status == "planned" and not case["id"].startswith("heal.")
+
+          selected = [(case["id"], module["id"]) for module in catalog["modules"]
+                      for case in module["cases"]
+                      if sweepable(case)]
+          if not selected:
+              raise SystemExit("catalog has no sweepable cases")
+
+          one_case = os.environ.get("SELECTED_CASE", "").strip()
+          module_filter = os.environ.get("SELECTED_MODULE", "").strip()
+          known = {case_id for case_id, _ in selected}
+          modules = {module_id for _, module_id in selected}
+          if one_case:
+              if one_case not in known:
+                  raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
+              selected = [(one_case, dict(selected)[one_case])]
+          elif module_filter:
+              if module_filter not in modules:
+                  raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
+              selected = [pair for pair in selected if pair[1] == module_filter]
+          elif os.environ.get("SWEEP_EVENT") == "schedule":
+              # Stateless rotation: every night take one window of the catalog
+              # so a full sweep needs len(selected)/limit nights and no state
+              # has to be carried between runs.
+              limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
+              day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
+              start = (day * limit) % len(selected)
+              selected = (selected[start:] + selected[:start])[:limit]
+          limit = int(os.environ.get("CASE_LIMIT") or 0)
+          if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
+              if limit > 0:
+                  selected = selected[:limit]
+
+          (sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
+          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+              output.write(f"count={len(selected)}\n")
+          print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
+          PY
+
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
+        run: python3 scripts/functional_environment.py enter --suite heal
+
+      - name: Run validation cases
+        id: run_cases
+        timeout-minutes: 120
+        env:
+          AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+        run: |
+          set -uo pipefail
+          mkdir -p "${SWEEP_ROOT}/results"
+          : > "${SWEEP_ROOT}/results.txt"
+          while read -r CASE_ID; do
+            [ -n "${CASE_ID}" ] || continue
+            echo "::group::case ${CASE_ID}"
+            rm -rf "${SWEEP_ROOT}/run"
+            if ./auto-testing/scripts/run_validation_case.sh \
+                 "${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
+                 "${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
+                 > "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
+              printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
+              echo "case ${CASE_ID}: PASS"
+            else
+              printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
+              echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
+            fi
+            if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
+              cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
+            fi
+            echo "::endgroup::"
+            # Fleet hygiene between cases: the case runners set up their own
+            # topology, this purge guarantees the next case starts clean even
+            # when the previous one died mid-run.
+            for node in ${RUSTFS_NODES}; do
+              # -n keeps ssh from consuming the case list this loop reads from stdin
+              ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
+                SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+                ${SUDO} systemctl stop rustfs 2>/dev/null || true
+                if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                  ${SUDO} dpkg -P rustfs
+                fi
+                for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+                ${SUDO} rm -f /etc/default/rustfs
+              ' || echo "inter-case cleanup failed on ${node} (continuing)"
+            done
+          done < "${SWEEP_ROOT}/sweep-cases.txt"
+          echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
+
+      - name: Generate sweep report
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        env:
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+          PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
+          SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
+          AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+
+          root = Path(os.environ["SWEEP_ROOT"])
+          catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
+          module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
+          rows = []
+          for line in (root / "results.txt").read_text().splitlines():
+            if not line:
+                continue
+            case_id, status = line.split("\t")
+            rows.append((case_id, module_of.get(case_id, "?"), status))
+          passed = sum(1 for _, _, s in rows if s == "PASS")
+          failed = len(rows) - passed
+          run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
+                     f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
+                     f"{os.environ.get('GITHUB_RUN_ID', '')}")
+
+          report = [
+              "# RustFS validation sweep report", "",
+              f"- Run: {run_url}",
+              f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
+              f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
+              f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
+              f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
+              f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
+              "Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
+          ]
+          # Group the executed cases by catalog module (the major category:
+          # pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
+          # data services, identity, s3 compatibility). Per-module subtotals
+          # deliberately use "- Subtotal:" so the dashboard overview regexes
+          # (which anchor on "- Total:") keep reading the grand totals below.
+          by_module = {}
+          module_order = []
+          for case_id, module, status in rows:
+              if module not in by_module:
+                  by_module[module] = []
+                  module_order.append(module)
+              by_module[module].append((case_id, status))
+          report += ["## Case Summary", ""]
+          for module in module_order:
+              cases = by_module[module]
+              mod_pass = sum(1 for _, st in cases if st == "PASS")
+              mod_fail = len(cases) - mod_pass
+              report += [f"### {module} ({len(cases)})", "",
+                         "| Case | Status |", "| --- | --- |"]
+              report += [f"| {case_id} | {status} |" for case_id, status in cases]
+              report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
+          report += [
+              "## Totals", "",
+              f"- Total: {len(rows)}",
+              f"- PASS: {passed}",
+              f"- FAIL: {failed}",
+              "- UNSUPPORTED: 0", "",
+              "Failing cases keep this workflow red only as a report; the sweep artifact",
+              "retains every case log and per-case report for diagnosis.", "",
+          ]
+          (root / "report.md").write_text("\n".join(report))
+          (root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
+          PY
+          cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload functional report to dashboard
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 2
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
+          SUITE: validation
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+          echo "dashboard updated: ${REPORT_PATH}"
+
+      - name: Retain sweep evidence
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            ${{ runner.temp }}/validation-sweep/report.md
+            ${{ runner.temp }}/validation-sweep/results.txt
+            ${{ runner.temp }}/validation-sweep/sweep-cases.txt
+            ${{ runner.temp }}/validation-sweep/candidate.json
+            ${{ runner.temp }}/validation-sweep/results/
+            ${{ runner.temp }}/validation-sweep/run/
+          # warn (not error): an early infra failure legitimately leaves no
+          # per-case evidence, and the failing step is already red.
+          if-no-files-found: warn
+          retention-days: 30
+
+      - name: Cleanup environment (after)
+        id: environment_cleanup
+        timeout-minutes: 5
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        run: |
+          set -uo pipefail
+          cleanup_status=0
+          for node in ${RUSTFS_NODES}; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -f /etc/default/rustfs
+            ' || cleanup_status=$?
+          done
+          exit "${cleanup_status}"
+
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Notify on failure
+        if: failure()
+        run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
+
+      - name: Enforce sweep result
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        env:
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+        run: |
+          set -euo pipefail
+          if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
+            echo "validation sweep has failing cases:"
+            grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
+            exit 1
+          fi
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    # A scheduled sweep failure usually means a canonical validation case
+    # regressed or the fleet environment broke; surface it as a tracking
+    # issue instead of leaving it in the Actions tab.
+    needs: [sweep]
+    if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/windows-filesystem.yml

@@ -107,6 +107,32 @@ jobs:
         shell: pwsh
         run: cargo test -p rustfs-ecstore --lib multipart_owned_record_parent_barriers_ -- --nocapture
 
+      # Native multipart completion publishes its payload from a different
+      # parent than its metadata, and abort renames under a precondition; both
+      # need their own Windows handle discipline.
+      - name: Test native multipart complete and abort publication
+        shell: pwsh
+        run: >-
+          cargo test -p rustfs-ecstore --lib --
+          native_abort_missing_state_with_actual_owned_records_never_becomes_legacy_or_missing
+          native_abort_pending_intent_is_not_a_delete_certificate_and_raw_cleanup_preserves_exact_evidence
+          --nocapture
+
+      # Owned-undo reclaim removes a directory it pinned, copy heal persists
+      # fault bytes through a handle, and metadata reads meet a directory.
+      - name: Test native multipart reclaim, copy heal, and directory reads
+        shell: pwsh
+        run: >-
+          cargo test -p rustfs-ecstore --lib --
+          native_multipart_owned_undo
+          native_multipart_copy_heal
+          orphan_multipart_cleanup_rejects_metadata_directory_on_one_disk
+          --nocapture
+
+      - name: Test directory removal fallback
+        shell: pwsh
+        run: "cargo test -p rustfs-ecstore --lib disk::fs::tests::test_remove -- --nocapture"
+
       - name: Test Windows path conversion
         shell: pwsh
         run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"

CHANGELOG.md

@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
 - A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
 - Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
+- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
 - Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
 
 ### Security

Cargo.toml

@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
 mysql_async = { default-features = false, version = "0.37.1" }
-async-compression = { version = "0.4.51" }
+async-compression = { version = "0.4.52" }
 async-trait = "0.1.92"
 async-nats = { version = "0.51.0", default-features = false }
 axum = "0.8.9"
@@ -168,16 +168,16 @@ http-body-util = "0.1.5"
 memchr = "2.8.3"
 minlz = "1.2.3"
 reqwest = "0.13.5"
-rustfs-kafka-async = { version = "1.4.0" }
+rustfs-kafka-async = { version = "1.4.1" }
 socket2 = { version = "0.6.5" }
 tokio = { version = "1.54.1" }
 tokio-rustls = { default-features = false, version = "0.26.6" }
 tokio-stream = { version = "0.1.19" }
 tokio-test = "0.4.6"
 tokio-util = { version = "0.7.20" }
-tonic = { version = "0.14.8" }
-tonic-prost = { version = "0.14.8" }
-tonic-prost-build = { version = "0.14.8" }
+tonic = { version = "0.14.9" }
+tonic-prost = { version = "0.14.9" }
+tonic-prost-build = { version = "0.14.9" }
 tower = { version = "0.5.3" }
 tower-http = { version = "0.7.1" }
 h3 = "0.0.8"
@@ -191,6 +191,7 @@ quinn = { version = "0.11.12", default-features = false, features = [
 # Serialization and Data Formats
 apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
 bytes = { version = "1.12.1" }
+csv-core = "0.1.13"
 bytesize = "2.7.0"
 byteorder = "1.5.0"
 flatbuffers = "25.12.19"
@@ -212,12 +213,13 @@ serde_urlencoded = "0.7.1"
 # releases.
 aes-gcm = { version = "0.11.1" }
 argon2 = { version = "0.6.0" }
+aws-lc-rs = { version = "1.19.0", default-features = false }
 blake2 = "0.11.0"
 chacha20poly1305 = { version = "0.11.0" }
 crc-fast = "1.10.0"
 hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "11.1.0" }
-openidconnect = { default-features = false, version = "4.0" }
+openidconnect = { default-features = false, version = "4.1.0" }
 pbkdf2 = "0.13.0"
 p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
 rsa = { version = "=0.10.0-rc.18" }
@@ -234,7 +236,7 @@ proptest = "1"
 # Time and Date
 chrono = { version = "0.4.45" }
 humantime = "2.4.0"
-jiff = { version = "0.2.38" }
+jiff = { version = "0.2.39" }
 time = { version = "0.3.55" }
 
 # Database
@@ -253,9 +255,9 @@ atoi = "3.1.1"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.13.0" }
 aws-credential-types = { version = "1.3.1" }
-aws-sdk-kms = { default-features = false, version = "1.128.0" }
-aws-sdk-s3 = { default-features = false, version = "1.156.0" }
-aws-sdk-sts = { default-features = false, version = "1.123.0" }
+aws-sdk-kms = { default-features = false, version = "1.129.0" }
+aws-sdk-s3 = { default-features = false, version = "1.157.0" }
+aws-sdk-sts = { default-features = false, version = "1.124.0" }
 aws-smithy-async = { version = "1.3.1" }
 aws-smithy-http-client = { default-features = false, version = "1.5.2" }
 aws-smithy-runtime-api = { version = "1.19.2" }
@@ -319,7 +321,7 @@ rustfs-erasure-codec = { version = "9.0.0" }
 reed-solomon-simd = "3.1.0"
 regex = { version = "1.13.1" }
 rumqttc = { package = "rumqttc-next", version = "0.34.0" }
-redis = { version = "1.8.0" }
+redis = { version = "1.8.1" }
 rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.6" }
 rust-embed = { version = "8.13.0" }
@@ -334,10 +336,10 @@ compact_str = "0.10.0"
 snap = "1.1.2"
 starshard = { version = "2.4.0" }
 strum = { version = "0.28.0" }
-sysinfo = "0.40.0"
+sysinfo = "0.40.1"
 temp-env = "0.3.6"
 tempfile = "3.27.0"
-test-case = "3.4.0"
+test-case = "3.4.1"
 thiserror = "2.0.21"
 tracing = { version = "0.1.44" }
 tracing-appender = "0.2.5"
@@ -356,13 +358,13 @@ winapi-util = "0.1.11"
 windows = { version = "0.62.2" }
 windows-sys = "0.61.2"
 xxhash-rust = { version = "0.8.19" }
-zip = "9.1.3"
+zip = "9.3.0"
 zstd = "0.14.3"
 
 # Observability and Metrics
 metrics = "0.24.6"
 metrics-util = "0.20"
-dial9 = { version = "0.5.6", default-features = false }
+dial9 = { version = "0.5.7", default-features = false }
 opentelemetry = { version = "0.33.1" }
 opentelemetry-appender-tracing = { version = "0.33.1" }
 opentelemetry-otlp = { version = "0.33.1" }
@@ -377,16 +379,16 @@ libunftp = { version = "0.24.0" }
 unftp-core = "0.1.0"
 suppaftp = { version = "12.2.1" }
 rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
-russh = { version = "0.65.0" }
+russh = { version = "0.65.1" }
 russh-sftp = "3.0.1"
 
 # WebDAV
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-rustfs-mimalloc = { version = "0.7.2" }
+rustfs-mimalloc = { version = "0.7.3" }
 # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
-hotpath = { version = "0.30.1", default-features = false }
+hotpath = { version = "0.30.3", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.49" }
 
@@ -427,7 +429,3 @@ strip = "none"
 # Pin upstream fixes until compatible crates.io releases are available.
 [patch.crates-io]
 zstd = { path = "crates/zstd-compat" }
-# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
-# Upstream: openidconnect-rs#244 and oauth2-rs#352.
-openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
-oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }

Dockerfile

@@ -16,6 +16,7 @@ FROM alpine:3.24.2 AS build
 
 ARG TARGETARCH
 ARG RELEASE=latest
+ARG RC_RELEASE=latest
 
 RUN apk add --no-cache ca-certificates curl jq unzip
 WORKDIR /build
@@ -69,6 +70,9 @@ RUN set -eux; \
     chmod +x /build/rustfs; \
     rm -rf rustfs.zip /build/.tmp || true
 
+COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
+RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /build/rc
+
 
 FROM alpine:3.24.2
 
@@ -95,14 +99,18 @@ RUN apk upgrade --no-cache && \
     ca-certificates \
     coreutils \
     curl \
+    jq \
     tzdata \
     && test "$(TZ=Asia/Kolkata date +%z)" = "+0530"
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /build/rustfs /usr/bin/rustfs
-COPY entrypoint.sh /entrypoint.sh
+COPY --from=build /build/rc /usr/bin/rc
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
 
-RUN chmod +x /usr/bin/rustfs /entrypoint.sh
+RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 RUN addgroup -g 10001 -S rustfs && \
     adduser -u 10001 -G rustfs -S rustfs -D && \
@@ -124,6 +132,6 @@ VOLUME ["/data"]
 
 USER rustfs
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 
 CMD ["rustfs"]

Dockerfile.decommission-local

@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rust:1.97.1-trixie
+FROM rust:1.99.1-trixie
+
+ARG TARGETARCH
+ARG RC_RELEASE=latest
 
 RUN set -eux; \
     export DEBIAN_FRONTEND=noninteractive; \
@@ -21,6 +24,7 @@ RUN set -eux; \
     build-essential \
     ca-certificates \
     curl \
+    jq \
     git \
     pkg-config \
     libssl-dev \
@@ -36,6 +40,7 @@ COPY . .
 RUN ./scripts/static.sh
 RUN cargo run --bin gproto
 RUN cargo build --release --locked --bin rustfs
+RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
 
 RUN set -eux; \
     groupadd -g 10001 rustfs; \
@@ -50,8 +55,12 @@ RUN set -eux; \
     chown -R rustfs:rustfs /data /logs /app; \
     chmod 0750 /data /logs
 
-COPY entrypoint.sh /entrypoint.sh
-RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && chmod +x /entrypoint.sh
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
+RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && \
+    install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
+    chmod +x /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 ENV RUSTFS_VOLUMES="/data" \
     RUST_LOG="warn" \
@@ -63,5 +72,5 @@ ENV RUSTFS_VOLUMES="/data" \
 
 EXPOSE 9000 9001
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 CMD ["/usr/bin/rustfs"]

Dockerfile.glibc

@@ -16,6 +16,7 @@ FROM ubuntu:26.04 AS build
 
 ARG TARGETARCH
 ARG RELEASE=latest
+ARG RC_RELEASE=latest
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     ca-certificates \
@@ -76,6 +77,9 @@ RUN set -eux; \
     chmod +x /build/rustfs; \
     rm -rf rustfs.zip /build/.tmp || true
 
+COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
+RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" gnu "$RC_RELEASE" /build/rc
+
 FROM ubuntu:26.04
 
 ARG RELEASE=latest
@@ -99,14 +103,18 @@ RUN apt-get update && apt-get upgrade -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
+    jq \
     tzdata \
     && test "$(TZ=Asia/Kolkata date +%z)" = "+0530" \
     && rm -rf /var/lib/apt/lists/*
 
 COPY --from=build /build/rustfs /usr/bin/rustfs
-COPY entrypoint.sh /entrypoint.sh
+COPY --from=build /build/rc /usr/bin/rc
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
 
-RUN chmod +x /usr/bin/rustfs /entrypoint.sh
+RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 RUN groupadd -g 10001 rustfs && \
     useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
@@ -131,6 +139,6 @@ VOLUME ["/data"]
 
 USER rustfs
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 
 CMD ["rustfs"]

Dockerfile.source

@@ -28,17 +28,19 @@ ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 ARG TARGETARCH
 ARG RUSTFS_BUILD_FEATURES=""
+ARG RC_RELEASE=latest
 
 # -----------------------------
 # Build stage
 # -----------------------------
-FROM rust:1.97.1-trixie AS builder
+FROM rust:1.99.1-trixie AS builder
 
 # Re-declare args after FROM
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 ARG TARGETARCH
 ARG RUSTFS_BUILD_FEATURES
+ARG RC_RELEASE
 
 # Debug: print platforms
 RUN echo "Build info -> BUILDPLATFORM=${BUILDPLATFORM}, TARGETPLATFORM=${TARGETPLATFORM}"
@@ -52,6 +54,7 @@ RUN set -eux; \
     build-essential \
     ca-certificates \
     curl \
+    jq \
     git \
     pkg-config \
     libssl-dev \
@@ -166,6 +169,8 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
     ;; \
     esac
 
+RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
+
 # -----------------------------
 # Development stage (keeps toolchain)
 # -----------------------------
@@ -199,10 +204,13 @@ ENV RUSTFS_VOLUMES="/data" \
 # We rely on cargo run to build and run
 EXPOSE 9000 9001
 
-COPY entrypoint.sh /entrypoint.sh
-RUN chmod +x /entrypoint.sh
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
+RUN install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
+    chmod +x /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 CMD ["cargo", "run", "--bin", "rustfs", "--"]
 
 # -----------------------------
@@ -227,6 +235,7 @@ RUN set -eux; \
     apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
+    jq \
     tzdata \
     coreutils; \
     rm -rf /var/lib/apt/lists/*
@@ -248,8 +257,11 @@ RUN set -eux; \
 
 # Copy the freshly built binary and the entrypoint
 COPY --from=builder /usr/local/bin/rustfs /usr/bin/rustfs
-COPY entrypoint.sh /entrypoint.sh
-RUN chmod +x /usr/bin/rustfs /entrypoint.sh
+COPY --from=builder /usr/local/bin/rc /usr/bin/rc
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
+RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 # Default environment (override in docker run/compose as needed)
 ENV RUSTFS_VOLUMES="/data" \
@@ -263,5 +275,5 @@ EXPOSE 9000
 VOLUME ["/data"]
 
 # Keep root here; entrypoint will drop privileges using chroot --userspec
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 CMD ["/usr/bin/rustfs"]

bootstrap/README.md

@@ -0,0 +1,21 @@
+# RustFS image bootstrap
+
+The official image bundles this directory at `/etc/rustfs/bootstrap`. The default `buckets.json` is an empty JSON array, so the image starts RustFS directly and does not require a bootstrap endpoint or bootstrap credentials.
+
+To create buckets, replace `buckets.json` in a derived image with a JSON array of bucket names. The entrypoint creates each bucket if it does not exist and applies `public-access-block.json`, whose default blocks all four public access paths. Edit that file in the derived image only when public access is an explicit requirement.
+
+Optional per-bucket files use the bucket name as the filename without an extension:
+
+| Directory | File format | Applied operation |
+| --- | --- | --- |
+| `policies/` | S3 bucket policy JSON | `rc bucket anonymous set-json` |
+| `versioning/` | JSON boolean (`true` or `false`) | Enable or suspend bucket versioning |
+| `lifecycle/` | `rc bucket lifecycle rule import` JSON | Replace lifecycle rules |
+| `cors/` | JSON or XML accepted by `rc bucket cors set` | Replace CORS rules |
+| `notifications/` | JSON array of `{ "arn": "...", "events": ["..."] }` | Add or replace event rules by ARN (prefix/suffix filters are not currently accepted) |
+
+Every per-bucket file must correspond to a bucket listed in `buckets.json`. Applying these files is repeatable: bucket creation ignores existing buckets, while each configuration operation replaces or reconciles that bucket's requested state.
+
+When `buckets.json` is non-empty, set `RUSTFS_BOOTSTRAP_ENDPOINT` to the RustFS S3 endpoint reachable from the container, for example `http://rustfs:9000` for a Compose service or the internal service DNS name in Kubernetes. The entrypoint waits on that endpoint's `/health/ready`; it never assumes `localhost`. `RUSTFS_BOOTSTRAP_TIMEOUT` controls the readiness wait in seconds (default `120`, maximum `3600`). `RUSTFS_BOOTSTRAP_DIR` can point to a different in-image configuration directory; it defaults to `/etc/rustfs/bootstrap`. Set `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY`, or their `_FILE` variants, for bootstrap authentication. The endpoint and credentials are read at runtime and are not baked into the image.
+
+RustFS built-in IAM policies remain unchanged; the default bootstrap installs no custom bucket policies, lifecycle rules, versioning, CORS rules, or notification rules. The bundled RustFS CLI is resolved from a published release and its release checksum is verified. Set the Docker build argument `RC_RELEASE` to a version to pin it. Build the RustFS image after a CLI release containing the `bucket public-access-block` commands is available. For Kubernetes, the configured service must route traffic to the starting pod before its readiness probe succeeds (for example, with `publishNotReadyAddresses: true`); otherwise the service endpoint cannot reach the pod while the entrypoint waits for readiness.

bootstrap/buckets.json

@@ -0,0 +1 @@
+[]

bootstrap/cors/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/lifecycle/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/notifications/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/policies/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/public-access-block.json

@@ -0,0 +1,6 @@
+{
+  "block_public_acls": true,
+  "ignore_public_acls": true,
+  "block_public_policy": true,
+  "restrict_public_buckets": true
+}

bootstrap/versioning/.gitkeep

@@ -0,0 +1 @@
+

crates/config/src/constants/object.rs

@@ -32,8 +32,9 @@ pub const ENV_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD: &str = "RUSTFS_OBJECT_MEDIUM_
 
 /// Environment variable name for maximum concurrent disk reads for object operations.
 /// - Purpose: Limit the number of concurrent disk read operations for object reads to prevent I/O saturation.
-/// - Unit: request count (usize).
-/// - Semantics: Throttling disk reads helps maintain overall system responsiveness under load.
+/// - Unit: request count (usize) for GET; estimated drive fanout for native LIST.
+/// - Semantics: GET bodies and native LIST walks use separate per-instance budgets of this size.
+///   A LIST consumes up to one budget, including its disk-producer cleanup. `0` disables both caps.
 /// - Example: `export RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=16`
 /// - Note: This setting may interact with OS-level I/O scheduling and should be tuned based on hardware capabilities.
 pub const ENV_OBJECT_MAX_CONCURRENT_DISK_READS: &str = "RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS";

crates/config/src/constants/proxy.rs

@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
 /// Default validation mode is "hop_by_hop".
 pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
 
-/// Environment variable to enable RFC 7239 "Forwarded" header support.
+/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
 pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
-/// RFC 7239 support is enabled by default.
-pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
+/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
+pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
 
 /// Environment variable for the maximum allowed proxy hops.
 pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";

crates/config/src/constants/targets.rs

@@ -146,3 +146,9 @@ pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
 
 /// Queue-store compression is enabled by default to reduce disk footprint.
 pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
+
+/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
+pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
+
+/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
+pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;

crates/config/src/constants/tls.rs

@@ -175,12 +175,17 @@ pub const ENV_HTTP_SOCKET_RECV_BUFFER_BYTES: &str = "RUSTFS_HTTP_SOCKET_RECV_BUF
 pub const DEFAULT_HTTP_SOCKET_RECV_BUFFER_BYTES: usize = 0;
 
 /// Environment variable for the S3 request-body inter-chunk read timeout
-/// (seconds). Default: 300. Set to 0 to disable.
+/// (seconds). Default: 300. Set to 0 to disable the inactivity timeout.
 ///
 /// This bounds how long a `PutObject`/upload may wait for the *next* body chunk
 /// while more bytes are still expected. It resets on every chunk, so it does not
 /// penalize slow-but-progressing uploads — it only fires when a peer sends a
 /// partial body and then goes silent *without* closing the connection (no EOF).
+/// Independently of this knob, `PutObject` and `UploadPart` require at least
+/// 64 KiB of body progress per 60 seconds of server wait (the minimum-progress
+/// floor, logged as `upload_body_progress_stalled`); it bounds trickle uploads
+/// that would otherwise pin upload admission indefinitely, and it stays in
+/// force when this timeout is set to 0.
 /// A reverse proxy or CDN that forwards a truncated body this way would
 /// otherwise make RustFS wait forever for bytes that never arrive; with this
 /// timeout the stalled read is aborted and logged with the received/expected

crates/data-usage/src/data_usage.rs

@@ -675,6 +675,10 @@ impl DataUsageInfo {
 /// cycle. The timestamp fallback preserves ordering for legacy snapshots that
 /// predate those fields.
 pub fn data_usage_snapshot_is_newer(candidate: &DataUsageInfo, baseline: &DataUsageInfo) -> bool {
+    data_usage_identity_is_newer(candidate.snapshot_identity(), baseline.snapshot_identity())
+}
+
+fn data_usage_identity_is_newer(candidate: DataUsageSnapshotIdentity, baseline: DataUsageSnapshotIdentity) -> bool {
     match (
         candidate.scanner_epoch.zip(candidate.scanner_cycle),
         baseline.scanner_epoch.zip(baseline.scanner_cycle),
@@ -697,10 +701,18 @@ pub fn data_usage_snapshot_is_newer(candidate: &DataUsageInfo, baseline: &DataUs
 /// already advance the authoritative timestamp when deleting a bucket, so a
 /// rollback delete/recreate fences the previous bucket incarnation too.
 pub fn observed_data_usage_is_newer(observed: &DataUsageInfo, authoritative: &DataUsageInfo) -> bool {
+    observed_data_usage_is_newer_than_identity(observed, authoritative.snapshot_identity())
+}
+
+/// Return whether a nonconverged observation may supersede `authoritative`.
+///
+/// This applies the structural, exact-baseline, and generation checks of
+/// [`observed_data_usage_is_newer`] without retaining the baseline's bucket maps.
+pub fn observed_data_usage_is_newer_than_identity(observed: &DataUsageInfo, authoritative: DataUsageSnapshotIdentity) -> bool {
     observed.usage_snapshot_converged == Some(false)
         && (observed.is_complete_bucket_usage_snapshot() || observed.is_valid_partial_snapshot())
-        && observed.usage_snapshot_authoritative_baseline.as_ref() == Some(&authoritative.snapshot_identity())
-        && data_usage_snapshot_is_newer(observed, authoritative)
+        && observed.usage_snapshot_authoritative_baseline == Some(authoritative)
+        && data_usage_identity_is_newer(observed.snapshot_identity(), authoritative)
 }
 
 /// Metadata describing the status of a disk-level data usage snapshot.
@@ -3140,12 +3152,17 @@ mod tests {
         };
 
         assert!(observed_data_usage_is_newer(&observed, &authoritative));
+        assert!(observed_data_usage_is_newer_than_identity(&observed, authoritative.snapshot_identity()));
 
         authoritative.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(601));
         assert!(
             !observed_data_usage_is_newer(&observed, &authoritative),
             "an old-binary namespace mutation must fence the prior bucket incarnation regardless of clock skew"
         );
+        assert!(
+            !observed_data_usage_is_newer_than_identity(&observed, authoritative.snapshot_identity()),
+            "a captured identity must preserve the exact namespace fence"
+        );
     }
 
     #[test]
@@ -3172,6 +3189,18 @@ mod tests {
         assert!(!observed_data_usage_is_newer(&candidate(2, 9, Some(false), true), &authoritative));
         assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(true), true), &authoritative));
         assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(false), false), &authoritative));
+        for (observed, expected) in [
+            (candidate(2, 11, Some(false), true), true),
+            (candidate(2, 9, Some(false), true), false),
+            (candidate(2, 11, Some(true), true), false),
+            (candidate(2, 11, Some(false), false), false),
+        ] {
+            assert_eq!(
+                observed_data_usage_is_newer_than_identity(&observed, authoritative.snapshot_identity()),
+                expected,
+                "observation identity selection must preserve convergence, completeness, and generation gates"
+            );
+        }
 
         let mut partial = candidate(2, 11, Some(false), false);
         partial.usage_snapshot_partial = true;
@@ -3186,6 +3215,30 @@ mod tests {
             segment_invalidation_proof: None,
         }];
         assert!(observed_data_usage_is_newer(&partial, &authoritative));
+        assert!(observed_data_usage_is_newer_than_identity(&partial, authoritative.snapshot_identity()));
+    }
+
+    #[test]
+    fn observation_identity_selection_keeps_legacy_timestamp_ordering() {
+        let baseline = DataUsageSnapshotIdentity {
+            last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
+            // An epoch without a cycle is not a coordinated position.
+            scanner_epoch: Some(2),
+            scanner_cycle: None,
+        };
+        let mut observed = DataUsageInfo {
+            last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(11)),
+            scanner_epoch: Some(1),
+            usage_snapshot_complete: true,
+            usage_snapshot_converged: Some(false),
+            usage_snapshot_authoritative_baseline: Some(baseline),
+            ..Default::default()
+        };
+        assert!(observed_data_usage_is_newer_than_identity(&observed, baseline));
+        observed.last_update = baseline.last_update;
+        assert!(!observed_data_usage_is_newer_than_identity(&observed, baseline));
+        observed.last_update = None;
+        assert!(!observed_data_usage_is_newer_than_identity(&observed, baseline));
     }
 
     #[test]

crates/e2e_test/README.md

@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
 | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
 | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
-| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
+| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
 
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
 | `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
 | ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
-| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
+| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
-| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
+| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
 | Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |

crates/e2e_test/src/conditional_write_policy_test.rs

@@ -260,7 +260,7 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
     let key = "guarded.txt";
 
     admin_create_user(env, &user, user_secret).await?;
-    let policy = serde_json::json!({
+    let mut policy = serde_json::json!({
         "Version": "2012-10-17",
         "Statement": [{
             "Effect": "Allow",
@@ -268,9 +268,8 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
             "Resource": [format!("arn:aws:s3:::{bucket}/*")],
             "Condition": { "Null": { "s3:if-match": "false" } }
         }]
-    })
-    .to_string();
-    admin_add_and_attach_policy(env, &policy_name, &policy, &user).await?;
+    });
+    admin_add_and_attach_policy(env, &policy_name, &policy.to_string(), &user).await?;
 
     admin.create_bucket().bucket(&bucket).send().await?;
     let original = admin
@@ -293,6 +292,46 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
         .expect_err("PutObject without If-Match must be denied");
     assert_error(&denied, "AccessDenied", 403, "unconditional PutObject");
 
+    let denied = client
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .if_match(&etag)
+        .body(ByteStream::from_static(b"missing-read-permission"))
+        .send()
+        .await
+        .expect_err("If-Match without GetObject must be denied");
+    assert_error(&denied, "AccessDenied", 403, "conditional PutObject without GetObject");
+
+    // S3 requires GetObject as well as PutObject for an ETag-based write.
+    policy["Statement"]
+        .as_array_mut()
+        .expect("policy statements")
+        .push(serde_json::json!({
+            "Effect": "Allow",
+            "Action": ["s3:GetObject"],
+            "Resource": [format!("arn:aws:s3:::{bucket}/{key}")]
+        }));
+    admin_add_canned_policy_via(
+        AdminTransport::Signed,
+        &env.url,
+        &env.access_key,
+        &env.secret_key,
+        &policy_name,
+        &policy.to_string(),
+    )
+    .await?;
+
+    let denied = client
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"blind-overwrite"))
+        .send()
+        .await
+        .expect_err("GetObject permission must not allow an unconditional PutObject");
+    assert_error(&denied, "AccessDenied", 403, "unconditional PutObject with GetObject");
+
     client
         .put_object()
         .bucket(&bucket)
@@ -301,6 +340,8 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
         .body(ByteStream::from_static(b"v2"))
         .send()
         .await?;
+    let written = client.get_object().bucket(&bucket).key(key).send().await?;
+    assert_eq!(written.body.collect().await?.into_bytes().as_ref(), b"v2");
 
     let _ = admin.delete_object().bucket(&bucket).key(key).send().await;
     let _ = admin.delete_bucket().bucket(&bucket).send().await;

crates/e2e_test/src/create_bucket_region_test.rs

@@ -502,10 +502,163 @@ mod tests {
                 assert!(response.contains(if expected == reqwest::StatusCode::FORBIDDEN {
                     "<Code>AccessDenied</Code>"
                 } else {
-                    "<Code>BucketAlreadyExists</Code>"
+                    "<Code>BucketAlreadyOwnedByYou</Code>"
                 }));
             }
         }
         Ok(())
     }
+
+    /// A deployment owns a single bucket namespace, so an IAM user that recreates
+    /// an existing bucket is the owner, not another account. In the default region
+    /// every such request returns the legacy idempotent `200 OK` (AWS CreateBucket
+    /// semantics), including buckets created by root and by another IAM user.
+    #[tokio::test]
+    async fn test_create_bucket_duplicate_in_default_region_is_idempotent_for_iam_user()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec!["--region", "us-east-1"]).await?;
+        let root = env.create_s3_client();
+
+        admin_create_user(&env, "duplicate-iam", "duplicate-iam-secret").await?;
+        admin_attach_user_policy_via(
+            AdminTransport::Signed,
+            &env.url,
+            &env.access_key,
+            &env.secret_key,
+            "readwrite",
+            "duplicate-iam",
+        )
+        .await?;
+        let iam = Client::from_conf(build_test_s3_config(
+            &env.url,
+            "duplicate-iam",
+            "duplicate-iam-secret",
+            None,
+            "duplicate-create",
+        ));
+
+        root.create_bucket()
+            .bucket("duplicate-root-owned")
+            .send()
+            .await
+            .expect("root creates the first bucket");
+        iam.create_bucket()
+            .bucket("duplicate-root-owned")
+            .send()
+            .await
+            .expect("an IAM user recreating a root-owned bucket in us-east-1 gets 200 OK");
+
+        iam.create_bucket()
+            .bucket("duplicate-iam-owned")
+            .send()
+            .await
+            .expect("the IAM user creates its own bucket");
+        iam.create_bucket()
+            .bucket("duplicate-iam-owned")
+            .send()
+            .await
+            .expect("the same IAM user recreating its bucket gets 200 OK");
+        root.create_bucket()
+            .bucket("duplicate-iam-owned")
+            .send()
+            .await
+            .expect("root recreating an IAM-created bucket gets 200 OK");
+
+        Ok(())
+    }
+
+    /// Outside the default region the same ownership answer becomes
+    /// `409 BucketAlreadyOwnedByYou` for every authorized requester. Requests that
+    /// lack `s3:CreateBucket` still fail authorization before the duplicate check.
+    #[tokio::test]
+    async fn test_create_bucket_duplicate_outside_default_region_reports_owned_by_you() -> Result<(), Box<dyn Error + Send + Sync>>
+    {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec!["--region", "eu-west-1"]).await?;
+        let region = "eu-west-1";
+        let root = Client::from_conf(
+            env.create_s3_client()
+                .config()
+                .to_builder()
+                .region(Region::new(region))
+                .build(),
+        );
+
+        admin_create_user(&env, "duplicate-eu-iam", "duplicate-eu-iam-secret").await?;
+        admin_attach_user_policy_via(
+            AdminTransport::Signed,
+            &env.url,
+            &env.access_key,
+            &env.secret_key,
+            "readwrite",
+            "duplicate-eu-iam",
+        )
+        .await?;
+        let iam = Client::from_conf(
+            build_test_s3_config(&env.url, "duplicate-eu-iam", "duplicate-eu-iam-secret", None, "duplicate-create")
+                .to_builder()
+                .region(Region::new(region))
+                .build(),
+        );
+
+        admin_create_user(&env, "duplicate-eu-readonly", "duplicate-eu-readonly-secret").await?;
+        admin_attach_user_policy_via(
+            AdminTransport::Signed,
+            &env.url,
+            &env.access_key,
+            &env.secret_key,
+            "readonly",
+            "duplicate-eu-readonly",
+        )
+        .await?;
+        let readonly = Client::from_conf(
+            build_test_s3_config(
+                &env.url,
+                "duplicate-eu-readonly",
+                "duplicate-eu-readonly-secret",
+                None,
+                "duplicate-create",
+            )
+            .to_builder()
+            .region(Region::new(region))
+            .build(),
+        );
+
+        for (client, bucket) in [(&root, "duplicate-eu-root"), (&iam, "duplicate-eu-iam")] {
+            client
+                .create_bucket()
+                .bucket(bucket)
+                .send()
+                .await
+                .unwrap_or_else(|error| panic!("{bucket} must be created: {error:?}"));
+            // Both the creator and the other authenticated identity own the namespace.
+            for creator in [&root, &iam] {
+                let error = creator
+                    .create_bucket()
+                    .bucket(bucket)
+                    .send()
+                    .await
+                    .expect_err("a duplicate create outside us-east-1 must report ownership");
+                assert_eq!(
+                    error.as_service_error().and_then(ProvideErrorMetadata::code),
+                    Some("BucketAlreadyOwnedByYou"),
+                    "{bucket}"
+                );
+                assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(409), "{bucket}");
+            }
+        }
+
+        let denied = readonly
+            .create_bucket()
+            .bucket("duplicate-eu-root")
+            .send()
+            .await
+            .expect_err("a requester without s3:CreateBucket must be denied before the duplicate check");
+        assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
+
+        Ok(())
+    }
 }

crates/e2e_test/src/delete_marker_migration_semantics_test.rs

@@ -16,8 +16,193 @@
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
     use aws_sdk_s3::Client;
+    use aws_sdk_s3::config::http::HttpResponse;
+    use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
     use aws_sdk_s3::primitives::ByteStream;
-    use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+    use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
+
+    fn assert_not_found<E: ProvideErrorMetadata + std::fmt::Debug>(error: &SdkError<E, HttpResponse>) {
+        assert_eq!(error.raw_response().map(|r| r.status().as_u16()), Some(404), "{error:?}");
+        assert_eq!(
+            error.as_service_error().and_then(ProvideErrorMetadata::code),
+            Some("NoSuchKey"),
+            "{error:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn conditional_writes_cannot_replace_delete_marker_with_if_match() {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
+        env.start_rustfs_server(vec![]).await.expect("start RustFS");
+        let client = env.create_s3_client();
+        let bucket = "conditional-delete-marker-proof";
+        create_versioned_bucket(&client, bucket).await;
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("source")
+            .body(ByteStream::from_static(b"source"))
+            .send()
+            .await
+            .expect("write copy source");
+
+        for operation in ["put", "copy", "multipart"] {
+            let old = client
+                .put_object()
+                .bucket(bucket)
+                .key(operation)
+                .body(ByteStream::from_static(b"old"))
+                .send()
+                .await
+                .expect("write historical version");
+            let old_etag = old.e_tag().expect("historical ETag");
+            let marker = client
+                .delete_object()
+                .bucket(bucket)
+                .key(operation)
+                .send()
+                .await
+                .expect("create delete marker");
+            let marker_id = marker.version_id().expect("delete marker version");
+
+            let (upload_id, completed) = if operation == "multipart" {
+                let upload = client
+                    .create_multipart_upload()
+                    .bucket(bucket)
+                    .key(operation)
+                    .send()
+                    .await
+                    .expect("initiate multipart upload");
+                let upload_id = upload.upload_id().expect("upload ID").to_owned();
+                let part = client
+                    .upload_part()
+                    .bucket(bucket)
+                    .key(operation)
+                    .upload_id(&upload_id)
+                    .part_number(1)
+                    .body(ByteStream::from_static(b"new"))
+                    .send()
+                    .await
+                    .expect("upload part");
+                let completed = CompletedMultipartUpload::builder()
+                    .parts(
+                        CompletedPart::builder()
+                            .part_number(1)
+                            .set_e_tag(part.e_tag().map(str::to_owned))
+                            .build(),
+                    )
+                    .build();
+                (Some(upload_id), Some(completed))
+            } else {
+                (None, None)
+            };
+
+            for etag in [old_etag, "\"00000000000000000000000000000000\""] {
+                match operation {
+                    "put" => {
+                        let err = client
+                            .put_object()
+                            .bucket(bucket)
+                            .key(operation)
+                            .if_match(etag)
+                            .body(ByteStream::from_static(b"new"))
+                            .send()
+                            .await
+                            .expect_err("If-Match must reject marker");
+                        assert_not_found(&err);
+                    }
+                    "copy" => {
+                        let err = client
+                            .copy_object()
+                            .bucket(bucket)
+                            .key(operation)
+                            .copy_source(format!("{bucket}/source"))
+                            .if_match(etag)
+                            .send()
+                            .await
+                            .expect_err("If-Match copy must reject marker");
+                        assert_not_found(&err);
+                    }
+                    _ => {
+                        let err = client
+                            .complete_multipart_upload()
+                            .bucket(bucket)
+                            .key(operation)
+                            .upload_id(upload_id.as_deref().expect("upload ID"))
+                            .multipart_upload(completed.clone().expect("completed parts"))
+                            .if_match(etag)
+                            .send()
+                            .await
+                            .expect_err("If-Match completion must reject marker");
+                        assert_not_found(&err);
+                    }
+                }
+                let versions = client
+                    .list_object_versions()
+                    .bucket(bucket)
+                    .prefix(operation)
+                    .send()
+                    .await
+                    .expect("list versions after rejected write");
+                assert_eq!(versions.versions().len(), 1, "{operation}: failed write created a version");
+                assert_eq!(versions.delete_markers().len(), 1);
+                assert_eq!(versions.delete_markers()[0].version_id(), Some(marker_id));
+                assert_eq!(versions.delete_markers()[0].is_latest(), Some(true));
+            }
+
+            match operation {
+                "put" => {
+                    client
+                        .put_object()
+                        .bucket(bucket)
+                        .key(operation)
+                        .if_none_match("*")
+                        .body(ByteStream::from_static(b"new"))
+                        .send()
+                        .await
+                        .expect("If-None-Match PUT over marker");
+                }
+                "copy" => {
+                    client
+                        .copy_object()
+                        .bucket(bucket)
+                        .key(operation)
+                        .copy_source(format!("{bucket}/source"))
+                        .if_none_match("*")
+                        .send()
+                        .await
+                        .expect("If-None-Match copy over marker");
+                }
+                _ => {
+                    client
+                        .complete_multipart_upload()
+                        .bucket(bucket)
+                        .key(operation)
+                        .upload_id(upload_id.as_deref().expect("upload ID"))
+                        .multipart_upload(completed.expect("completed parts"))
+                        .if_none_match("*")
+                        .send()
+                        .await
+                        .expect("If-None-Match completion over marker");
+                }
+            }
+            let current = client
+                .get_object()
+                .bucket(bucket)
+                .key(operation)
+                .send()
+                .await
+                .expect("read successful conditional write")
+                .body
+                .collect()
+                .await
+                .expect("read object body")
+                .into_bytes();
+            let expected: &[u8] = if operation == "copy" { b"source" } else { b"new" };
+            assert_eq!(current.as_ref(), expected, "{operation}: unexpected current object");
+        }
+    }
 
     async fn create_versioned_bucket(client: &Client, bucket: &str) {
         client

crates/e2e_test/src/distributed/concurrency_stability_test.rs

@@ -12,11 +12,249 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
+use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
 use crate::common::init_logging;
+use aws_sdk_s3::{
+    Client,
+    error::{ProvideErrorMetadata, SdkError},
+    primitives::ByteStream,
+};
+use std::any::TypeId;
 use std::collections::BTreeSet;
+use std::fmt::Debug;
+use std::future::Future;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::sync::Barrier;
+use tokio::time::Instant;
+
+const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
+
+async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
+where
+    E: Debug + ProvideErrorMetadata + 'static,
+    F: FnMut() -> Fut,
+    Fut: Future<Output = Result<T, SdkError<E>>>,
+{
+    let started = Instant::now();
+    let mut attempts = 0;
+    // The workload uses unique unversioned keys. Repeat only explicitly
+    // advertised lock backpressure; every step still needs its real result.
+    let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
+        loop {
+            attempts += 1;
+            let result = call().await;
+            let retry = result.as_ref().err().is_some_and(|error| {
+                let Some(response) = error.raw_response() else {
+                    return false;
+                };
+                let Some(service_error) = error.as_service_error() else {
+                    return false;
+                };
+                // HEAD has no XML error body, so its SDK service error can
+                // lack a code even when the server supplies retry headers.
+                let namespace_code = service_error.code() == Some("ServiceUnavailable")
+                    || (service_error.code().is_none()
+                        && TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
+                response.status().as_u16() == 503
+                    && namespace_code
+                    && response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
+                    && response.headers().get("retry-after") == Some("1")
+            });
+            if !retry {
+                return result;
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+    })
+    .await;
+    if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
+        eprintln!(
+            "mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
+            started.elapsed()
+        );
+    }
+    outcome.map_err(|_| {
+        format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
+    })
+}
+
+async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
+    let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
+    let actual = output.body.collect().await?.into_bytes();
+    if actual.as_ref() != expected {
+        return Err(format!(
+            "object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
+            expected.len(),
+            sha256_hex(expected),
+            actual.len(),
+            sha256_hex(&actual)
+        )
+        .into());
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use aws_smithy_http_client::test_util::infallible_client_fn;
+    use aws_smithy_types::body::SdkBody;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
+
+    fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
+        let requests = Arc::new(AtomicUsize::new(0));
+        let seen = Arc::clone(&requests);
+        let transport = infallible_client_fn(move |_| {
+            let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
+            let body = if code.is_empty() {
+                body.to_string()
+            } else {
+                format!("<Error><Code>{code}</Code></Error>")
+            };
+            let mut response = http::Response::builder()
+                .status(status)
+                .header("content-type", "application/xml")
+                .header("content-length", body.len());
+            if let Some(cause) = cause {
+                response = response.header("x-rustfs-error-cause", cause);
+            }
+            if let Some(retry_after) = retry_after {
+                response = response.header("retry-after", retry_after);
+            }
+            response.body(SdkBody::from(body)).expect("scripted S3 response")
+        });
+        let config = crate::common::build_test_s3_config(
+            "http://contention.example.invalid",
+            "synthetic-access",
+            "synthetic-secret",
+            None,
+            "contention-test",
+        );
+        let client = Client::from_conf(config.to_builder().http_client(transport).build());
+        (s3_probe_client(&client), requests)
+    }
+
+    fn contention() -> Reply {
+        (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
+        let (client, requests) = client(|attempt| {
+            if attempt < 3 {
+                contention()
+            } else {
+                (200, "", None, None, "")
+            }
+        });
+        let standard = Client::from_conf(
+            client
+                .config()
+                .to_builder()
+                .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
+                .build(),
+        );
+        standard
+            .put_object()
+            .bucket("bucket")
+            .key("key")
+            .body(ByteStream::from_static(b"body"))
+            .send()
+            .await
+            .expect_err("three advertised lock responses exhaust the standard SDK policy");
+        assert_eq!(requests.load(Ordering::SeqCst), 3);
+        requests.store(0, Ordering::SeqCst);
+        let started = Instant::now();
+        after_lock_contention("PUT recovery", || {
+            client
+                .put_object()
+                .bucket("bucket")
+                .key("key")
+                .body(ByteStream::from_static(b"body"))
+                .send()
+        })
+        .await??;
+        assert_eq!(requests.load(Ordering::SeqCst), 4);
+        assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
+        let (client, requests) = client(|attempt| {
+            if attempt == 0 {
+                (503, "", Some("namespace_lock_timeout"), Some("1"), "")
+            } else {
+                (200, "", None, None, "")
+            }
+        });
+        after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
+        assert_eq!(requests.load(Ordering::SeqCst), 2);
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
+        for reply in [
+            (500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
+            (403, "AccessDenied", None, None, ""),
+            (404, "NoSuchKey", None, None, ""),
+            (503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
+            (503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
+            (503, "", Some("namespace_lock_timeout"), Some("1"), ""),
+            (503, "ServiceUnavailable", None, Some("1"), ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
+        ] {
+            let (client, requests) = client(move |_| reply);
+            let result = after_lock_contention("PUT fatal response", || {
+                client
+                    .put_object()
+                    .bucket("bucket")
+                    .key("key")
+                    .body(ByteStream::from_static(b"body"))
+                    .send()
+            })
+            .await?;
+            assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
+            assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
+        }
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn persistent_lock_contention_exhausts_the_request_budget() {
+        let (client, requests) = client(|_| contention());
+        let started = Instant::now();
+        let result = after_lock_contention("PUT unavailable", || {
+            client
+                .put_object()
+                .bucket("bucket")
+                .key("key")
+                .body(ByteStream::from_static(b"body"))
+                .send()
+        })
+        .await;
+        assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
+        assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
+        assert!(requests.load(Ordering::SeqCst) > 1);
+        assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn acknowledged_wrong_bytes_fail_without_retry() {
+        let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
+        let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
+            .await
+            .expect_err("a successful but incorrect read is a consistency failure");
+        assert!(error.to_string().contains("bytes mismatch"));
+        assert_eq!(requests.load(Ordering::SeqCst), 1);
+    }
+}
 
 #[tokio::test]
 async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
@@ -24,7 +262,9 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     let dist = DistCluster::start(DistLayout::FourByFour).await?;
     let bucket = unique_bucket("conc");
     dist.create_bucket(&bucket).await?;
-    let clients = Arc::new(dist.clients()?);
+    // This consistency probe owns its admission policy; unrelated responses
+    // must not be hidden by the SDK's general-purpose retries.
+    let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
     let worker_count = 24;
     let rounds = 4;
     let barrier = Arc::new(Barrier::new(worker_count));
@@ -44,31 +284,46 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
                 let key = format!("source/worker-{idx:02}-round-{round}.bin");
                 let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
                 let body = payload_for(&key, 64 * 1024);
-                put_object(writer, &bucket, &key, body.clone()).await?;
+                let context = format!("worker={idx} round={round}");
+                after_lock_contention(&format!("{context} PUT source"), || {
+                    writer
+                        .put_object()
+                        .bucket(&bucket)
+                        .key(&key)
+                        .body(ByteStream::from(body.clone()))
+                        .send()
+                })
+                .await??;
 
-                let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
+                let head = after_lock_contention(&format!("{context} HEAD source"), || {
+                    reader.head_object().bucket(&bucket).key(&key).send()
+                })
+                .await??;
                 if head.content_length() != Some(body.len() as i64) {
                     return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
                 }
-                assert_object_bytes(reader, &bucket, &key, &body).await?;
+                assert_workload_bytes(reader, &bucket, &key, &body).await?;
 
-                copier
-                    .copy_object()
-                    .bucket(&bucket)
-                    .key(&copy_key)
-                    .copy_source(format!("{bucket}/{key}"))
-                    .send()
-                    .await?;
-                assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
+                after_lock_contention(&format!("{context} COPY retained"), || {
+                    copier
+                        .copy_object()
+                        .bucket(&bucket)
+                        .key(&copy_key)
+                        .copy_source(format!("{bucket}/{key}"))
+                        .send()
+                })
+                .await??;
+                assert_workload_bytes(writer, &bucket, &copy_key, &body).await?;
 
-                writer.delete_object().bucket(&bucket).key(&key).send().await?;
-                let missing = reader
-                    .head_object()
-                    .bucket(&bucket)
-                    .key(&key)
-                    .send()
-                    .await
-                    .expect_err("deleted source key must not remain visible");
+                after_lock_contention(&format!("{context} DELETE source"), || {
+                    writer.delete_object().bucket(&bucket).key(&key).send()
+                })
+                .await??;
+                let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
+                    reader.head_object().bucket(&bucket).key(&key).send()
+                })
+                .await?
+                .expect_err("deleted source key must not remain visible");
                 if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
                     return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
                 }
@@ -82,14 +337,22 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     for handle in handles {
         inventory.extend(handle.await??);
     }
+    assert_eq!(
+        inventory.len(),
+        worker_count * rounds,
+        "every worker round must retain its acknowledged copy"
+    );
 
     let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
     for (node_idx, client) in clients.iter().enumerate() {
-        let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
+        let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
+            client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
+        })
+        .await??;
         let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
         assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
         for (key, body) in &inventory {
-            assert_object_bytes(client, &bucket, key, body)
+            assert_workload_bytes(client, &bucket, key, body)
                 .await
                 .map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
         }

crates/e2e_test/src/distributed/scanner_heal_topology_fixture.rs

@@ -25,7 +25,8 @@
 
 use super::harness::{
     DistCluster, DistLayout, TestResult, cluster_admin, cluster_admin_ok, decommission_status_json, get_object_bytes,
-    payload_for, put_object, s3_probe_client, s3_probe_error_code, sha256_hex, unique_bucket, wait_until,
+    payload_for, put_object, rebalance_complete, rebalance_status_json, s3_probe_client, s3_probe_error_code, sha256_hex,
+    unique_bucket, wait_until,
 };
 use crate::chaos::{VersionShardCensus, census_object_version_on_disk};
 use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment};
@@ -47,6 +48,7 @@ use tokio::time::{Instant, sleep};
 use uuid::Uuid;
 
 pub(crate) const MOVEMENT_BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
+pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
 pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
 pub(crate) const STAGE_BEFORE_SOURCE_CLEANUP: &str = "before_source_cleanup";
 pub(crate) const STAGE_AFTER_RESPONSIBILITY_PERSISTED: &str = "after_responsibility_persisted";
@@ -1166,8 +1168,21 @@ pub(crate) async fn clear_decommission(cluster: &RustFSTestClusterEnvironment, p
     .await
 }
 
-pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment) -> TestResult<String> {
-    cluster_admin_ok(cluster, Method::POST, "/rustfs/admin/v3/rebalance/stop", None).await
+pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment, expected_id: &str) -> TestResult<String> {
+    let path = "/rustfs/admin/v3/rebalance/stop";
+    let (status, response) = cluster_admin(cluster, Method::POST, path, None).await?;
+    if status.is_success() {
+        return Ok(response);
+    }
+    // The run can finish between the active observation and the stop request.
+    // Only completion of that same run makes a not-started response acceptable.
+    if status == StatusCode::NOT_FOUND
+        && response.contains("<Code>NoSuchResource</Code>")
+        && rebalance_complete(&rebalance_status_json(cluster).await?, expected_id)?
+    {
+        return Ok(response);
+    }
+    Err(format!("POST {path} failed: {status} {response}").into())
 }
 
 pub(crate) fn decommission_pool_entry(status: &Value, pool_id: usize) -> TestResult<Value> {
@@ -1585,6 +1600,129 @@ pub(crate) async fn assert_every_node_serves(dist: &DistCluster, bucket: &str, l
 mod tests {
     use super::*;
 
+    const REBALANCE_NOT_STARTED: &str =
+        "<Error><Code>NoSuchResource</Code><Message>pool rebalance is not started</Message></Error>";
+
+    async fn mock_rebalance_stop(
+        stop_status: StatusCode,
+        stop_body: &str,
+        run_status: Value,
+    ) -> TestResult<(TestResult<String>, Vec<String>)> {
+        use bytes::Bytes;
+        use http_body_util::Full;
+        use hyper::service::service_fn;
+        use hyper_util::rt::TokioIo;
+        use tokio::net::TcpListener;
+        use tokio::sync::mpsc;
+        use tokio::task::JoinSet;
+
+        let listener = TcpListener::bind("127.0.0.1:0").await?;
+        let mut cluster = RustFSTestClusterEnvironment::new(1).await?;
+        cluster.nodes[0].url = format!("http://{}", listener.local_addr()?);
+        let stop_body = stop_body.to_string();
+        let (requests_tx, mut requests_rx) = mpsc::unbounded_channel();
+        let mut server = JoinSet::new();
+        server.spawn(async move {
+            loop {
+                let (stream, _) = match listener.accept().await {
+                    Ok(connection) => connection,
+                    Err(error) => return Err::<(), Box<dyn std::error::Error + Send + Sync>>(error.into()),
+                };
+                let stop_body = stop_body.clone();
+                let run_status = run_status.clone();
+                let requests_tx = requests_tx.clone();
+                let handler = service_fn(move |request: hyper::Request<hyper::body::Incoming>| {
+                    let route = format!("{} {}", request.method(), request.uri().path());
+                    requests_tx.send(route.clone()).expect("record mock admin request");
+                    let (status, body) = match route.as_str() {
+                        "POST /rustfs/admin/v3/rebalance/stop" => (stop_status, stop_body.clone()),
+                        "GET /rustfs/admin/v3/rebalance/status" => (StatusCode::OK, run_status.to_string()),
+                        _ => (StatusCode::BAD_REQUEST, "unexpected admin request".to_string()),
+                    };
+                    async move { http::Response::builder().status(status).body(Full::new(Bytes::from(body))) }
+                });
+                hyper::server::conn::http1::Builder::new()
+                    .keep_alive(false)
+                    .serve_connection(TokioIo::new(stream), handler)
+                    .await?;
+            }
+        });
+        let outcome = tokio::time::timeout(Duration::from_secs(5), stop_rebalance(&cluster, "run-1")).await?;
+        server.abort_all();
+        let requests = std::iter::from_fn(|| requests_rx.try_recv().ok()).collect();
+        Ok((outcome, requests))
+    }
+
+    fn completed_rebalance() -> Value {
+        serde_json::json!({
+            "id": "run-1",
+            "pools": [
+                { "id": 0, "status": "Completed", "progress": { "versions": 113 } },
+                { "id": 1, "status": "None" }
+            ]
+        })
+    }
+
+    #[tokio::test]
+    async fn rebalance_stop_success_does_not_query_completion() -> TestResult {
+        let (outcome, requests) = mock_rebalance_stop(StatusCode::OK, "", Value::Null).await?;
+        assert_eq!(outcome?, "");
+        assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn rebalance_stop_accepts_completed_run_after_not_started_response() -> TestResult {
+        let (outcome, requests) =
+            mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, completed_rebalance()).await?;
+        outcome?;
+        assert_eq!(
+            requests,
+            [
+                "POST /rustfs/admin/v3/rebalance/stop",
+                "GET /rustfs/admin/v3/rebalance/status"
+            ]
+        );
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn rebalance_stop_rejects_unproven_completion_and_other_errors() -> TestResult {
+        let complete = completed_rebalance();
+        let mut invalid = vec![Value::Null];
+        for (pointer, value) in [
+            ("/id", serde_json::json!("run-2")),
+            ("/pools/0/status", serde_json::json!("Started")),
+            ("/pools/0/status", serde_json::json!("Failed")),
+            ("/pools/0/status", serde_json::json!("Stopped")),
+            ("/pools/0/progress/versions", serde_json::json!(0)),
+        ] {
+            let mut status = complete.clone();
+            *status.pointer_mut(pointer).expect("completion fixture field") = value;
+            invalid.push(status);
+        }
+        let mut pending = complete.clone();
+        pending["stopPropagation"] = serde_json::json!({ "pendingTerminalReload": true });
+        invalid.push(pending);
+        let mut failed = complete.clone();
+        failed["pools"][0]["lastError"] = serde_json::json!("migration failed");
+        invalid.push(failed);
+        for status in invalid {
+            let (outcome, _) = mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, status.clone()).await?;
+            assert!(outcome.is_err(), "unproven completion accepted: {status}");
+        }
+        for (code, body) in [
+            (StatusCode::FORBIDDEN, REBALANCE_NOT_STARTED),
+            (StatusCode::INTERNAL_SERVER_ERROR, REBALANCE_NOT_STARTED),
+            (StatusCode::NOT_FOUND, "<Error><Code>NoSuchBucket</Code></Error>"),
+        ] {
+            let (outcome, requests) = mock_rebalance_stop(code, body, complete.clone()).await?;
+            assert!(outcome.is_err(), "unrelated error accepted: {code} {body}");
+            assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
+        }
+        Ok(())
+    }
+
     #[test]
     fn version_ids_normalize_null_spellings() {
         for spelling in [None, Some(""), Some("null"), Some("00000000-0000-0000-0000-000000000000")] {

crates/e2e_test/src/distributed/scanner_heal_topology_test.rs

@@ -28,7 +28,7 @@ use super::harness::{
     wait_for_rebalance_active, wait_for_rebalance_complete,
 };
 use super::scanner_heal_topology_fixture::{
-    LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL,
+    LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL, STAGE_AFTER_ENTRY_ADMISSION,
     STAGE_AFTER_RESPONSIBILITY_PERSISTED, STAGE_BEFORE_SOURCE_CLEANUP, STAGE_BEFORE_TARGET_COMMIT, TopologyManifest, TopologyRun,
     assert_bucket_durability_strict, assert_every_node_serves, assert_probe_objects, assert_process_durability_strict,
     assert_seed_pool_census_unchanged, assert_single_pool_placement, cancel_decommission, capture_seed_pool_census,
@@ -274,6 +274,7 @@ async fn wait_for_rebalance_stopped_or_completed(run: &TopologyRun, rebalance_id
 #[tokio::test]
 async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart() -> TestResult {
     init_logging();
+    require_test_hooks();
     let mut run = TopologyRun::start("rebalance").await?;
     let manifest = seed_manifest(&run).await?;
     assert_bucket_durability_strict(&run.dist.cluster, &manifest.buckets()).await?;
@@ -288,23 +289,52 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
         .map(|entry| (entry.key.clone(), entry.sha256.clone()))
         .collect();
 
-    let first_id = start_rebalance(&run.dist.cluster).await?;
-    wait_for_rebalance_active(&run.dist.cluster, &first_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
-    let during_rebalance = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
-    let workload = tokio::spawn(run_live_workload(LiveWorkload {
-        clients: run.dist.clients()?,
-        bucket: scratch.clone(),
-        prefix: "live".to_string(),
-        writers: LIVE_WRITERS,
-        rounds: LIVE_ROUNDS,
-        baseline_bucket: manifest.plain_bucket.clone(),
-        baseline: baseline.clone(),
-        pause: Duration::from_millis(100),
-    }));
-    sleep(Duration::from_secs(2)).await;
-    stop_rebalance(&run.dist.cluster).await?;
-    let (first_outcome, first_status) = wait_for_rebalance_stopped_or_completed(&run, &first_id).await?;
-    let ops = workload.await??;
+    // Keep an admitted entry on the source until stop cancels admission. Start
+    // propagation may await that entry, so observe and stop before awaiting ACK.
+    let first_key = manifest.plain_key("external", 0)?;
+    run.movement_barrier
+        .arm_movement("rebalance", &manifest.plain_bucket, &first_key, STAGE_AFTER_ENTRY_ADMISSION)?;
+    let observe_first = async {
+        let (observed_id, reached) = tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, async {
+            let reached = run.movement_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
+            let status = rebalance_status_json(&run.dist.cluster).await?;
+            let observed_id = status
+                .get("id")
+                .and_then(Value::as_str)
+                .ok_or("rebalance status omitted id")?;
+            wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
+        })
+        .await??;
+        let probes = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
+        let workload = run_live_workload(LiveWorkload {
+            clients: run.dist.clients()?,
+            bucket: scratch.clone(),
+            prefix: "live".to_string(),
+            writers: LIVE_WRITERS,
+            rounds: LIVE_ROUNDS,
+            baseline_bucket: manifest.plain_bucket.clone(),
+            baseline: baseline.clone(),
+            pause: Duration::from_millis(100),
+        });
+        let stop = async {
+            sleep(Duration::from_secs(2)).await;
+            stop_rebalance(&run.dist.cluster, &observed_id).await?;
+            let (outcome, status) = wait_for_rebalance_stopped_or_completed(&run, &observed_id).await?;
+            if outcome != "stopped" {
+                return Err("rebalance completed before stop; mid-run stop was not exercised".into());
+            }
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((outcome, status))
+        };
+        let (ops, (outcome, status)) = tokio::try_join!(workload, stop)?;
+        Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id, reached, probes, ops, outcome, status))
+    };
+    let (first_id, (observed_id, first_reached, during_rebalance, ops, first_outcome, first_status)) =
+        tokio::try_join!(start_rebalance(&run.dist.cluster), observe_first)?;
+    if first_id != observed_id {
+        return Err(format!("rebalance start acknowledged {first_id}, but the stopped run was {observed_id}").into());
+    }
+    run.movement_barrier.disarm()?;
     let live_state = expected_live_state(&ops);
 
     for node_idx in 0..run.dist.cluster.nodes.len() {
@@ -333,8 +363,40 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
     verify_from_every_node(&run.dist, &manifest, "after restart following the stop").await?;
     verify_live_state(&run.dist.client(0)?, &scratch, &live_state, "after restart following the stop").await?;
 
-    let second_id = start_rebalance(&run.dist.cluster).await?;
-    wait_for_rebalance_active(&run.dist.cluster, &second_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
+    // A short second run can finish before the start response reaches us.
+    // Hold one remaining source entry until its active state is observed.
+    let mut second_key = None;
+    for entry in manifest.plain_entries() {
+        if pools_holding(&run.dist, &manifest.plain_bucket, &entry.key, None)?.contains(&SEED_POOL) {
+            second_key = Some(entry.key.clone());
+            break;
+        }
+    }
+    let second_key = second_key.ok_or("no seed-pool object remains for the second rebalance")?;
+    let second_barrier = run.movement_barrier.next_phase();
+    second_barrier.arm_movement("rebalance", &manifest.plain_bucket, &second_key, STAGE_BEFORE_SOURCE_CLEANUP)?;
+    let observe_second = async {
+        let reached = second_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
+        let status = rebalance_status_json(&run.dist.cluster).await?;
+        let observed_id = status
+            .get("id")
+            .and_then(Value::as_str)
+            .ok_or("rebalance status omitted id")?;
+        if observed_id == first_id {
+            return Err("the second rebalance reused the stopped run id".into());
+        }
+        wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
+        second_barrier.disarm()?;
+        second_barrier.release_movement()?;
+        Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
+    };
+    // Worker propagation can hold the start response until movement drains.
+    let (second_id, (observed_id, second_reached)) = tokio::try_join!(start_rebalance(&run.dist.cluster), async {
+        tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, observe_second).await?
+    })?;
+    if second_id != observed_id {
+        return Err(format!("rebalance start acknowledged {second_id}, but the active run was {observed_id}").into());
+    }
     wait_for_rebalance_complete(&run.dist.cluster, &second_id, MOVEMENT_COMPLETE_TIMEOUT).await?;
     verify_from_every_node(&run.dist, &manifest, "after the second rebalance completed").await?;
     verify_live_state(&run.dist.client(3)?, &scratch, &live_state, "after the second rebalance completed").await?;
@@ -351,8 +413,8 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
         &json!({
             "gate": "G-REB",
             "manifest": manifest_summary(&manifest),
-            "first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop },
-            "second_run": { "id": second_id, "versions_off_seed_pool": moved },
+            "first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop, "barrier_object": first_key, "barrier": first_reached },
+            "second_run": { "id": second_id, "versions_off_seed_pool": moved, "barrier_object": second_key, "barrier": second_reached },
             "live_ops": ops,
             "probe_placement_during_rebalance": during_placement,
             "process_durability_strict_nodes": strict_nodes,

crates/e2e_test/src/distributed/scanner_peer_bucket_test.rs

@@ -1636,20 +1636,53 @@ async fn hot_window_with_old_followers(
     let mixed_initial_physical = if mixed { json!(converged_shards(&run)?) } else { Value::Null };
     let scheduler_config = run.assert_default_cadence().await?;
     let baseline = run.quiet_baseline().await?;
-    let leader = run.leader().await?;
-    let leader_pid = run.pid(leader)?;
+    // A leader-lock refresh gap during old-binary replacement can make
+    // `leader()` briefly observe a different node. Poll until the coordinator
+    // is stable before comparing against the measured fleet. A persistent
+    // change to an old-binary node is a genuine election outcome and fails.
+    let (leader, leader_pid) = if mixed {
+        let expected_leader = usize::try_from(
+            mixed_fleet["leader"]
+                .as_u64()
+                .ok_or("mixed fleet omitted its measured leader")?,
+        )?;
+        let expected_pid = mixed_fleet["leader_pid"]
+            .as_u64()
+            .ok_or("mixed fleet omitted its measured leader pid")?;
+        let mut resolved = None;
+        for _ in 0..20 {
+            let candidate = run.leader().await?;
+            let candidate_pid = u64::from(run.pid(candidate)?);
+            if Some(u64::try_from(candidate)?) == mixed_fleet["leader"].as_u64() && Some(candidate_pid) == Some(expected_pid) {
+                resolved = Some((candidate, run.pid(candidate)?));
+                break;
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+        }
+        match resolved {
+            Some(pair) => pair,
+            None => {
+                let current = match run.leader().await {
+                    Ok(node) => Some((node, run.pid(node).ok())),
+                    Err(_) => None,
+                };
+                return Err(format!(
+                    "the hot window did not retain its measured new coordinator after stabilization \
+                     (expected node {expected_leader} pid {expected_pid}, observed {current:?})"
+                )
+                .into());
+            }
+        }
+    } else {
+        let leader = run.leader().await?;
+        (leader, run.pid(leader)?)
+    };
     let producer = (leader + 1) % 4;
     let producers = if fault.is_some() {
         vec![producer, (leader + 2) % 4]
     } else {
         vec![producer]
     };
-    if mixed
-        && (mixed_fleet["leader"].as_u64() != Some(u64::try_from(leader)?)
-            || mixed_fleet["leader_pid"].as_u64() != Some(u64::from(leader_pid)))
-    {
-        return Err("the hot window did not retain its measured new coordinator".into());
-    }
     let producer_pid = run.pid(producer)?;
     let producer_address = run.peer_address(producer);
     let [cold_a, cold_b, hot] = run.buckets.clone();

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -33,7 +33,7 @@ mod tests {
     use sha2::{Digest, Sha256};
     use std::collections::HashSet;
     use std::error::Error;
-    use std::io::{Read, Write};
+    use std::io::{Read, Seek, SeekFrom, Write};
     use std::net::SocketAddr;
     use std::path::{Path, PathBuf};
     use std::process::Command;
@@ -1654,6 +1654,89 @@ mod tests {
         TargetEndpointBlackhole,
     }
 
+    async fn release_heal_commit_barrier_after_shutdown(
+        log: &Path,
+        log_offset: u64,
+        barrier: &Path,
+    ) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
+        // Observe within the existing graceful process-stop budget.
+        timeout(Duration::from_secs(45), async {
+            loop {
+                let mut file = std::fs::File::open(log)?;
+                file.seek(SeekFrom::Start(log_offset))?;
+                let mut tail = String::new();
+                file.read_to_string(&mut tail)?;
+                if let Some(event) = tail
+                    .lines()
+                    .filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
+                    .find(|event| {
+                        event["event"] == "server_shutdown_state"
+                            && event["component"] == "main"
+                            && event["subsystem"] == "startup"
+                            && event["state"] == "stopping"
+                    })
+                {
+                    // Local heal has stopped and HTTP shutdown has been signalled.
+                    // Let admitted peer RPCs finish; keeping this synthetic barrier
+                    // until process exit would make the fixture block its own drain.
+                    std::fs::remove_file(barrier)?;
+                    return Ok(event);
+                }
+                sleep(Duration::from_millis(10)).await;
+            }
+        })
+        .await?
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_commit_barrier_waits_for_new_shutdown_event() -> Result<(), Box<dyn Error + Send + Sync>> {
+        let directory = tempfile::tempdir()?;
+        let log = directory.path().join("server.log");
+        let barrier = directory.path().join("barrier");
+        let event = serde_json::json!({
+            "event": "server_shutdown_state", "component": "main", "subsystem": "startup", "state": "stopping"
+        });
+        std::fs::write(&log, format!("{event}\n"))?;
+        std::fs::write(&barrier, "bucket/prefix/")?;
+        let log_offset = log.metadata()?.len();
+        let mut writer = std::fs::OpenOptions::new().append(true).open(&log)?;
+        writeln!(writer, "{}", serde_json::json!({"event": "shutdown_signal_received"}))?;
+        writeln!(writer, "{}", serde_json::json!({"event": "server_shutdown_state", "state": "stopping"}))?;
+
+        let release = release_heal_commit_barrier_after_shutdown(&log, log_offset, &barrier);
+        tokio::pin!(release);
+        assert!(
+            futures::poll!(release.as_mut()).is_pending(),
+            "old or unrelated events must not release the barrier"
+        );
+        assert!(barrier.is_file(), "the partial-rebuild barrier must remain armed before shutdown");
+        tokio::time::advance(Duration::from_secs(6)).await;
+        assert!(
+            futures::poll!(release.as_mut()).is_pending(),
+            "background shutdown may take longer than five seconds within the process-stop budget"
+        );
+        writeln!(writer, "{event}")?;
+        tokio::time::advance(Duration::from_millis(10)).await;
+        assert_eq!(release.await?, event, "retain the actual new shutdown witness");
+        assert!(!barrier.exists(), "shutdown must release the synthetic hold");
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_commit_barrier_missing_shutdown_fails_without_releasing() -> Result<(), Box<dyn Error + Send + Sync>> {
+        let directory = tempfile::tempdir()?;
+        let log = directory.path().join("server.log");
+        let barrier = directory.path().join("barrier");
+        std::fs::write(&log, "")?;
+        std::fs::write(&barrier, "bucket/prefix/")?;
+        let error = release_heal_commit_barrier_after_shutdown(&log, 0, &barrier)
+            .await
+            .expect_err("missing shutdown evidence must fail the fixture");
+        assert!(error.is::<tokio::time::error::Elapsed>());
+        assert!(barrier.is_file(), "a missing shutdown witness must not release repairs");
+        Ok(())
+    }
+
     async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
         let server_binary = rustfs_binary_path();
         let evidence_case = match scenario {
@@ -1791,6 +1874,7 @@ mod tests {
         let mut created_online_objects = Vec::with_capacity(online_object_count);
         let mut expected_manifests = Vec::with_capacity(online_object_count);
         let mut unclean_shutdown_marker_observed = None;
+        let mut shutdown_barrier_release = None;
         let mut attempt_count = 0usize;
         let max_online_attempts = online_object_count.saturating_mul(topology.total_drives().max(1));
         while expected_manifests.len() < online_object_count {
@@ -2373,7 +2457,21 @@ mod tests {
                     | InterruptionScenario::BackgroundTargetRestartEc84MultiSet
                     | InterruptionScenario::BackgroundCoordinatorRestart
             ) {
-                cluster.stop_node_gracefully(interruption_node).await?;
+                if interruption_node == 1
+                    && let Some(barrier) = &commit_barrier
+                {
+                    let target_log = PathBuf::from(format!("{log_dir}/node1.log"));
+                    let log_offset = target_log.metadata()?.len();
+                    let (stopped, released) = tokio::join!(
+                        cluster.stop_node_gracefully(interruption_node),
+                        release_heal_commit_barrier_after_shutdown(&target_log, log_offset, barrier),
+                    );
+                    // Await both branches even on error so the child is reaped.
+                    stopped?;
+                    shutdown_barrier_release = Some(released?);
+                } else {
+                    cluster.stop_node_gracefully(interruption_node).await?;
+                }
             } else {
                 cluster.stop_node(interruption_node)?;
             }
@@ -2415,7 +2513,9 @@ mod tests {
                     }
                 }
             }
-            if let Some(barrier) = &commit_barrier {
+            if shutdown_barrier_release.is_none()
+                && let Some(barrier) = &commit_barrier
+            {
                 std::fs::remove_file(barrier)?;
             }
             cluster.start_node_from_binary(interruption_node, &server_binary).await?;
@@ -2746,6 +2846,7 @@ mod tests {
                 "all_peers_bound_to_generation_window": true,
                 "pid_before": target_pid, "pid_after": restarted_pid,
                 "unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
+                "graceful_barrier_release": shutdown_barrier_release,
                 "objects": evidence_objects, "node_listings": node_listings,
             });
             if let Some(outage_write) = outage_write_diagnostic {

crates/e2e_test/src/lib.rs

@@ -283,6 +283,9 @@ mod ssec_copy_test;
 #[cfg(test)]
 mod multipart_storage_class_test;
 
+#[cfg(test)]
+mod multipart_upload_id_test;
+
 #[cfg(test)]
 mod storage_class_capability_test;
 

crates/e2e_test/src/multipart_upload_id_test.rs

@@ -0,0 +1,175 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Multipart upload ID regression tests.
+//!
+//! Every operation that addresses an existing multipart upload must report an
+//! upload ID that does not name a live upload as `404 NoSuchUpload`, whether
+//! the ID is malformed or is a well-formed ID of an aborted upload.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
+    use aws_sdk_s3::primitives::ByteStream;
+    use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+
+    type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+
+    const BUCKET: &str = "multipart-upload-id";
+    const KEY: &str = "target.bin";
+    const COPY_SOURCE_KEY: &str = "copy-source.bin";
+    // RustFS upload IDs are URL-safe base64; '.' is outside that alphabet, so
+    // the object layer rejects this ID as malformed instead of looking it up.
+    const MALFORMED_UPLOAD_ID: &str = "malformed.upload.id";
+    const PART_NUMBER: i32 = 1;
+    const PART_BODY: &[u8] = b"multipart upload id regression part";
+    const PART_ETAG: &str = "\"00000000000000000000000000000000\"";
+    const NO_SUCH_UPLOAD_STATUS: u16 = 404;
+    const NO_SUCH_UPLOAD_CODE: &str = "NoSuchUpload";
+    const OBJECT_NOT_FOUND_STATUS: u16 = 404;
+
+    /// Describes how `result` differs from `404 NoSuchUpload`, or returns `None` when it matches.
+    fn no_such_upload_mismatch<T, E>(operation: &str, result: Result<T, SdkError<E>>) -> Option<String>
+    where
+        E: ProvideErrorMetadata + std::fmt::Debug,
+    {
+        let Err(error) = result else {
+            return Some(format!("{operation}: succeeded"));
+        };
+        let status = error.raw_response().map(|response| response.status().as_u16());
+        let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+        (status != Some(NO_SUCH_UPLOAD_STATUS) || code != Some(NO_SUCH_UPLOAD_CODE))
+            .then(|| format!("{operation}: status {status:?}, code {code:?}: {error:?}"))
+    }
+
+    async fn start_server_with_copy_source() -> Result<(RustFSTestEnvironment, Client), Box<dyn std::error::Error + Send + Sync>>
+    {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(Vec::new()).await?;
+        env.create_test_bucket(BUCKET).await?;
+        let client = env.create_s3_client();
+        client
+            .put_object()
+            .bucket(BUCKET)
+            .key(COPY_SOURCE_KEY)
+            .body(ByteStream::from_static(PART_BODY))
+            .send()
+            .await?;
+        Ok((env, client))
+    }
+
+    /// Sends every operation that addresses an existing upload and asserts that all of them report `404 NoSuchUpload`.
+    async fn assert_every_upload_operation_reports_no_such_upload(client: &Client, upload_id: &str) {
+        let upload_part = client
+            .upload_part()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .part_number(PART_NUMBER)
+            .body(ByteStream::from_static(PART_BODY))
+            .send()
+            .await;
+        let upload_part_copy = client
+            .upload_part_copy()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .part_number(PART_NUMBER)
+            .copy_source(format!("/{BUCKET}/{COPY_SOURCE_KEY}"))
+            .send()
+            .await;
+        let list_parts = client.list_parts().bucket(BUCKET).key(KEY).upload_id(upload_id).send().await;
+        let complete = client
+            .complete_multipart_upload()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .multipart_upload(
+                CompletedMultipartUpload::builder()
+                    .parts(CompletedPart::builder().part_number(PART_NUMBER).e_tag(PART_ETAG).build())
+                    .build(),
+            )
+            .send()
+            .await;
+        let abort = client
+            .abort_multipart_upload()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .send()
+            .await;
+
+        let mismatches: Vec<String> = [
+            no_such_upload_mismatch("UploadPart", upload_part),
+            no_such_upload_mismatch("UploadPartCopy", upload_part_copy),
+            no_such_upload_mismatch("ListParts", list_parts),
+            no_such_upload_mismatch("CompleteMultipartUpload", complete),
+            no_such_upload_mismatch("AbortMultipartUpload", abort),
+        ]
+        .into_iter()
+        .flatten()
+        .collect();
+        assert!(
+            mismatches.is_empty(),
+            "upload ID {upload_id:?}: operations not reporting {NO_SUCH_UPLOAD_STATUS} {NO_SUCH_UPLOAD_CODE}:\n{}",
+            mismatches.join("\n")
+        );
+    }
+
+    /// Asserts that the rejected operations left neither a multipart upload nor an object under `KEY`.
+    async fn assert_no_upload_or_object_published(client: &Client) -> TestResult {
+        let uploads = client.list_multipart_uploads().bucket(BUCKET).prefix(KEY).send().await?;
+        assert!(uploads.uploads().is_empty(), "rejected operations must not leave an upload: {uploads:?}");
+        let head = client.head_object().bucket(BUCKET).key(KEY).send().await;
+        let status = head
+            .as_ref()
+            .err()
+            .and_then(|error| error.raw_response())
+            .map(|response| response.status().as_u16());
+        assert_eq!(
+            status,
+            Some(OBJECT_NOT_FOUND_STATUS),
+            "rejected operations must not publish {KEY}: {head:?}"
+        );
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn malformed_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
+        let (_env, client) = start_server_with_copy_source().await?;
+
+        assert_every_upload_operation_reports_no_such_upload(&client, MALFORMED_UPLOAD_ID).await;
+        assert_no_upload_or_object_published(&client).await
+    }
+
+    #[tokio::test]
+    async fn aborted_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
+        let (_env, client) = start_server_with_copy_source().await?;
+        let created = client.create_multipart_upload().bucket(BUCKET).key(KEY).send().await?;
+        let upload_id = created.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?;
+        client
+            .abort_multipart_upload()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .send()
+            .await?;
+
+        assert_every_upload_operation_reports_no_such_upload(&client, upload_id).await;
+        assert_no_upload_or_object_published(&client).await
+    }
+}

crates/e2e_test/src/protocols/README.md

@@ -77,9 +77,7 @@ core protocol surface plus cross-protocol consistency:
   existing path returns an error, bad-password authentication is rejected
 - Cross-protocol via aws-sdk-s3: SFTP write then S3 read with SHA256 match,
   S3 write then SFTP read with SHA256 match
-- Cross-API directory visibility: SFTP-created sub-directory visible via S3
-  ListObjectsV2, S3-created `__XLDIR__` marker visible via SFTP readdir as a
-  directory entry
+- Cross-API directory visibility: SFTP-created sub-directory visible via S3 ListObjectsV2, and an S3 trailing-slash directory object visible via SFTP readdir as a directory entry
 
 #### sftp_compliance (`test_sftp_compliance_suite`)
 

crates/e2e_test/src/protocols/sftp_core.rs

@@ -394,9 +394,9 @@ pub async fn test_sftp_core_operations() -> Result<()> {
         info!("PASS: S3-written object matches via S3 GetObject and SFTP READ");
 
         // --- Cross-API directory visibility: SFTP mkdir, S3 ListObjectsV2 ---
-        // SFTP mkdir writes a __XLDIR__ marker. The rustfs S3 listing path
-        // decodes that marker back to a trailing-slash key, so the asserted
-        // pattern is "subdir_sftp/".
+        // SFTP mkdir writes a zero-byte object at the trailing-slash key.
+        // Storage encodes that key to the __XLDIR__ marker, and the S3
+        // listing path decodes the marker back to a trailing-slash key.
         info!("Testing SFTP: SFTP-created sub-directory visible via S3 ListObjectsV2");
         let sftp_subdir_name = "subdir_sftp";
         let sftp_subdir_path = format!("/{bucket}/{sftp_subdir_name}");
@@ -422,20 +422,21 @@ pub async fn test_sftp_core_operations() -> Result<()> {
         );
         info!("PASS: SFTP mkdir visible to S3 ListObjectsV2");
 
-        // --- Cross-API directory visibility: S3 marker, SFTP readdir ---
-        // aws-sdk-s3 PutObject writes a zero-byte marker keyed with the
-        // __XLDIR__ suffix. SFTP readdir must decode the marker back to a
-        // bare directory entry whose file_type reports as a directory.
-        info!("Testing SFTP: S3-created __XLDIR__ marker visible via SFTP readdir");
+        // --- Cross-API directory visibility: S3 directory object, SFTP readdir ---
+        // aws-sdk-s3 PutObject writes a zero-byte object at the trailing-slash
+        // key. The __XLDIR__ suffix is reserved on the client-key boundary, so
+        // a direct put of that suffix is rejected. Storage encodes the slash
+        // key, and SFTP readdir must show a directory entry.
+        info!("Testing SFTP: S3-created directory object visible via SFTP readdir");
         let s3_subdir_name = "subdir_s3";
-        let s3_subdir_marker_key = format!("{s3_subdir_name}{XLDIR_SUFFIX}");
+        let s3_subdir_key = format!("{s3_subdir_name}/");
         s3.put_object()
             .bucket(bucket)
-            .key(&s3_subdir_marker_key)
+            .key(&s3_subdir_key)
             .body(ByteStream::from_static(b""))
             .send()
             .await
-            .map_err(|e| anyhow!("S3 PutObject {}/{} failed: {:?}", bucket, s3_subdir_marker_key, e))?;
+            .map_err(|e| anyhow!("S3 PutObject {}/{} failed: {:?}", bucket, s3_subdir_key, e))?;
         let bucket_entries: Vec<(String, bool)> = sftp
             .read_dir(&bucket_path)
             .await?
@@ -444,9 +445,9 @@ pub async fn test_sftp_core_operations() -> Result<()> {
         let visible_via_sftp = bucket_entries.iter().any(|(name, is_dir)| name == s3_subdir_name && *is_dir);
         assert!(
             visible_via_sftp,
-            "S3-created marker must appear as a directory in SFTP readdir: entries were {bucket_entries:?}"
+            "S3-created directory object must appear as a directory in SFTP readdir: entries were {bucket_entries:?}"
         );
-        info!("PASS: S3 marker visible to SFTP readdir as a directory");
+        info!("PASS: S3 directory object visible to SFTP readdir as a directory");
 
         // --- Pre-cleanup of cross-protocol fixtures ---
         // Removes the new files and sub-directories so the existing rmdir

crates/e2e_test/src/quota_test.rs

@@ -412,6 +412,38 @@ mod integration_tests {
         );
     }
 
+    async fn upload_after_quota_reconciliation(
+        env: &QuotaTestEnv,
+        key: &str,
+        size_bytes: usize,
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let upload = async {
+            loop {
+                let result = env
+                    .client
+                    .put_object()
+                    .bucket(&env.bucket_name)
+                    .key(key)
+                    .body(aws_sdk_s3::primitives::ByteStream::from(vec![0u8; size_bytes]))
+                    .send()
+                    .await;
+                match result {
+                    Ok(_) => break,
+                    Err(error) => assert_quota_rejection(
+                        error.raw_response().map(|response| response.status().as_u16()),
+                        error.as_service_error(),
+                        &error,
+                    ),
+                }
+                sleep(Duration::from_secs(1)).await;
+            }
+        };
+        // The durable V1 ledger can retain the old charge for its 30-second reconciliation cooldown.
+        timeout(Duration::from_secs(45), upload)
+            .await
+            .map_err(|_| format!("PUT {key} did not succeed after quota reconciliation within 45 seconds").into())
+    }
+
     async fn assert_put_rejected_by_quota(env: &QuotaTestEnv, key: &str, size_bytes: usize) {
         let error = env
             .client
@@ -573,8 +605,7 @@ mod integration_tests {
 
         assert!(!env.object_exists("file1.txt").await?);
 
-        // Now we should be able to upload again (quota freed up)
-        env.upload_object("file3.txt", 256 * 1024).await?;
+        upload_after_quota_reconciliation(&env, "file3.txt", 256 * 1024).await?;
         assert!(env.object_exists("file3.txt").await?);
 
         env.cleanup_bucket().await?;
@@ -981,8 +1012,7 @@ mod integration_tests {
         );
         assert_eq!(delete_result.deleted().len(), 2);
 
-        // Now should be able to upload again (quota freed up)
-        env.upload_object("file3.txt", 256 * 1024).await?;
+        upload_after_quota_reconciliation(&env, "file3.txt", 256 * 1024).await?;
         assert!(env.object_exists("file3.txt").await?);
 
         env.cleanup_bucket().await?;

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -42,6 +42,7 @@ fn lock_result_from_response(response: rustfs_lock::LockResponse) -> GenerallyLo
         success: response.success,
         error_info: response.error,
         lock_info: response.lock_info.and_then(|info| serde_json::to_string(&info).ok()),
+        ..Default::default()
     }
 }
 
@@ -50,6 +51,7 @@ fn lock_result_from_error(error: impl Into<String>) -> GenerallyLockResult {
         success: false,
         error_info: Some(error.into()),
         lock_info: None,
+        ..Default::default()
     }
 }
 
@@ -59,6 +61,7 @@ fn lock_result_from_release(lock_id: &rustfs_lock::LockId, success: bool) -> Gen
             success: true,
             error_info: None,
             lock_info: None,
+            ..Default::default()
         }
     } else {
         lock_result_from_error(format!("lock not found for release: {lock_id}"))
@@ -164,6 +167,7 @@ impl NodeService for MinimalLockNodeService {
                     success: false,
                     error_info: Some(format!("can not decode args, err: {err}")),
                     lock_info: None,
+                    ..Default::default()
                 }));
             }
         };
@@ -175,6 +179,7 @@ impl NodeService for MinimalLockNodeService {
                     success: result.success,
                     error_info: result.error,
                     lock_info: lock_info_json,
+                    ..Default::default()
                 }))
             }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
@@ -184,6 +189,7 @@ impl NodeService for MinimalLockNodeService {
                     args.resource, args.owner, err
                 )),
                 lock_info: None,
+                ..Default::default()
             })),
         }
     }
@@ -197,6 +203,7 @@ impl NodeService for MinimalLockNodeService {
                     success: false,
                     error_info: Some(format!("can not decode args, err: {err}")),
                     lock_info: None,
+                    ..Default::default()
                 }));
             }
         };
@@ -208,6 +215,7 @@ impl NodeService for MinimalLockNodeService {
                     success: result.success,
                     error_info: result.error_info,
                     lock_info: None,
+                    ..Default::default()
                 }))
             }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
@@ -217,6 +225,7 @@ impl NodeService for MinimalLockNodeService {
                     args.resource, args.owner, err
                 )),
                 lock_info: None,
+                ..Default::default()
             })),
         }
     }
@@ -230,6 +239,7 @@ impl NodeService for MinimalLockNodeService {
                     success: false,
                     error_info: Some(format!("can not decode args, err: {err}")),
                     lock_info: None,
+                    ..Default::default()
                 }));
             }
         };
@@ -241,6 +251,7 @@ impl NodeService for MinimalLockNodeService {
                     success: result.success,
                     error_info: result.error_info,
                     lock_info: None,
+                    ..Default::default()
                 }))
             }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
@@ -250,6 +261,7 @@ impl NodeService for MinimalLockNodeService {
                     args.resource, args.owner, err
                 )),
                 lock_info: None,
+                ..Default::default()
             })),
         }
     }
@@ -263,6 +275,7 @@ impl NodeService for MinimalLockNodeService {
                     success: false,
                     error_info: Some(format!("can not decode args, err: {err}")),
                     lock_info: None,
+                    ..Default::default()
                 }));
             }
         };
@@ -272,11 +285,13 @@ impl NodeService for MinimalLockNodeService {
                 success,
                 error_info: None,
                 lock_info: None,
+                ..Default::default()
             })),
             Err(err) => Ok(Response::new(GenerallyLockResponse {
                 success: false,
                 error_info: Some(format!("can not refresh, err: {err}")),
                 lock_info: None,
+                ..Default::default()
             })),
         }
     }

crates/e2e_test/src/replication_extension_test.rs

@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
-    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
-    VersioningConfiguration,
+    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
+    ServerSideEncryption, VersioningConfiguration,
 };
 use base64_simd::STANDARD as BASE64_STANDARD;
 use bytes::Bytes;
@@ -3729,6 +3729,8 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
     init_logging();
 
     let mut source_env = RustFSTestEnvironment::new().await?;
+    let source_log = format!("{}/source.log", source_env.temp_dir);
+    source_env.capture_log_path = Some(source_log.clone());
     let mut source_env_vars = replication_fast_env();
     source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
     source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
@@ -3791,19 +3793,66 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         "target did not preserve the source delete-marker version ID"
     );
 
-    source_client
-        .delete_object()
+    let wait_for_source_writeback = || async {
+        let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
+        timeout(Duration::from_secs(30), async {
+            loop {
+                let response = signed_request(
+                    http::Method::GET,
+                    &metrics_url,
+                    &source_env.access_key,
+                    &source_env.secret_key,
+                    None,
+                    None,
+                )
+                .await?;
+                let metrics: serde_json::Value = response.error_for_status()?.json().await?;
+                let queued = metrics["queued"]["curr"]["count"]
+                    .as_f64()
+                    .ok_or("missing replication queue count")?;
+                if metrics["provider_available"] == true && queued == 0.0 {
+                    return Ok::<(), Box<dyn Error + Send + Sync>>(());
+                }
+                sleep(Duration::from_millis(50)).await;
+            }
+        })
+        .await?
+    };
+    wait_for_source_writeback().await?;
+
+    let purged = source_client
+        .delete_objects()
         .bucket(source_bucket)
-        .key(object_key)
-        .version_id(&purged_version_id)
+        .delete(
+            Delete::builder()
+                .objects(
+                    ObjectIdentifier::builder()
+                        .key(object_key)
+                        .version_id(&purged_version_id)
+                        .build()?,
+                )
+                .objects(
+                    ObjectIdentifier::builder()
+                        .key(object_key)
+                        .version_id(delete_marker_version_id)
+                        .build()?,
+                )
+                .build()?,
+        )
         .send()
         .await?;
+    assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
+    assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
     assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
     let target_state = list_replication_state(&target_client, target_bucket).await?;
     assert!(
         target_state.iter().all(|entry| entry.version_id != purged_version_id),
         "target retained the explicitly purged object version"
     );
+    assert!(
+        target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
+        "target retained the explicitly purged delete marker"
+    );
     assert!(
         target_state
             .iter()
@@ -3819,6 +3868,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         .await?;
     assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
 
+    // Wait for the source workers to finish their final writeback, which runs
+    // after the remote purge has already become visible (rustfs#9121).
+    wait_for_source_writeback().await?;
+    let logs = fs::read_to_string(source_log).await?;
+    let writeback_failures: Vec<_> = logs
+        .lines()
+        .filter(|line| line.contains("apply_replication_delete_state"))
+        .collect();
+    assert!(
+        writeback_failures.is_empty(),
+        "a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
+    );
+
     Ok(())
 }
 

crates/e2e_test/src/rpc_fault_proxy.rs

@@ -2146,3 +2146,4 @@ mod tests {
         );
     }
 }
+// event-drill: isolated harness-only change

crates/e2e_test/src/storage_api.rs

@@ -94,3 +94,12 @@ pub(crate) mod scanner_peer_bucket {
     pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
     pub(crate) use ecstore_rpc::PeerRestClient;
 }
+
+/// Storage handles used to seed a pre-validation bucket record while the fixture server is stopped.
+#[cfg(test)]
+pub(crate) mod bucket_tag_fixture {
+    pub(crate) use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
+    pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools};
+    pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
+    pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks_with_instance_ctx};
+}

crates/e2e_test/src/sts_query_compat_test.rs

@@ -360,12 +360,26 @@ impl OpaMock {
 
     pub(crate) async fn expect_bucket_tags(&mut self, action: &str, bucket: &str, department: Option<&str>) -> TestResult {
         let payload = self.next_request().await?;
+        Self::assert_bucket_tags(&payload, action, bucket, department);
+        Ok(())
+    }
+
+    async fn expect_versioned_tag_permission(&mut self, bucket: &str, object: &str, version: &str) -> TestResult {
+        let payload = self.next_request().await?;
+        Self::assert_bucket_tags(&payload, "s3:GetObjectVersionTagging", bucket, Some("finance"));
+        assert_eq!(payload["input"]["resource"]["object"], object);
+        assert_eq!(payload["input"]["context"]["conditions"]["versionid"], serde_json::json!([version]));
+        Ok(())
+    }
+
+    fn assert_bucket_tags(payload: &Value, action: &str, bucket: &str, department: Option<&str>) {
         let input = &payload["input"];
+        assert_eq!(input["identity"]["account"], "opabuckettags");
+        assert_eq!(input["identity"]["is_owner"], false);
         assert_eq!(input["action"], action);
         assert_eq!(input["resource"]["bucket"], bucket);
         let expected = department.map(|value| serde_json::json!([value]));
         assert_eq!(input["context"]["conditions"].get("ExistingBucketTag/department"), expected.as_ref());
-        Ok(())
     }
 
     async fn wait_for_validation(&mut self) -> TestResult {
@@ -678,22 +692,61 @@ async fn test_list_buckets_opa_contract() -> TestResult {
     Ok(())
 }
 
+/// Stop and reap the directory's owning server before opening the fixture store.
+async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, bucket: &str) -> TestResult {
+    use crate::storage_api::bucket_tag_fixture::{
+        DisksLayout, ECStore, EndpointServerPools, InstanceContext, init_local_disks_with_instance_ctx, load_bucket_metadata,
+    };
+
+    let server = env
+        .process
+        .as_mut()
+        .ok_or("missing server process before offline fixture write")?;
+    server.kill()?;
+    server.wait()?;
+    env.process = None;
+    let layout = DisksLayout::from_volumes(std::slice::from_ref(&env.temp_dir))?;
+    let (pools, _) = EndpointServerPools::create_server_endpoints(&env.address, &layout).await?;
+    let context = std::sync::Arc::new(InstanceContext::new());
+    init_local_disks_with_instance_ctx(&context, pools.clone()).await?;
+    let store = ECStore::new_with_instance_ctx(env.address.parse()?, pools, Default::default(), context.clone()).await?;
+    let _cancel = context
+        .background_cancel_token()
+        .expect("store binds its shutdown token")
+        .drop_guard();
+    let mut metadata = load_bucket_metadata(store.clone(), bucket).await?;
+    let incarnation = metadata.bucket_incarnation_id;
+    let policy = metadata.policy_config_json.clone();
+    assert!(!incarnation.is_nil());
+    assert!(!policy.is_empty(), "the bucket-policy Allow must survive legacy seeding");
+    // Encode the duplicate tag set with the same bucket record and erasure writer used by the store.
+    let xml = b"<Tagging><TagSet><Tag><Key>department</Key><Value>finance</Value></Tag><Tag><Key>department</Key><Value>engineering</Value></Tag></TagSet></Tagging>";
+    metadata.tagging_config_xml = xml.to_vec();
+    metadata.save_with_store(store.clone()).await?;
+    let persisted = load_bucket_metadata(store, bucket).await?;
+    assert_eq!(persisted.bucket_incarnation_id, incarnation);
+    assert_eq!(persisted.policy_config_json, policy);
+    assert_eq!(persisted.tagging_config_xml.as_slice(), xml);
+    Ok(())
+}
+
 #[tokio::test]
 async fn test_bucket_tags_opa_contract() -> TestResult {
     use aws_sdk_s3::primitives::ByteStream;
     use aws_sdk_s3::types::{
-        BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
+        BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, TaggingDirective, VersioningConfiguration,
     };
 
     init_logging();
     let mut opa = OpaMock::start().await?;
     let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server_without_cleanup_with_env(&[
-        ("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
+    let opa_url = opa.url.clone();
+    let server_env = [
+        ("RUSTFS_POLICY_PLUGIN_URL", opa_url.as_str()),
         ("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
         ("NO_PROXY", "127.0.0.1,localhost"),
-    ])
-    .await?;
+    ];
+    env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
     let secret = uuid::Uuid::new_v4().to_string();
     create_user(&env, "opabuckettags", &secret).await?;
     let client = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, "opabuckettags", &secret, None, "e2e-bucket-tags"));
@@ -733,7 +786,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
         .await?;
     assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
     opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-    client
+    let head = client
         .head_object()
         .bucket(source)
         .key("report")
@@ -741,6 +794,8 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
         .send()
         .await?;
     opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+    assert_eq!(head.tag_count(), Some(1));
+    opa.expect_versioned_tag_permission(source, "report", version).await?;
     client.list_objects_v2().bucket(source).send().await?;
     opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?;
     client.list_object_versions().bucket(source).send().await?;
@@ -750,28 +805,65 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     admin.create_bucket().bucket(destination).send().await?;
     set_department(&admin, destination, "engineering").await?;
     let copy_source = format!("{source}/report?versionId={version}");
-    let error = client
-        .copy_object()
-        .bucket(destination)
-        .key("copied")
-        .copy_source(&copy_source)
-        .send()
-        .await
-        .expect_err("source tags must not authorize a different destination");
-    assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
-    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-    opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
-        .await?;
+    for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
+        let error = client
+            .copy_object()
+            .bucket(destination)
+            .key("copied")
+            .copy_source(&copy_source)
+            .set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
+            .set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
+            .send()
+            .await
+            .expect_err("source tags must not authorize a different destination");
+        assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
+        opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+        if directive == TaggingDirective::Copy {
+            opa.expect_versioned_tag_permission(source, "report", version).await?;
+        }
+        opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
+            .await?;
+        let missing = admin
+            .head_object()
+            .bucket(destination)
+            .key("copied")
+            .send()
+            .await
+            .expect_err("denied COPY and REPLACE must leave no destination object");
+        assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
+    }
     set_department(&admin, destination, "finance").await?;
-    client
-        .copy_object()
-        .bucket(destination)
-        .key("copied")
-        .copy_source(&copy_source)
-        .send()
-        .await?;
-    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-    opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
+    for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
+        client
+            .copy_object()
+            .bucket(destination)
+            .key("copied")
+            .copy_source(&copy_source)
+            .set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
+            .set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
+            .send()
+            .await?;
+        opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+        // The access hook checks preliminary tags, then the copy use case checks
+        // the exact source reader's tags before publishing the destination.
+        for _ in 0..2 {
+            if directive == TaggingDirective::Copy {
+                opa.expect_versioned_tag_permission(source, "report", version).await?;
+            }
+            opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
+        }
+        let tags = admin.get_object_tagging().bucket(destination).key("copied").send().await?;
+        assert_eq!(tags.tag_set().len(), 1);
+        assert_eq!(tags.tag_set()[0].key(), "department");
+        assert_eq!(
+            tags.tag_set()[0].value(),
+            if directive == TaggingDirective::Copy {
+                "engineering"
+            } else {
+                "legal"
+            }
+        );
+    }
 
     let upload = client
         .create_multipart_upload()
@@ -847,6 +939,14 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
     opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("engineering"))
         .await?;
+    let missing = admin
+        .head_object()
+        .bucket(destination)
+        .key("denied-source")
+        .send()
+        .await
+        .expect_err("a denied source must leave no destination object");
+    assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
     set_department(&client, source, "finance")
         .await
         .expect_err("requested tags must not authorize their own mutation");
@@ -885,7 +985,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
     opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering"))
         .await?;
-    admin
+    let invalid_write = admin
         .put_bucket_tagging()
         .bucket(destination)
         .tagging(
@@ -895,7 +995,25 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
                 .build()?,
         )
         .send()
+        .await
+        .expect_err("new duplicate tags must be rejected before publication");
+    assert_eq!(invalid_write.as_service_error().and_then(ProvideErrorMetadata::code), Some("InvalidTag"));
+    assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400));
+
+    set_department(&admin, source, "finance").await?;
+    seed_legacy_duplicate_bucket_tags(&mut env, destination).await?;
+    env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
+    // Prove this process can authorize the same caller and read a healthy bucket's persisted tags.
+    let read = client
+        .get_object()
+        .bucket(source)
+        .key("report")
+        .version_id(version)
+        .send()
         .await?;
+    assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
+    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+
     let error = client
         .get_object()
         .bucket(destination)

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -48,6 +48,7 @@ const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
 const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
 const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
 const STABLE_1_0_1_COMMIT: &str = "6de965ae3c965a78ff819fbcd7acd4aa44177d92";
+const STABLE_1_0_4_COMMIT: &str = "46337942d55f3a8e6a97b147ca98d9ea8115555d";
 const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
 const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
 const PLAIN_BUCKET: &str = "upgrade-plain-data";
@@ -954,7 +955,8 @@ async fn exercise_mixed_cluster(
     phase: &str,
     current_node: usize,
     previous_node: usize,
-) -> TestResult {
+    previous_has_delete_recovery: bool,
+) -> Result<Option<String>, BoxError> {
     let clients = cluster.create_all_clients()?;
     wait_for_upgrade_write_readiness(&clients, MIXED_BUCKET, phase, LISTING_CONVERGENCE_TIMEOUT).await?;
     wait_for_drive_readmission(&clients, MIXED_BUCKET, phase, DRIVE_READMISSION_TIMEOUT).await?;
@@ -1009,6 +1011,30 @@ async fn exercise_mixed_cluster(
         vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
     );
 
+    // 1.0.4 already implements the delete decision protocol. Use a separate
+    // old-writer key so the original seed still spans the entire rollout.
+    if previous_has_delete_recovery {
+        let key = format!(".upgrade-delete/{phase}/previous-writer");
+        previous_client
+            .put_object()
+            .bucket(MIXED_BUCKET)
+            .key(&key)
+            .body(ByteStream::from_static(UPGRADE_READINESS_BODY))
+            .send()
+            .await?;
+        for client in &clients {
+            assert_eq!(read_object(client, MIXED_BUCKET, &key, None).await?.1, UPGRADE_READINESS_BODY);
+        }
+        upgrade_probe_client(current_client)
+            .delete_object()
+            .bucket(MIXED_BUCKET)
+            .key(&key)
+            .send()
+            .await?;
+        assert_mixed_deletes_visible(&clients, std::slice::from_ref(&key)).await?;
+        return Ok(Some(key));
+    }
+
     // Every online participant must understand the delete decision protocol.
     // An upgrade phase can serve reads/writes while intentionally refusing a
     // modern DELETE; rejecting it must leave the old writer's seed intact.
@@ -1029,6 +1055,31 @@ async fn exercise_mixed_cluster(
         );
     }
 
+    Ok(None)
+}
+
+async fn assert_mixed_deletes_visible(clients: &[Client], keys: &[String]) -> TestResult {
+    for (node, client) in clients.iter().enumerate() {
+        for key in keys {
+            let error = upgrade_probe_client(client)
+                .get_object()
+                .bucket(MIXED_BUCKET)
+                .key(key)
+                .send()
+                .await
+                .expect_err("a successful mixed-version DELETE must remain visible through every reader");
+            assert_eq!(
+                error.raw_response().map(|response| response.status().as_u16()),
+                Some(404),
+                "node {node}: {key}"
+            );
+            assert_eq!(
+                error.as_service_error().and_then(ProvideErrorMetadata::code),
+                Some("NoSuchKey"),
+                "node {node}: {key}"
+            );
+        }
+    }
     Ok(())
 }
 
@@ -1112,8 +1163,9 @@ async fn wait_for_upgrade_writes(
             match response {
                 Ok(Ok(_)) => break,
                 Ok(Err(error)) => {
+                    let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
                     if error.raw_response().map(|response| response.status().as_u16()) != Some(503)
-                        || error.as_service_error().and_then(ProvideErrorMetadata::code) != Some("ServiceUnavailable")
+                        || !matches!(code, Some("ServiceUnavailable" | "SlowDown" | "SlowDownRead"))
                     {
                         return Err(format!("{phase}: node {node} {probe} failed: {error:?}").into());
                     }
@@ -2239,6 +2291,9 @@ async fn direct_upgrade_from_previous_release_preserves_object_contracts() -> Te
 async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts() -> TestResult {
     init_logging();
     let previous_binary = source_binary()?;
+    // Bind the expected protocol to the verified release source, never to the
+    // DELETE result itself. The older CI source pins retain their 503 contract.
+    let previous_has_delete_recovery = binary_source_revision(&previous_binary).await? == STABLE_1_0_4_COMMIT;
     let evidence_context = g09_evidence_context(&previous_binary).await?;
     let current_binary = rustfs_binary_path();
     let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
@@ -2259,7 +2314,10 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
 
     cluster.stop_node(0)?;
     cluster.start_node_from_binary(0, &current_binary).await?;
-    exercise_mixed_cluster(&cluster, "one-current-node", 0, 1).await?;
+    let mut mixed_deleted_keys = Vec::new();
+    if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, previous_has_delete_recovery).await? {
+        mixed_deleted_keys.push(key);
+    }
     // One controlled out-of-policy attempt proves fail-closed behavior below
     // native write quorum; positive MPU writers remain paused while mixed.
     assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
@@ -2267,7 +2325,9 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
     for (node_idx, phase) in [(1, "two-current-nodes"), (2, "one-previous-node")] {
         cluster.stop_node(node_idx)?;
         cluster.start_node_from_binary(node_idx, &current_binary).await?;
-        exercise_mixed_cluster(&cluster, phase, 0, 3).await?;
+        if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, previous_has_delete_recovery).await? {
+            mixed_deleted_keys.push(key);
+        }
     }
 
     cluster.stop_node(3)?;
@@ -2277,6 +2337,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
     let completed_pending = finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?;
     exercise_homogeneous_multipart(&cluster, "homogeneous-current").await?;
     verify_load_bodies(&clients, "homogeneous-previous", "multipart").await?;
+    assert_mixed_deletes_visible(&clients, &mixed_deleted_keys).await?;
 
     for (node_idx, client) in clients.iter().enumerate() {
         assert_eq!(
@@ -2325,6 +2386,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
     cluster.start_with_binary(&previous_binary).await?;
     prepare_release_baseline(&mut cluster, &previous_binary, "rollback").await?;
     let old_clients = cluster.create_all_clients()?;
+    assert_mixed_deletes_visible(&old_clients, &mixed_deleted_keys).await?;
     assert_resumed_multipart(&old_clients, &completed_pending).await?;
     for phase in ["homogeneous-previous", "homogeneous-current"] {
         verify_load_bodies(&old_clients, phase, "multipart").await?;
@@ -2389,6 +2451,8 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
                 "phases": phases,
                 "objects_per_phase": objects_per_phase,
                 "mixed_load": "plain-put",
+                "mixed_delete_protocol_supported": previous_has_delete_recovery,
+                "mixed_deleted_keys": mixed_deleted_keys,
                 "homogeneous_multipart_phases": ["homogeneous-previous", "homogeneous-current", "homogeneous-rollback"],
                 "multipart_objects_per_homogeneous_phase": MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER,
                 "assertions": [
@@ -2421,7 +2485,11 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
                     "16x16 multipart load runs in homogeneous previous, current and rollback phases",
                     "one-current-node Complete returns typed 503 and preserves actual source authority and parts",
                     "the same legacy upload completes and replays once fully upgraded without adding a version",
-                    "modern DELETE fails closed and preserves every byte while old peers remain"
+                    if previous_has_delete_recovery {
+                        "mixed-version DELETE succeeds and stays absent through every reader after upgrade and rollback"
+                    } else {
+                        "modern DELETE fails closed and preserves every byte while old peers remain"
+                    }
                 ],
             }),
         )?;

crates/ecstore/Cargo.toml

@@ -31,6 +31,7 @@ workspace = true
 
 [features]
 default = []
+fips = ["rustfs-s3-client/fips", "rustfs-utils/fips", "rustfs-tls-runtime/fips"]
 diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
 # Sampled wall/CPU diagnostics only; does not change the read backend.
 diagnostic-local-batch = ["rustix/time"]
@@ -202,6 +203,9 @@ libc.workspace = true
 # (backlog#1178); "thread" provides cleanup affinity; "fs" is a workspace default.
 rustix = { workspace = true, features = ["process", "fs", "thread"] }
 rustfs-madmin.workspace = true
+rustfs-signer.workspace = true
+rustfs-tls-runtime.workspace = true
+quick-xml.workspace = true
 reqwest = { workspace = true }
 aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 urlencoding = { workspace = true }

crates/ecstore/src/api/mod.rs

@@ -191,7 +191,11 @@ pub mod bucket {
             OBJECT_LOCK_CONFIG, UnreadableBucketConfig, is_unreadable_config_error, load_bucket_metadata,
             table_catalog_path_hash, unreadable_config_refusal,
         };
-        pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
+        pub use crate::bucket::metadata::{
+            BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG, MAX_SWIFT_METADATA_COUNT, MAX_SWIFT_METADATA_VALUE_SIZE,
+            SWIFT_ACCOUNT_META_TAG_PREFIX, SWIFT_CONTAINER_META_TAG_PREFIX, SWIFT_TAGGING_MARKER,
+        };
+        pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, is_swift_bucket_metadata_tag_key, validate_tag_set_entries};
     }
 
     pub mod config_parse_mode {
@@ -209,19 +213,20 @@ pub mod bucket {
     pub mod metadata_sys {
         pub use crate::bucket::metadata_sys::{
             BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys,
-            ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
+            ObjectLockConfigState, ScannerBucketIncarnationReadGuard, acquire_bucket_metadata_transaction_lock,
             acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
-            capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_if_incarnation_at,
-            delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy, get_bucket_policy_raw,
-            get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
+            acquire_scanner_bucket_incarnation_read_fence, capture_bucket_metadata_incarnation, delete, delete_if_incarnation,
+            delete_if_incarnation_at, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
+            get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
             get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
             get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
             get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config,
-            get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
-            init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata,
+            get_request_payment_config, get_s3_tagging_config, get_sse_config, get_tagging_config, get_versioning_config,
+            get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata,
             remove_bucket_metadata_if_absent, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock,
             update_config_with, update_if_incarnation, update_if_incarnation_at, update_quota_if_incarnation,
-            update_quota_if_incarnation_at, update_under_transaction_lock, update_under_transaction_lock_at,
+            update_quota_if_incarnation_at, update_s3_tagging_config, update_under_transaction_lock,
+            update_under_transaction_lock_at,
         };
         #[cfg(feature = "test-util")]
         pub use crate::bucket::metadata_sys::{ConfigWriteLockProbe, test_support};
@@ -331,8 +336,8 @@ pub mod bucket {
     pub mod utils {
         pub use crate::bucket::utils::{
             check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
-            check_valid_bucket_name_strict, deserialize, has_bad_path_component, is_meta_bucketname, is_valid_object_prefix,
-            serialize,
+            check_valid_bucket_name_strict, deserialize, has_bad_path_component, has_xml_invalid_control_characters,
+            is_meta_bucketname, is_valid_object_prefix, serialize,
         };
     }
 
@@ -449,11 +454,18 @@ pub mod data_usage {
 pub mod disk {
     pub use crate::cluster::rpc::background_pipe::{BackgroundPipeReader, BackgroundPipeWriter, background_pipe};
     pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
+    pub use crate::disk::BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION;
     pub use crate::disk::background_memory::BackgroundEncodeBuffer;
     pub use crate::disk::disk_store::get_object_disk_read_timeout;
+    #[cfg(feature = "test-util")]
+    pub use crate::disk::disk_store::wait_for_multipart_index_bootstrap;
     pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
     #[cfg(feature = "test-util")]
     pub use crate::disk::local::{local_durability_modes, set_local_rename_data_fail_after_metadata_commit};
+    pub use crate::disk::multipart_index::{
+        MULTIPART_INDEX_RPC_MAX_BYTES, MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES, MultipartIndexPage, MultipartIndexRpcReply,
+        MultipartIndexRpcRequest,
+    };
     pub use crate::disk::multipart_source::{
         MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind,
         MultipartAbortRound, MultipartSourceOperation, MultipartSourcePhase, MultipartSourceRequest, MultipartSourceResponse,
@@ -466,13 +478,15 @@ pub mod disk {
     pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
     pub use crate::disk::{
         BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
-        CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
+        BoundedDeleteRecoveryDiscovery, BoundedDeleteRecoveryInspection, BoundedDeleteRecoverySettlement,
+        BoundedDeleteRecoverySettlementReply, CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration,
+        CurrentDeleteTransition, DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, DELETE_RECOVERY_MAX_INSPECTION_ENTRIES,
         DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
         DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
         Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
-        GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
-        POOL_META_NATIVE_CAS_METADATA_SUFFIX, PartTransactionAction, PoolMetaNativeBackup, PoolMetaNativeInspection,
-        PoolMetaNativeObject, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
+        GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEAL_REPLACEMENT_RECOVERY_DIR, HEALING_MARKER_PATH,
+        NsScannerOpenRequest, OldCurrentSize, POOL_META_NATIVE_CAS_METADATA_SUFFIX, PartTransactionAction, PoolMetaNativeBackup,
+        PoolMetaNativeInspection, PoolMetaNativeObject, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
         QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET,
         ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken,
         UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
@@ -617,7 +631,7 @@ pub mod rebalance {
 pub mod rio {
     pub use crate::io_support::rio::{
         DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
-        boxed_reader, compression_metadata_value, wrap_reader,
+        boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
     };
 }
 
@@ -654,7 +668,9 @@ pub mod set_disk {
 
     #[cfg(feature = "test-util")]
     pub mod test_util {
-        pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
+        pub use crate::bucket::quota::reservation::{
+            fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test, retry_quota_reconciliation_for_test,
+        };
         pub use crate::set_disk::ConditionalReadBenchmarkMetadataGuard;
         pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
 

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -16,7 +16,8 @@ use crate::bucket::metadata::BucketMetadata;
 use crate::bucket::metadata_sys::get_bucket_targets_config;
 use crate::bucket::metadata_sys::get_replication_config;
 use crate::bucket::remote_s3_client::{
-    PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
+    PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, RemoteS3RetryPolicy,
+    UploadResponseDeadline, build_remote_s3_client,
 };
 use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
 use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
@@ -119,6 +120,7 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
             ca_cert_pem: (!target.ca_cert_pem.trim().is_empty()).then(|| target.ca_cert_pem.clone()),
             connect_timeout: None,
             read_timeout: None,
+            operation_timeout: Some(TargetRequestTimeouts::from_env().request),
             // Replication has no retry budget of its own on the request path,
             // so it keeps the SDK's standard three attempts; stating it here
             // pins the behaviour to this line instead of an SDK default.
@@ -1191,6 +1193,7 @@ impl BucketTargetSys {
             health_check_duration: target.health_check_duration,
             replicate_sync: target.replication_sync,
             client: Arc::new(client),
+            timeouts: TargetRequestTimeouts::from_env(),
         })
     }
 
@@ -1922,6 +1925,83 @@ impl std::fmt::Display for S3ClientError {
     }
 }
 
+/// Replication request budgets, snapshotted when a target client is built.
+/// Streamed GET bodies are not subject to the upload-response budget.
+#[derive(Debug, Clone, Copy)]
+pub struct TargetRequestTimeouts {
+    request: Duration,
+    complete: Duration,
+    minimum_transfer_rate: u64,
+}
+
+impl Default for TargetRequestTimeouts {
+    fn default() -> Self {
+        Self {
+            request: Duration::from_secs(60),
+            complete: Duration::from_secs(15 * 60),
+            minimum_transfer_rate: 1024 * 1024,
+        }
+    }
+}
+
+impl TargetRequestTimeouts {
+    fn from_env() -> Self {
+        let defaults = Self::default();
+        let seconds = |name: &str, fallback: Duration| {
+            let value = rustfs_utils::get_env_u64(name, fallback.as_secs());
+            Duration::from_secs(if (1..=86400).contains(&value) {
+                value
+            } else {
+                fallback.as_secs()
+            })
+        };
+        let minimum_transfer_rate =
+            rustfs_utils::get_env_u64("RUSTFS_REPLICATION_MIN_TRANSFER_RATE_BYTES_PER_SEC", defaults.minimum_transfer_rate);
+        Self {
+            request: seconds("RUSTFS_REPLICATION_REQUEST_TIMEOUT_SECS", defaults.request),
+            complete: seconds("RUSTFS_REPLICATION_COMPLETE_TIMEOUT_SECS", defaults.complete),
+            minimum_transfer_rate: if minimum_transfer_rate == 0 {
+                defaults.minimum_transfer_rate
+            } else {
+                minimum_transfer_rate
+            },
+        }
+    }
+
+    fn upload_timeout(self, size: i64) -> Result<Duration, S3ClientError> {
+        let size = u64::try_from(size).map_err(|_| S3ClientError::new("replication upload size must be non-negative"))?;
+        // The SDK can make three attempts. Leave transmission headroom for
+        // all three; the separate response deadline still bounds a trickling
+        // target after each upload finishes.
+        let attempts = match REPLICATION_TARGET_RETRY_POLICY {
+            RemoteS3RetryPolicy::Disabled => 1,
+            RemoteS3RetryPolicy::Standard { max_attempts } => u64::from(max_attempts.max(1)),
+        };
+        let seconds = size
+            .div_ceil(self.minimum_transfer_rate)
+            .checked_mul(attempts)
+            .ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
+        let timeout = self
+            .request
+            .checked_add(Duration::from_secs(seconds))
+            .ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
+        if Instant::now().checked_add(timeout).is_none() {
+            return Err(S3ClientError::new("replication upload timeout exceeds supported duration"));
+        }
+        Ok(timeout)
+    }
+
+    fn upload_override(self, size: i64) -> Result<aws_sdk_s3::config::Builder, S3ClientError> {
+        Ok(aws_sdk_s3::config::Builder::new()
+            .timeout_config(
+                aws_smithy_types::timeout::TimeoutConfig::builder()
+                    .operation_timeout(self.upload_timeout(size)?)
+                    .build(),
+            )
+            .interceptor(UploadResponseDeadline(self.request)))
+    }
+}
+
 #[derive(Debug)]
 pub struct TargetClient {
     pub endpoint: String,
@@ -1935,6 +2015,7 @@ pub struct TargetClient {
     pub health_check_duration: Duration,
     pub replicate_sync: bool,
     pub client: Arc<S3Client>,
+    pub timeouts: TargetRequestTimeouts,
 }
 
 impl TargetClient {
@@ -2360,12 +2441,11 @@ impl TargetClient {
 
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             });
+        let mut config = self.timeouts.upload_override(size)?;
         if forwards_source_checksum {
-            operation = operation.config_override(
-                aws_sdk_s3::config::Builder::new()
-                    .request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
-            );
+            config = config.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired);
         }
+        operation = operation.config_override(config);
         match operation.send().await {
             Ok(output) => {
                 // Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
@@ -2501,6 +2581,7 @@ impl TargetClient {
                 }
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             })
+            .config_override(self.timeouts.upload_override(size)?)
             .send()
             .await
         {
@@ -2538,6 +2619,13 @@ impl TargetClient {
                 }
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             })
+            .config_override(
+                aws_sdk_s3::config::Builder::new().timeout_config(
+                    aws_smithy_types::timeout::TimeoutConfig::builder()
+                        .operation_timeout(self.timeouts.complete)
+                        .build(),
+                ),
+            )
             .send()
             .await
         {
@@ -2966,11 +3054,241 @@ mod tests {
                 health_check_duration: Duration::from_secs(5),
                 replicate_sync: false,
                 client: Arc::new(client),
+                timeouts: TargetRequestTimeouts::default(),
             },
             request_headers,
         )
     }
 
+    #[derive(Debug, Clone)]
+    struct TimedResponseConnector {
+        upload_delay: Duration,
+        response_delay: Duration,
+        prefix_bytes: usize,
+        response: bytes::Bytes,
+        frames: Arc<std::sync::atomic::AtomicUsize>,
+        dropped: Arc<std::sync::atomic::AtomicUsize>,
+    }
+
+    struct ResponseDrop(Arc<std::sync::atomic::AtomicUsize>);
+    impl Drop for ResponseDrop {
+        fn drop(&mut self) {
+            self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+        }
+    }
+
+    impl SmithyHttpConnector for TimedResponseConnector {
+        fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
+            let settings = self.clone();
+            HttpConnectorFuture::new(async move {
+                http_body_util::BodyExt::collect(request.into_body())
+                    .await
+                    .map_err(aws_smithy_runtime_api::client::result::ConnectorError::io)?;
+                tokio::time::sleep(settings.upload_delay).await;
+                let guard = ResponseDrop(Arc::clone(&settings.dropped));
+                let stream = futures::stream::unfold(
+                    (settings.prefix_bytes, Some(settings.response), true, guard),
+                    move |(remaining, mut body, first, guard)| {
+                        let frames = Arc::clone(&settings.frames);
+                        async move {
+                            if remaining == 0 && body.is_none() {
+                                return None;
+                            }
+                            if !first {
+                                tokio::time::sleep(settings.response_delay).await;
+                            }
+                            let bytes = if remaining > 0 {
+                                bytes::Bytes::from_static(b" ")
+                            } else {
+                                body.take()?
+                            };
+                            frames.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                            Some((
+                                Ok::<_, std::io::Error>(http_body::Frame::data(bytes)),
+                                (remaining.saturating_sub(1), body, false, guard),
+                            ))
+                        }
+                    },
+                )
+                .fuse();
+                let body = SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream));
+                let mut response =
+                    HttpResponse::new(aws_smithy_runtime_api::http::StatusCode::try_from(200u16).expect("status"), body);
+                response.headers_mut().insert("etag", "\"test-etag\"");
+                Ok(response)
+            })
+        }
+    }
+
+    fn timed_response_target(
+        upload_delay: Duration,
+        response_delay: Duration,
+        prefix_bytes: usize,
+        response: &'static [u8],
+    ) -> (TargetClient, Arc<std::sync::atomic::AtomicUsize>, Arc<std::sync::atomic::AtomicUsize>) {
+        let frames = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let dropped = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let connector = SharedHttpConnector::new(TimedResponseConnector {
+            upload_delay,
+            response_delay,
+            prefix_bytes,
+            response: bytes::Bytes::from_static(response),
+            frames: Arc::clone(&frames),
+            dropped: Arc::clone(&dropped),
+        });
+        let http_client = http_client_fn(move |_settings, _components| connector.clone());
+        let (mut target, _) = header_recording_target_client(Vec::new());
+        target.timeouts = TargetRequestTimeouts {
+            request: Duration::from_secs(2),
+            complete: Duration::from_secs(10),
+            minimum_transfer_rate: 1024 * 1024,
+        };
+        target.client = Arc::new(S3Client::from_conf(
+            target
+                .client
+                .config()
+                .to_builder()
+                .http_client(http_client)
+                .retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
+                .timeout_config(
+                    aws_smithy_types::timeout::TimeoutConfig::builder()
+                        .operation_timeout(target.timeouts.request)
+                        .build(),
+                )
+                .build(),
+        ));
+        (target, frames, dropped)
+    }
+
+    #[test]
+    fn replication_upload_budgets_preserve_large_transfers_and_reject_invalid_sizes() {
+        let policy = TargetRequestTimeouts::default();
+        assert_eq!(policy.upload_timeout(0).expect("empty upload"), Duration::from_secs(60));
+        assert_eq!(
+            policy.upload_timeout(5 * 1024 * 1024 * 1024).expect("maximum single PUT"),
+            Duration::from_secs(15420)
+        );
+        assert!(policy.upload_timeout(-1).is_err());
+        let slow = TargetRequestTimeouts {
+            minimum_transfer_rate: 1,
+            ..policy
+        };
+        assert!(slow.upload_timeout(i64::MAX).is_err());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_upload_response_deadline_is_independent_of_size_and_checksum_override() {
+        for operation in ["put", "forwarded-checksum", "part"] {
+            let (target, frames, dropped) = timed_response_target(Duration::from_secs(4), Duration::from_millis(500), 100, b"");
+            let body = vec![b'x'; 2 * 1024 * 1024];
+            let size = i64::try_from(body.len()).expect("size");
+            let started = tokio::time::Instant::now();
+            let error = if operation == "part" {
+                target
+                    .put_object_part(
+                        "bucket",
+                        "key",
+                        "upload",
+                        1,
+                        size,
+                        ByteStream::from(body),
+                        &PutObjectPartOptions::default(),
+                    )
+                    .await
+                    .expect_err("response deadline")
+            } else {
+                let mut opts = PutObjectOptions::default();
+                if operation == "forwarded-checksum" {
+                    opts.user_metadata.insert(
+                        "x-amz-checksum-sha256".to_owned(),
+                        "OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_owned(),
+                    );
+                }
+                target
+                    .put_object("bucket", "key", size, ByteStream::from(body), &opts)
+                    .await
+                    .expect_err("response deadline")
+            };
+            assert!(
+                started.elapsed() >= Duration::from_secs(6),
+                "upload gets its full independent window: {error}"
+            );
+            assert!(
+                started.elapsed() < target.timeouts.upload_timeout(size).expect("budget"),
+                "response must end before the larger total deadline"
+            );
+            assert!(
+                frames.load(std::sync::atomic::Ordering::SeqCst) >= 2,
+                "trickle delivered progress before failure"
+            );
+            assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1, "transport body released");
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_slow_upload_preserves_transfer_window_and_streaming_get_body() {
+        let (target, _, _) = timed_response_target(Duration::from_secs(4), Duration::from_millis(100), 5, b"");
+        let body = vec![b'x'; 2 * 1024 * 1024];
+        let size = i64::try_from(body.len()).expect("size");
+        target
+            .put_object("bucket", "key", size, ByteStream::from(body), &PutObjectOptions::default())
+            .await
+            .expect("slow upload, prompt response");
+        let (target, _, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), 5, b"payload");
+        let response = target
+            .get_object("bucket", "key", None, None, None, HeaderMap::new())
+            .await
+            .expect("GET headers");
+        let started = tokio::time::Instant::now();
+        let body = response
+            .body
+            .collect()
+            .await
+            .expect("streamed GET remains outside upload deadline")
+            .into_bytes();
+        assert!(started.elapsed() > target.timeouts.request);
+        assert_eq!(&body[..], b"     payload");
+        assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_complete_processing_whitespace_has_a_separate_bounded_window() {
+        const RESULT: &[u8] = b"<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>\"etag\"</ETag></CompleteMultipartUploadResult>";
+        for prefix in [5, 100] {
+            let (target, frames, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), prefix, RESULT);
+            let started = tokio::time::Instant::now();
+            let result = target
+                .complete_multipart_upload(
+                    "bucket",
+                    "key",
+                    "upload",
+                    vec![CompletedPart::builder().part_number(1).e_tag("etag").build()],
+                    &PutObjectOptions::default(),
+                )
+                .await;
+            if prefix == 5 {
+                assert!(result.is_ok(), "legal processing whitespace: {result:?}");
+            } else {
+                assert!(result.is_err());
+                assert!(started.elapsed() <= Duration::from_secs(11));
+            }
+            assert!(started.elapsed() > target.timeouts.request);
+            assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
+            assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_control_operation_deadline_covers_response_trickle() {
+        let (target, frames, dropped) =
+            timed_response_target(Duration::ZERO, Duration::from_millis(500), 100, b"<VersioningConfiguration/>");
+        let started = tokio::time::Instant::now();
+        assert!(target.get_bucket_versioning("bucket").await.is_err());
+        assert!(started.elapsed() <= Duration::from_secs(3));
+        assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
+        assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
+    }
+
     fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
         let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
         let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
@@ -3279,6 +3597,7 @@ mod tests {
                 health_check_duration: Duration::from_secs(5),
                 replicate_sync: false,
                 client: Arc::new(client),
+                timeouts: TargetRequestTimeouts::default(),
             },
             request_uris,
         )
@@ -3591,6 +3910,7 @@ mod tests {
             health_check_duration: Duration::from_secs(5),
             replicate_sync: false,
             client: Arc::new(client),
+            timeouts: TargetRequestTimeouts::default(),
         })
     }
 

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -5487,6 +5487,14 @@ pub async fn put_restore_opts(
     if !oi.user_tags.is_empty() {
         meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
     }
+    if let Some(expires) = oi.expires {
+        meta.insert(
+            "expires".to_string(),
+            expires
+                .format(&time::format_description::well_known::Rfc3339)
+                .map_err(std::io::Error::other)?,
+        );
+    }
     let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
     meta.insert(
         metadata_keys::RESTORE.to_string(),
@@ -5506,7 +5514,6 @@ pub async fn put_restore_opts(
         // computed MD5 is not the object's public plaintext ETag.
         preserve_etag: oi.etag.clone(),
         shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
-        //expires:           oi.expires,
         ..Default::default()
     })
 }
@@ -7144,6 +7151,25 @@ mod tests {
             0,
             "tier generation lease should release after EOF"
         );
+        let unread = get_transitioned_object_reader_with_tier_manager(
+            &object_info.bucket,
+            &object_info.name,
+            &None,
+            &HeaderMap::new(),
+            &object_info,
+            &ObjectOptions::default(),
+            &manager,
+            None,
+        )
+        .await
+        .expect("open an unread transitioned response");
+        assert_eq!(TierConfigMgr::active_operation_lease_count(&manager, &tier).await, 1);
+        drop(unread);
+        assert_eq!(
+            TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
+            0,
+            "canceling an unread response must release the tier lease"
+        );
     }
 
     #[cfg(feature = "test-util")]

crates/ecstore/src/bucket/metadata.rs

@@ -256,6 +256,128 @@ pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
 pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
 pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
 pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
+/// Validate tag entries supplied to the S3 API.
+pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
+    validate_tag_entries(tag_set, validate_s3_tag_entry)
+}
+
+fn validate_tag_entries(
+    tag_set: &[s3s::dto::Tag],
+    mut validate_entry: impl FnMut(&str, &str) -> std::result::Result<(), &'static str>,
+) -> std::result::Result<(), &'static str> {
+    let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
+    for tag in tag_set {
+        let key = tag
+            .key
+            .as_deref()
+            .filter(|key| !key.is_empty())
+            .ok_or("Tag key cannot be empty")?;
+        let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
+        validate_entry(key, value)?;
+        if !keys.insert(key) {
+            return Err("Cannot provide multiple Tags with the same key");
+        }
+    }
+    Ok(())
+}
+
+fn validate_s3_tag_entry(key: &str, value: &str) -> std::result::Result<(), &'static str> {
+    fn is_valid_tag_text(value: &str) -> bool {
+        static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
+        VALID_TAG_TEXT
+            .get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
+            .as_ref()
+            .is_ok_and(|pattern| pattern.is_match(value))
+    }
+    if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
+        return Err("The TagKey you have provided is invalid");
+    }
+    if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
+        return Err("The TagValue you have provided is invalid");
+    }
+    Ok(())
+}
+
+pub const MAX_BUCKET_TAGS: usize = 50;
+
+/// Maximum number of metadata headers on a Swift resource.
+pub const MAX_SWIFT_METADATA_COUNT: usize = 90;
+/// Maximum byte length of a Swift metadata value.
+pub const MAX_SWIFT_METADATA_VALUE_SIZE: usize = 256;
+
+/// Prefix for Swift account metadata persisted in tagging.xml.
+pub const SWIFT_ACCOUNT_META_TAG_PREFIX: &str = "swift-account-meta-";
+/// Prefix for Swift container metadata persisted in tagging.xml.
+pub const SWIFT_CONTAINER_META_TAG_PREFIX: &str = "swift-meta-";
+
+/// Marks a bucket whose shared tag config contains Swift-owned metadata.
+/// The key cannot be supplied through S3's tag alphabet.
+pub const SWIFT_TAGGING_MARKER: &str = "!rustfs-swift-metadata";
+
+/// Whether a persisted tag belongs to Swift's internal metadata namespace.
+pub fn is_swift_bucket_metadata_tag_key(key: &str) -> bool {
+    matches!(
+        key,
+        "swift-acl-read" | "swift-acl-write" | "swift-versions-location" | SWIFT_TAGGING_MARKER
+    ) || key.starts_with(SWIFT_ACCOUNT_META_TAG_PREFIX)
+        || key.starts_with(SWIFT_CONTAINER_META_TAG_PREFIX)
+}
+
+fn validate_persisted_bucket_tags(tags: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
+    const SWIFT_ACL_MAX_BYTES: usize = 8192;
+    const SWIFT_META_NAME_MAX_BYTES: usize = 128;
+    const MAX_PERSISTED_TAGS: usize = MAX_BUCKET_TAGS + 2 * MAX_SWIFT_METADATA_COUNT + 4;
+    if tags.len() > MAX_PERSISTED_TAGS {
+        return Err("Too many persisted bucket tags");
+    }
+
+    // Swift stores ACLs, versioning and metadata in the same XML as S3 tags.
+    // Only those known keys may carry values outside the S3 tag alphabet.
+    let mut account_metadata = 0;
+    let mut container_metadata = 0;
+    let mut s3_tags = 0;
+    validate_tag_entries(tags, |key, value| {
+        let swift_metadata = key
+            .strip_prefix(SWIFT_ACCOUNT_META_TAG_PREFIX)
+            .map(|name| (name, &mut account_metadata))
+            .or_else(|| {
+                key.strip_prefix(SWIFT_CONTAINER_META_TAG_PREFIX)
+                    .map(|name| (name, &mut container_metadata))
+            });
+        if let Some((name, count)) = swift_metadata
+            && name.len() <= SWIFT_META_NAME_MAX_BYTES
+            && http::HeaderName::from_bytes(name.as_bytes()).is_ok()
+            && value.len() <= MAX_SWIFT_METADATA_VALUE_SIZE
+            && http::HeaderValue::from_str(value).is_ok()
+        {
+            *count += 1;
+            if *count > MAX_SWIFT_METADATA_COUNT {
+                return Err("Too many Swift metadata tags");
+            }
+            return Ok(());
+        }
+
+        if key == SWIFT_TAGGING_MARKER && value == "1" {
+            return Ok(());
+        }
+
+        let valid_swift_header = match key {
+            "swift-acl-read" | "swift-acl-write" => value.len() <= SWIFT_ACL_MAX_BYTES,
+            "swift-versions-location" => value.len() <= 256 && !value.is_empty() && !value.contains('/'),
+            _ => false,
+        };
+        if valid_swift_header && http::HeaderValue::from_str(value).is_ok() {
+            return Ok(());
+        }
+
+        s3_tags += 1;
+        if s3_tags > MAX_BUCKET_TAGS {
+            return Err("Cannot have more than 50 tags per bucket");
+        }
+        validate_s3_tag_entry(key, value)
+    })
+}
+
 pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
 pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
 pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
@@ -990,6 +1112,13 @@ impl BucketMetadata {
                 self.encryption_config_updated_at = updated;
             }
             BUCKET_TAGGING_CONFIG => {
+                if !data.is_empty() {
+                    let tagging = deserialize::<Tagging>(&data).map_err(|_| {
+                        Error::InvalidArgument(self.name.clone(), String::new(), "Invalid bucket tagging XML".to_owned())
+                    })?;
+                    validate_persisted_bucket_tags(&tagging.tag_set)
+                        .map_err(|message| Error::InvalidArgument(self.name.clone(), String::new(), message.to_owned()))?;
+                }
                 self.tagging_config_xml = data;
                 // Drop the parsed form (like lifecycle above) so clearing the
                 // payload can't leave stale parsed tags to be cached.
@@ -1781,9 +1910,68 @@ mod test {
         assert_eq!(metadata.bucket_incarnation_id, incarnation);
     }
 
+    #[test]
+    fn persisted_bucket_tags_keep_separate_s3_and_swift_limits() {
+        use s3s::dto::Tag;
+
+        let tag = |key: String, value: &str| Tag {
+            key: Some(key),
+            value: Some(value.to_owned()),
+        };
+        let acl = tag("swift-acl-read".to_owned(), ".r:*,AUTH_other");
+        assert_eq!(
+            validate_tag_set_entries(std::slice::from_ref(&acl)),
+            Err("The TagValue you have provided is invalid")
+        );
+
+        let mut tags = vec![acl.clone()];
+        tags.extend((0..MAX_BUCKET_TAGS).map(|i| tag(format!("team-{i}"), "blue")));
+        tags.extend((0..MAX_SWIFT_METADATA_COUNT).map(|i| tag(format!("{SWIFT_CONTAINER_META_TAG_PREFIX}item!{i}"), "a,b")));
+        assert_eq!(validate_persisted_bucket_tags(&tags), Ok(()));
+
+        tags.push(tag(format!("{SWIFT_CONTAINER_META_TAG_PREFIX}overflow"), "a,b"));
+        assert_eq!(validate_persisted_bucket_tags(&tags), Err("Too many Swift metadata tags"));
+        tags.pop();
+
+        tags.push(acl);
+        assert_eq!(
+            validate_persisted_bucket_tags(&tags),
+            Err("Cannot provide multiple Tags with the same key")
+        );
+        tags.pop();
+
+        tags[0].value = Some(".r:*,".repeat(1700));
+        assert_eq!(validate_persisted_bucket_tags(&tags), Err("The TagValue you have provided is invalid"));
+    }
+
     /// backlog#2292: a replicated config is stamped with the source
     /// `updated_at` it was given, not the local clock, while the plain
     /// `update_config` entry keeps stamping the local clock.
+    #[test]
+    fn bucket_tag_updates_reject_unreadable_shapes_without_replacing_metadata() {
+        let mut metadata = BucketMetadata::new("validated-tags");
+        let valid = br#"<Tagging><TagSet><Tag><Key>team</Key><Value></Value></Tag></TagSet></Tagging>"#.to_vec();
+        metadata
+            .update_config(BUCKET_TAGGING_CONFIG, valid.clone())
+            .expect("empty value is valid");
+        let updated = metadata.tagging_config_updated_at;
+        for invalid in [
+            "<Tagging><TagSet><Tag><Key>team</Key></Tag></TagSet></Tagging>".to_owned(),
+            "<Tagging><TagSet><Tag><Key></Key><Value>x</Value></Tag></TagSet></Tagging>".to_owned(),
+            "<Tagging><TagSet><Tag><Key>team</Key><Value>x</Value></Tag><Tag><Key>team</Key><Value>y</Value></Tag></TagSet></Tagging>".to_owned(),
+            format!("<Tagging><TagSet>{}</TagSet></Tagging>", (0..51).map(|i| format!("<Tag><Key>k{i}</Key><Value>v</Value></Tag>")).collect::<String>()),
+        ] {
+            assert!(metadata.update_config_at(BUCKET_TAGGING_CONFIG, invalid.into_bytes(), updated + time::Duration::hours(1)).is_err());
+            assert_eq!(metadata.tagging_config_xml, valid);
+            assert_eq!(metadata.tagging_config_updated_at, updated);
+        }
+        metadata.tagging_config_xml = b"legacy corrupt tags".to_vec();
+        metadata
+            .update_config(BUCKET_TAGGING_CONFIG, Vec::new())
+            .expect("deletion repairs legacy corruption");
+        assert!(metadata.tagging_config_xml.is_empty());
+    }
+
     #[test]
     fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
         let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);

crates/ecstore/src/bucket/metadata_sys.rs

@@ -22,6 +22,7 @@ use super::target::BucketTargets;
 use crate::bucket::bucket_target_sys::BucketTargetSys;
 use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_parse_with_presence};
 use crate::bucket::utils::is_meta_bucketname;
+use crate::diagnostics::object_lock::ObjectLockAttempt;
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
 use crate::object_api::ObjectOptions;
@@ -599,7 +600,9 @@ pub(crate) async fn get_bucket_incarnation_id_for_options_in(
     }
     let sys = bucket_metadata_sys_of(ctx)?;
     let sys = sys.read().await.clone();
-    let guard = acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
+    let guard =
+        acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts, "bucket_incarnation_options_read")
+            .await?;
     sys.get_bucket_incarnation_id_under_transaction_lock(bucket, &guard).await
 }
 
@@ -683,6 +686,7 @@ pub(crate) async fn update_validated_in<F>(
     bucket: &str,
     config_file: &str,
     data: Vec<u8>,
+    expected_incarnation_id: Option<Uuid>,
     validate: F,
 ) -> Result<OffsetDateTime>
 where
@@ -690,7 +694,7 @@ where
 {
     Box::pin(async {
         let sys = require_bucket_metadata_sys_in(ctx)?;
-        let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
+        let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
         validate()?;
         update_under_config_write_guard(sys, &guard, config_file, data, None).await
     })
@@ -865,6 +869,30 @@ impl BucketMetadataMutationGuard {
     }
 }
 
+/// Read-only scanner authority for one bucket incarnation.
+///
+/// Lock order is bucket lifecycle (read), then metadata transaction (read).
+/// This guard cannot authorize a configuration mutation or scanner ACK write.
+pub struct ScannerBucketIncarnationReadGuard {
+    bucket: String,
+    incarnation_id: Uuid,
+    lifecycle_guard: rustfs_lock::NamespaceLockGuard,
+    transaction_guard: rustfs_lock::NamespaceLockGuard,
+}
+
+impl ScannerBucketIncarnationReadGuard {
+    /// Returns the verified identity while both read fences remain valid.
+    ///
+    /// # Errors
+    /// Returns an error if either lock has lost its lease.
+    pub fn checked_bucket_incarnation(&self) -> Result<(&str, Uuid)> {
+        if self.lifecycle_guard.is_lock_lost() || self.transaction_guard.is_lock_lost() {
+            return Err(Error::other("scanner bucket incarnation read lock was lost"));
+        }
+        Ok((&self.bucket, self.incarnation_id))
+    }
+}
+
 async fn acquire_config_write_guard(sys: Arc<RwLock<BucketMetadataSys>>, bucket: &str) -> Result<BucketMetadataMutationGuard> {
     acquire_config_write_guard_for_incarnation(sys, bucket, None).await
 }
@@ -891,6 +919,68 @@ pub async fn acquire_scanner_bucket_incarnation_fence(
     acquire_config_write_guard_with_migration(sys, bucket, Some(expected_incarnation_id), false).await
 }
 
+/// Validate scanner read authority without serializing compatible metadata readers.
+///
+/// The returned guard retains both read locks. It never migrates metadata or
+/// creates an incarnation; mutation callers must use the exclusive scanner fence.
+///
+/// # Errors
+/// Returns an error for invalid bucket/owner identities, unavailable locks or
+/// storage, absent incarnation metadata, or a deleted/recreated bucket.
+pub async fn acquire_scanner_bucket_incarnation_read_fence(
+    bucket: &str,
+    expected_incarnation_id: Uuid,
+    expected_owner_id: Uuid,
+) -> Result<ScannerBucketIncarnationReadGuard> {
+    acquire_scanner_bucket_incarnation_read_fence_with_sys(
+        get_bucket_metadata_sys()?,
+        bucket,
+        expected_incarnation_id,
+        expected_owner_id,
+    )
+    .await
+}
+
+async fn acquire_scanner_bucket_incarnation_read_fence_with_sys(
+    sys: Arc<RwLock<BucketMetadataSys>>,
+    bucket: &str,
+    expected_incarnation_id: Uuid,
+    expected_owner_id: Uuid,
+) -> Result<ScannerBucketIncarnationReadGuard> {
+    super::utils::check_valid_bucket_name(bucket)?;
+    let metadata_sys = sys.read().await.clone();
+    if expected_owner_id.is_nil() || metadata_sys.object_store().id != expected_owner_id || expected_incarnation_id.is_nil() {
+        return Err(Error::other("scanner bucket incarnation owner does not match"));
+    }
+    let lifecycle_guard = metadata_sys.object_store().acquire_bucket_lifecycle_read_lock(bucket).await?;
+    let transaction_guard = await_bucket_namespace_operation(
+        Some(&lifecycle_guard),
+        bucket,
+        "scanner incarnation read transaction lock acquisition",
+        acquire_metadata_transaction_guard(
+            metadata_sys.object_store(),
+            bucket,
+            "scanner_incarnation_probe",
+            rustfs_lock::LockType::Shared,
+        ),
+    )
+    .await?;
+    let incarnation_id = validate_bucket_metadata_guard_incarnation(
+        &metadata_sys,
+        bucket,
+        Some(expected_incarnation_id),
+        &lifecycle_guard,
+        &transaction_guard,
+    )
+    .await?;
+    Ok(ScannerBucketIncarnationReadGuard {
+        bucket: bucket.to_owned(),
+        incarnation_id,
+        lifecycle_guard,
+        transaction_guard,
+    })
+}
+
 async fn acquire_config_write_guard_with_migration(
     sys: Arc<RwLock<BucketMetadataSys>>,
     bucket: &str,
@@ -916,15 +1006,47 @@ async fn acquire_config_write_guard_with_migration(
         Some(&lifecycle_guard),
         bucket,
         "bucket config transaction lock acquisition",
-        acquire_transaction_lock_with_sys(&sys, bucket),
+        acquire_transaction_lock_with_sys_for_purpose(
+            &sys,
+            bucket,
+            if migrate {
+                "bucket_config_update"
+            } else {
+                "scanner_incarnation_validation"
+            },
+        ),
     )
     .await?;
+    let current_incarnation_id = validate_bucket_metadata_guard_incarnation(
+        &metadata_sys,
+        bucket,
+        expected_incarnation_id,
+        &lifecycle_guard,
+        &transaction_guard,
+    )
+    .await?;
+
+    Ok(BucketMetadataMutationGuard {
+        bucket: bucket.to_string(),
+        incarnation_id: current_incarnation_id,
+        lifecycle_guard,
+        transaction_guard,
+    })
+}
+
+async fn validate_bucket_metadata_guard_incarnation(
+    metadata_sys: &BucketMetadataSys,
+    bucket: &str,
+    expected_incarnation_id: Option<Uuid>,
+    lifecycle_guard: &rustfs_lock::NamespaceLockGuard,
+    transaction_guard: &rustfs_lock::NamespaceLockGuard,
+) -> Result<Uuid> {
     await_bucket_namespace_operation(
-        Some(&lifecycle_guard),
+        Some(lifecycle_guard),
         bucket,
         "bucket config existence validation",
         await_bucket_namespace_operation(
-            Some(&transaction_guard),
+            Some(transaction_guard),
             bucket,
             "bucket config existence transaction validation",
             async {
@@ -942,11 +1064,11 @@ async fn acquire_config_write_guard_with_migration(
     )
     .await?;
     let current_incarnation_id = await_bucket_namespace_operation(
-        Some(&lifecycle_guard),
+        Some(lifecycle_guard),
         bucket,
         "bucket config incarnation validation",
         await_bucket_namespace_operation(
-            Some(&transaction_guard),
+            Some(transaction_guard),
             bucket,
             "bucket config incarnation transaction validation",
             load_bucket_incarnation(metadata_sys.object_store(), bucket),
@@ -959,12 +1081,7 @@ async fn acquire_config_write_guard_with_migration(
         return Err(Error::BucketNotFound(bucket.to_string()));
     }
 
-    Ok(BucketMetadataMutationGuard {
-        bucket: bucket.to_string(),
-        incarnation_id: current_incarnation_id,
-        lifecycle_guard,
-        transaction_guard,
-    })
+    Ok(current_incarnation_id)
 }
 
 /// Rewrite one config file while the caller already holds this bucket's
@@ -1173,11 +1290,23 @@ async fn delete_under_config_write_guard(
 /// targets one, so a writer on another node cannot land a whole-file save in
 /// the middle of this read-modify-write.
 pub async fn update_config_with<F>(bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
+where
+    F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
+{
+    update_config_with_expected(bucket, config_file, None, mutate).await
+}
+
+async fn update_config_with_expected<F>(
+    bucket: &str,
+    config_file: &str,
+    expected_incarnation_id: Option<Uuid>,
+    mutate: F,
+) -> Result<OffsetDateTime>
 where
     F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
 {
     let sys = get_bucket_metadata_sys()?;
-    let guard = Box::pin(acquire_config_write_guard(sys.clone(), bucket)).await?;
+    let guard = Box::pin(acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id)).await?;
     guard.ensure_valid(bucket)?;
     let metadata_sys = sys.read().await.clone();
     let updated = await_bucket_namespace_operation(
@@ -1196,6 +1325,70 @@ where
     Ok(updated)
 }
 
+fn swift_manages_bucket_tags(tagging: &Tagging) -> bool {
+    tagging
+        .tag_set
+        .iter()
+        .any(|tag| tag.key.as_deref() == Some(super::metadata::SWIFT_TAGGING_MARKER) && tag.value.as_deref() == Some("1"))
+}
+
+/// Read the S3 tag set while keeping Swift's internal tags private.
+pub async fn get_s3_tagging_config(bucket: &str) -> Result<(Tagging, OffsetDateTime)> {
+    let (mut tagging, updated) = get_tagging_config(bucket).await?;
+    if swift_manages_bucket_tags(&tagging) {
+        tagging.tag_set.retain(|tag| {
+            !tag.key
+                .as_deref()
+                .is_some_and(super::metadata::is_swift_bucket_metadata_tag_key)
+        });
+    }
+    Ok((tagging, updated))
+}
+
+/// Replace or clear S3 tags atomically while retaining Swift metadata.
+///
+/// The expected incarnation, when present, is checked under the bucket
+/// lifecycle and metadata guards. A failed write leaves the previous config
+/// intact. Repeated writes replace the same S3 set.
+pub async fn update_s3_tagging_config(
+    bucket: &str,
+    tagging: Option<Tagging>,
+    expected_incarnation_id: Option<Uuid>,
+) -> Result<OffsetDateTime> {
+    let tags = tagging.map(|tagging| tagging.tag_set).unwrap_or_default();
+    let invalid = |message: &str| Error::InvalidArgument(bucket.to_owned(), String::new(), message.to_owned());
+    if tags.len() > super::metadata::MAX_BUCKET_TAGS {
+        return Err(invalid("Cannot have more than 50 tags per bucket"));
+    }
+    super::metadata::validate_tag_set_entries(&tags).map_err(invalid)?;
+    update_config_with_expected(bucket, BUCKET_TAGGING_CONFIG, expected_incarnation_id, |metadata| {
+        let mut merged = metadata.tagging_config.clone().unwrap_or_default();
+        if swift_manages_bucket_tags(&merged) {
+            if tags.iter().any(|tag| {
+                tag.key
+                    .as_deref()
+                    .is_some_and(super::metadata::is_swift_bucket_metadata_tag_key)
+            }) {
+                return Err(invalid("Tag key is reserved for Swift metadata on this bucket"));
+            }
+            merged.tag_set.retain(|tag| {
+                tag.key
+                    .as_deref()
+                    .is_some_and(super::metadata::is_swift_bucket_metadata_tag_key)
+            });
+        } else {
+            merged.tag_set.clear();
+        }
+        merged.tag_set.extend(tags);
+        if merged.tag_set.is_empty() {
+            Ok(Vec::new())
+        } else {
+            super::utils::serialize(&merged).map_err(Error::other)
+        }
+    })
+    .await
+}
+
 /// Acquire a bucket's metadata transaction lock, held across a whole
 /// read-modify-write of its metadata file.
 ///
@@ -1234,13 +1427,18 @@ pub(crate) async fn acquire_bucket_metadata_transaction_lock_in(
 pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_in(
     ctx: &crate::runtime::instance::InstanceContext,
     bucket: &str,
+) -> Result<rustfs_lock::NamespaceLockGuard> {
+    acquire_bucket_metadata_transaction_read_lock_in_for_purpose(ctx, bucket, "bucket_metadata_read").await
+}
+
+pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_in_for_purpose(
+    ctx: &crate::runtime::instance::InstanceContext,
+    bucket: &str,
+    op: &'static str,
 ) -> Result<rustfs_lock::NamespaceLockGuard> {
     let sys = bucket_metadata_sys_of(ctx)?;
     let api = sys.read().await.object_store();
-    let lock = api
-        .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
-        .await?;
-    Ok(lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?)
+    acquire_metadata_transaction_guard(api, bucket, op, rustfs_lock::LockType::Shared).await
 }
 
 /// Readers already holding an Object Lock snapshot must share its guard:
@@ -1249,6 +1447,7 @@ pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_for_options_in
     ctx: &crate::runtime::instance::InstanceContext,
     bucket: &str,
     opts: &ObjectOptions,
+    op: &'static str,
 ) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
     if let Some(snapshot) = opts.object_lock_config_snapshot.as_ref() {
         let store = object_store_in(ctx).await?;
@@ -1258,12 +1457,22 @@ pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_for_options_in
                 Error::other("Object Lock snapshot does not hold a valid metadata transaction fence for this bucket")
             });
     }
-    Ok(Arc::new(acquire_bucket_metadata_transaction_read_lock_in(ctx, bucket).await?))
+    Ok(Arc::new(
+        acquire_bucket_metadata_transaction_read_lock_in_for_purpose(ctx, bucket, op).await?,
+    ))
 }
 
 async fn acquire_transaction_lock_with_sys(
     sys: &Arc<RwLock<BucketMetadataSys>>,
     bucket: &str,
+) -> Result<rustfs_lock::NamespaceLockGuard> {
+    acquire_transaction_lock_with_sys_for_purpose(sys, bucket, "bucket_metadata_transaction_write").await
+}
+
+async fn acquire_transaction_lock_with_sys_for_purpose(
+    sys: &Arc<RwLock<BucketMetadataSys>>,
+    bucket: &str,
+    op: &'static str,
 ) -> Result<rustfs_lock::NamespaceLockGuard> {
     // Resolve the store under a short-lived read guard: this runs before the
     // write guard in `acquire_config_write_guards`, and must not still hold a
@@ -1272,12 +1481,22 @@ async fn acquire_transaction_lock_with_sys(
     let lock = api
         .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
         .await?;
-    let acquire = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout());
+    let timeout = crate::set_disk::get_lock_acquire_timeout();
+    let mut attempt = ObjectLockAttempt::start(
+        op,
+        RUSTFS_META_BUCKET,
+        &bucket_metadata_transaction_lock_key(bucket),
+        None,
+        &lock,
+        "write",
+        timeout,
+    );
+    let acquire = lock.get_write_lock(timeout);
     #[cfg(any(test, feature = "test-util"))]
     {
         tokio::pin!(acquire);
         let mut notified = false;
-        let guard = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
+        let result = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
             std::task::Poll::Pending => {
                 if !notified {
                     notify_config_write_lock_attempt(bucket);
@@ -1287,11 +1506,38 @@ async fn acquire_transaction_lock_with_sys(
             }
             std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
         })
-        .await?;
-        Ok(guard)
+        .await;
+        attempt.observe_acquisition(&result);
+        Ok(result?)
     }
     #[cfg(not(any(test, feature = "test-util")))]
-    Ok(acquire.await?)
+    {
+        let result = acquire.await;
+        attempt.observe_acquisition(&result);
+        Ok(result?)
+    }
+}
+
+pub(crate) async fn acquire_metadata_transaction_guard(
+    api: Arc<ECStore>,
+    bucket: &str,
+    op: &'static str,
+    lock_type: rustfs_lock::LockType,
+) -> Result<rustfs_lock::NamespaceLockGuard> {
+    let object = bucket_metadata_transaction_lock_key(bucket);
+    let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
+    let timeout = crate::set_disk::get_lock_acquire_timeout();
+    let mode = match lock_type {
+        rustfs_lock::LockType::Shared => "read",
+        rustfs_lock::LockType::Exclusive => "write",
+    };
+    let mut attempt = ObjectLockAttempt::start(op, RUSTFS_META_BUCKET, &object, None, &lock, mode, timeout);
+    let result = match lock_type {
+        rustfs_lock::LockType::Shared => lock.get_read_lock(timeout).await,
+        rustfs_lock::LockType::Exclusive => lock.get_write_lock(timeout).await,
+    };
+    attempt.observe_acquisition(&result);
+    Ok(result?)
 }
 
 /// The lock resource name is deliberately still the `bucket-targets` one it
@@ -1901,6 +2147,15 @@ impl BucketMetadataSys {
         self.api.upgrade()
     }
 
+    /// Whether the backing `ECStore` has been dropped.
+    ///
+    /// Used by `InstanceContext::clear_bucket_metadata_sys_if_stale` during
+    /// `ECStore::drop` so a shared bootstrap context does not retain a
+    /// `BucketMetadataSys` whose weak handle is dangling.
+    pub(crate) fn is_store_stale(&self) -> bool {
+        self.api.upgrade().is_none()
+    }
+
     fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
         Self::bucket_lock_in(&self.metadata_publish_locks, bucket)
     }
@@ -2784,13 +3039,13 @@ impl BucketMetadataSys {
     }
 
     async fn find_bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Option<Uuid>> {
-        let transaction_lock = self
-            .object_store()
-            .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
-            .await?;
-        let _transaction_guard = transaction_lock
-            .get_read_lock(crate::set_disk::get_lock_acquire_timeout())
-            .await?;
+        let _transaction_guard = acquire_metadata_transaction_guard(
+            self.object_store(),
+            bucket,
+            "bucket_incarnation_disk_read",
+            rustfs_lock::LockType::Shared,
+        )
+        .await?;
         self.find_bucket_incarnation_id_under_transaction_lock(bucket, &_transaction_guard)
             .await
     }
@@ -2866,13 +3121,13 @@ impl BucketMetadataSys {
     /// revalidated at write quorum, so a stale snapshot can neither revert an
     /// acknowledged configuration update nor outlive a delete/recreate.
     async fn migrate_bucket_creation_commit(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
-        let transaction_lock = self
-            .object_store()
-            .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
-            .await?;
-        let transaction_guard = transaction_lock
-            .get_write_lock(crate::set_disk::get_lock_acquire_timeout())
-            .await?;
+        let transaction_guard = acquire_metadata_transaction_guard(
+            self.object_store(),
+            bucket,
+            "bucket_creation_commit_migration",
+            rustfs_lock::LockType::Exclusive,
+        )
+        .await?;
         let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?;
         let namespace_guard = namespace_lock
             .get_read_lock(crate::set_disk::get_lock_acquire_timeout())
@@ -2922,13 +3177,13 @@ impl BucketMetadataSys {
     }
 
     async fn migrate_legacy_metadata(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
-        let transaction_lock = self
-            .object_store()
-            .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
-            .await?;
-        let _transaction_guard = transaction_lock
-            .get_write_lock(crate::set_disk::get_lock_acquire_timeout())
-            .await?;
+        let _transaction_guard = acquire_metadata_transaction_guard(
+            self.object_store(),
+            bucket,
+            "bucket_metadata_legacy_migration",
+            rustfs_lock::LockType::Exclusive,
+        )
+        .await?;
 
         let authority = self
             .load_authoritative_metadata_from_disk_under_transaction_lock(bucket)
@@ -3418,7 +3673,7 @@ mod tests {
         ),
         (
             BUCKET_TAGGING_CONFIG,
-            r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
+            r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-rust</Value></Tag></TagSet></Tagging>"#.as_bytes(),
         ),
         (BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
         (
@@ -3967,6 +4222,12 @@ mod tests {
                 .await
                 .is_err()
         );
+        assert!(
+            acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, Uuid::new_v4(), store.id)
+                .await
+                .is_err(),
+            "read-only scanner validation must not migrate an absent incarnation"
+        );
         assert!(load_bucket_incarnation(store, bucket).await.expect("read sidecar").is_none());
         assert!(
             sys.read()
@@ -4023,6 +4284,128 @@ mod tests {
         );
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    #[serial]
+    async fn scanner_incarnation_read_fence_shares_reads_and_blocks_mutations() {
+        let (_dirs, store) = isolated_store_over_temp_disks().await;
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner");
+        let bucket = "scanner-read-fence";
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket");
+        let incarnation = store
+            .bucket_incarnation_id_from_disk(bucket)
+            .await
+            .expect("bucket incarnation");
+        let guard = acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id)
+            .await
+            .expect("read-only scanner fence");
+        assert_eq!(guard.checked_bucket_incarnation().expect("valid read fences"), (bucket, incarnation));
+
+        let transaction_lock = store
+            .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
+            .await
+            .expect("metadata transaction namespace");
+        assert!(
+            transaction_lock.get_write_lock(Duration::from_millis(20)).await.is_err(),
+            "scanner read authority must exclude metadata writers"
+        );
+        let lifecycle_lock = store
+            .new_ns_lock(bucket, crate::storage_api_contracts::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT)
+            .await
+            .expect("bucket lifecycle namespace");
+        assert!(
+            lifecycle_lock.get_write_lock(Duration::from_millis(20)).await.is_err(),
+            "scanner read authority must exclude delete and recreation"
+        );
+        let another = timeout(
+            Duration::from_secs(1),
+            acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id),
+        )
+        .await
+        .expect("scanner readers must not serialize")
+        .expect("second read-only scanner fence");
+        let compatible = timeout(
+            Duration::from_secs(1),
+            acquire_bucket_metadata_transaction_read_lock_in(&store.ctx, bucket),
+        )
+        .await
+        .expect("metadata readers must remain compatible")
+        .expect("metadata transaction reader");
+        drop(compatible);
+        drop(another);
+
+        let mut mutation = Box::pin(acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(incarnation), false));
+        assert!(matches!(futures::poll!(&mut mutation), std::task::Poll::Pending));
+        let mut deletion = Box::pin(store.acquire_bucket_lifecycle_write_lock(bucket));
+        assert!(matches!(futures::poll!(&mut deletion), std::task::Poll::Pending));
+        drop(deletion);
+        drop(guard);
+        let mutation = timeout(Duration::from_secs(1), mutation)
+            .await
+            .expect("metadata mutation must progress after scanner read release")
+            .expect("exclusive mutation fence");
+        drop(mutation);
+        let lifecycle = timeout(Duration::from_secs(1), store.acquire_bucket_lifecycle_write_lock(bucket))
+            .await
+            .expect("bucket delete fence must progress after readers release")
+            .expect("exclusive lifecycle fence");
+        drop(lifecycle);
+
+        for (owner, expected) in [
+            (Uuid::new_v4(), incarnation),
+            (Uuid::nil(), incarnation),
+            (store.id, Uuid::nil()),
+        ] {
+            assert!(
+                acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, expected, owner)
+                    .await
+                    .is_err(),
+                "invalid owner or nil incarnation must not receive authority"
+            );
+        }
+        assert!(
+            acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, Uuid::new_v4(), store.id)
+                .await
+                .is_err(),
+            "a stale incarnation must not receive authority"
+        );
+        store
+            .delete_bucket(bucket, &DeleteBucketOptions::default())
+            .await
+            .expect("delete bucket");
+        assert!(
+            acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id)
+                .await
+                .is_err(),
+            "deleted bucket must not receive authority"
+        );
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("recreate bucket");
+        let replacement = store
+            .bucket_incarnation_id_from_disk(bucket)
+            .await
+            .expect("replacement incarnation");
+        assert_ne!(replacement, incarnation);
+        assert!(
+            acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id)
+                .await
+                .is_err(),
+            "recreation must reject the previous scanner identity"
+        );
+        let guard = acquire_scanner_bucket_incarnation_read_fence_with_sys(sys, bucket, replacement, store.id)
+            .await
+            .expect("replacement read-only scanner fence");
+        assert_eq!(
+            guard.checked_bucket_incarnation().expect("replacement remains fenced"),
+            (bucket, replacement)
+        );
+    }
+
     #[tokio::test]
     async fn old_node_metadata_rewrite_cannot_replace_bucket_incarnation_sidecar() {
         let (dirs, ecstore) = isolated_store_over_temp_disks().await;
@@ -4546,8 +4929,13 @@ mod tests {
             std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
         }
         let mut bm = BucketMetadata::new(bucket);
-        bm.update_config(config_file, raw.to_vec())
-            .expect("raw config should be stored");
+        if config_file == crate::bucket::metadata::BUCKET_TAGGING_CONFIG {
+            // Model legacy persisted corruption without passing the write-time validator.
+            bm.tagging_config_xml = raw.to_vec();
+        } else {
+            bm.update_config(config_file, raw.to_vec())
+                .expect("raw config should be stored");
+        }
         sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
     }
 
@@ -5200,6 +5588,61 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn validated_config_update_rejects_a_different_bucket_incarnation() {
+        let (_dirs, store) = isolated_store_over_temp_disks().await;
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = "catalog-cleanup-incarnation";
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket");
+        let incarnation = store
+            .bucket_incarnation_id_from_disk(bucket)
+            .await
+            .expect("read bucket incarnation");
+        let called = std::sync::atomic::AtomicBool::new(false);
+        let result = store
+            .update_bucket_metadata_config_for_incarnation_validated(
+                bucket,
+                "quota.json",
+                br#"{"quota":1}"#.to_vec(),
+                Uuid::new_v4(),
+                || {
+                    called.store(true, std::sync::atomic::Ordering::SeqCst);
+                    Ok(())
+                },
+            )
+            .await;
+        assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
+        assert!(
+            !called.load(std::sync::atomic::Ordering::SeqCst),
+            "incarnation must be checked before the external fence"
+        );
+        let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
+            .await
+            .expect("read unchanged bucket metadata");
+        assert!(
+            metadata.quota_config_json.is_empty(),
+            "a stale cleanup must not modify the replacement bucket"
+        );
+        store
+            .update_bucket_metadata_config_for_incarnation_validated(
+                bucket,
+                "quota.json",
+                br#"{"quota":1}"#.to_vec(),
+                incarnation,
+                || Ok(()),
+            )
+            .await
+            .expect("the current incarnation permits the validated update");
+        let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
+            .await
+            .expect("read updated bucket metadata");
+        assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
+    }
+
     #[tokio::test]
     #[serial]
     async fn quota_admission_rejects_stale_cached_no_quota() {
@@ -5673,7 +6116,8 @@ mod tests {
         // update only needs one interleaving to show up.
         const ROUNDS: usize = 8;
         for round in 0..ROUNDS {
-            let tagging = format!("<Tagging><Round>{round}</Round></Tagging>").into_bytes();
+            let tagging =
+                format!("<Tagging><TagSet><Tag><Key>round</Key><Value>{round}</Value></Tag></TagSet></Tagging>").into_bytes();
             let policy = format!(r#"{{"Version":"2012-10-17","Round":{round}}}"#).into_bytes();
 
             let start = Arc::new(tokio::sync::Barrier::new(2));
@@ -5746,7 +6190,7 @@ mod tests {
 
         let blocked = tokio::spawn({
             let node_b = node_b.clone();
-            async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec()).await }
+            async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging><TagSet/></Tagging>".to_vec()).await }
         });
 
         // Long enough for the write to have finished had it not waited: the

crates/ecstore/src/bucket/quota/checker/tests/reporting_retirement.rs

@@ -119,6 +119,21 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
             // Exercise the real storage entry point before testing the new
             // reporting semantics: the same ledger must admit and refuse
             // these writes even when a scanner floor claims a million bytes.
+            if protocol == 1 {
+                let mut bootstrap = PutObjReader::from_vec(vec![0x5a; 512]);
+                assert!(
+                    matches!(
+                        store
+                            .put_object(&bucket, "bootstrap", &mut bootstrap, &ObjectOptions::default())
+                            .await,
+                        Err(StorageError::SlowDown)
+                    ),
+                    "a missing authoritative ledger must not admit a write from scanner reporting"
+                );
+                crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
+                    .await
+                    .expect("bootstrap authoritative usage through the production worker");
+            }
             let mut too_large = PutObjReader::from_vec(vec![0x5a; 8192]);
             assert!(
                 matches!(

crates/ecstore/src/bucket/remote_s3_client.rs

@@ -40,7 +40,9 @@ use aws_smithy_runtime_api::client::http::{
     HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
 };
 use aws_smithy_runtime_api::client::interceptors::Intercept;
-use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
+use aws_smithy_runtime_api::client::interceptors::context::{
+    BeforeDeserializationInterceptorContextMut, BeforeTransmitInterceptorContextMut,
+};
 use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
 use aws_smithy_runtime_api::client::result::ConnectorError;
 use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
@@ -155,6 +157,8 @@ pub struct RemoteS3EndpointSpec {
     pub ca_cert_pem: Option<String>,
     pub connect_timeout: Option<Duration>,
     pub read_timeout: Option<Duration>,
+    /// Entire SDK call, including non-streaming response bodies and retries.
+    pub operation_timeout: Option<Duration>,
     /// How many wire requests one logical call may cost. Every consumer
     /// declares it; see [`RemoteS3RetryPolicy`].
     pub retry: RemoteS3RetryPolicy,
@@ -252,6 +256,78 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
     Ok(builder.build())
 }
 
+/// Only installed on non-streaming upload operations. The budget begins at
+/// response headers, independently of the potentially long request upload.
+#[derive(Debug)]
+pub(crate) struct UploadResponseDeadline(pub Duration);
+
+impl Intercept for UploadResponseDeadline {
+    fn name(&self) -> &'static str {
+        "RustfsUploadResponseDeadline"
+    }
+
+    fn modify_before_deserialization(
+        &self,
+        context: &mut BeforeDeserializationInterceptorContextMut<'_>,
+        _runtime_components: &RuntimeComponents,
+        _cfg: &mut ConfigBag,
+    ) -> Result<(), BoxError> {
+        let response = context.response_mut();
+        let inner = std::mem::replace(response.body_mut(), SdkBody::taken());
+        *response.body_mut() = SdkBody::from_body_1_x(UploadResponseBody {
+            inner: Some(inner),
+            deadline: Box::pin(tokio::time::sleep(self.0)),
+        });
+        Ok(())
+    }
+}
+
+struct UploadResponseBody {
+    inner: Option<SdkBody>,
+    deadline: std::pin::Pin<Box<tokio::time::Sleep>>,
+}
+
+impl http_body::Body for UploadResponseBody {
+    type Data = bytes::Bytes;
+    type Error = BoxError;
+
+    fn poll_frame(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
+        use std::future::Future as _;
+        use std::task::Poll;
+        if self.inner.is_none() {
+            return Poll::Ready(None);
+        }
+        // The absolute response budget never renews on data, empty frames,
+        // or trailers. A retry gets its own bounded response window.
+        if self.deadline.as_mut().poll(cx).is_ready() {
+            self.inner.take();
+            return Poll::Ready(Some(Err(std::io::Error::new(
+                std::io::ErrorKind::TimedOut,
+                "replication response body deadline exceeded",
+            )
+            .into())));
+        }
+        let Some(inner) = self.inner.as_mut() else {
+            return Poll::Ready(None);
+        };
+        let result = std::pin::Pin::new(inner).poll_frame(cx);
+        if matches!(result, Poll::Ready(None) | Poll::Ready(Some(Err(_)))) {
+            self.inner.take();
+        }
+        result
+    }
+
+    fn is_end_stream(&self) -> bool {
+        self.inner.as_ref().is_none_or(http_body::Body::is_end_stream)
+    }
+    fn size_hint(&self) -> http_body::SizeHint {
+        self.inner.as_ref().map(http_body::Body::size_hint).unwrap_or_default()
+    }
+}
+
 /// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
 /// signing: SigV4 excludes `user-agent` from the canonical request, so the
 /// signature stays valid.
@@ -309,7 +385,7 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
         config_builder = config_builder.http_client(http_client);
     }
 
-    if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
+    if spec.connect_timeout.is_some() || spec.read_timeout.is_some() || spec.operation_timeout.is_some() {
         let mut timeouts = TimeoutConfig::builder();
         if let Some(connect_timeout) = spec.connect_timeout {
             timeouts = timeouts.connect_timeout(connect_timeout);
@@ -317,6 +393,9 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
         if let Some(read_timeout) = spec.read_timeout {
             timeouts = timeouts.read_timeout(read_timeout);
         }
+        if let Some(operation_timeout) = spec.operation_timeout {
+            timeouts = timeouts.operation_timeout(operation_timeout);
+        }
         config_builder = config_builder.timeout_config(timeouts.build());
     }
 
@@ -674,6 +753,7 @@ mod tests {
             ca_cert_pem: None,
             connect_timeout: None,
             read_timeout: None,
+            operation_timeout: None,
             retry: RemoteS3RetryPolicy::Disabled,
             user_agent_suffix: "",
         }
@@ -1027,11 +1107,16 @@ mod tests {
         spec.user_agent_suffix = "RustFS-Test/0.0";
         spec.connect_timeout = Some(Duration::from_secs(5));
         spec.read_timeout = Some(Duration::from_secs(5));
+        spec.operation_timeout = Some(Duration::from_secs(12));
         let config = build_remote_s3_config(&spec)
             .await
             .expect("spec should build")
             .http_client(http_client)
             .build();
+        assert_eq!(
+            config.timeout_config().and_then(TimeoutConfig::operation_timeout),
+            Some(Duration::from_secs(12))
+        );
         S3Client::from_conf(config)
             .head_bucket()
             .bucket("bucket")

crates/ecstore/src/bucket/replication/replication_metadata_boundary.rs

@@ -33,6 +33,7 @@ pub(crate) struct ReplicationMetadataStore;
 impl ReplicationMetadataStore {
     pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
     pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
+    pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
     pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
     pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
 
@@ -116,6 +117,10 @@ mod tests {
             ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
             "config/replication/mrf.bin.recovery"
         );
+        assert_eq!(
+            ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
+            "config/replication/mrf.bin.replay"
+        );
         assert_eq!(
             ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
             "config/replication/force-delete.bin"

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -16,8 +16,8 @@ use super::replication_config_store::ReplicationConfigStore;
 use super::replication_error_boundary::{Error as EcstoreError, is_err_object_not_found, is_err_version_not_found};
 use super::replication_filemeta_boundary::{
     MrfOpKind, MrfReplicateEntry, REPLICATE_HEAL_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedTargetInfo,
-    ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, replicate_decision_for_admitted_targets,
-    replication_statuses_map, version_purge_statuses_map,
+    ReplicationGenerationSnapshot, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
+    replicate_decision_for_admitted_targets, replication_statuses_map, version_purge_statuses_map,
 };
 use super::replication_lock_boundary::ReplicationLockTiming;
 use super::replication_logging::{EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION};
@@ -86,6 +86,10 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
 const MRF_PENDING_CAP: usize = 200_000;
 const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
 const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
+/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
+/// re-attempts every retained entry, so a target that stays down costs one
+/// attempt per entry per interval.
+const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
 
 #[derive(Debug, Default)]
 pub struct DurableMrfBacklog {
@@ -615,7 +619,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
 }
 
 /// Acknowledge only the generation read by the recovery leader. Entries appended
-/// after that generation are retained as a suffix and replayed on the next startup.
+/// after that generation are retained as a suffix and replayed on the next pass.
 /// Lock order: recovery leader lock -> MRF journal object lock.
 async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     storage: Arc<S>,
@@ -667,7 +671,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     Err(EcstoreError::PreconditionFailed)
 }
 
-/// Acquires the MRF recovery leader lock for the startup replay.
+/// Acquires the MRF recovery leader lock for one journal replay pass.
 /// Returns `None` (after logging) when the lock cannot be created or another
 /// node is already processing the backlog.
 async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
@@ -1018,7 +1022,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
 }
 
 /// Acknowledges the replayed MRF prefix and returns the retained backlog.
-/// On acknowledgement failure the backlog is preserved for the next startup and
+/// On acknowledgement failure the backlog is preserved for the next replay pass and
 /// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
 async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     storage: &Arc<S>,
@@ -1033,7 +1037,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
                 component = LOG_COMPONENT_ECSTORE,
                 subsystem = LOG_SUBSYSTEM_REPLICATION,
                 error = %error,
-                "Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
+                "Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
             );
             match read_mrf_entries(storage.clone()).await {
                 Ok(current) => current,
@@ -1051,6 +1055,120 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     }
 }
 
+fn unix_millis_now() -> i64 {
+    i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
+}
+
+/// Whether a periodic pass may run, given the start time of the last pass on
+/// any node. A stamp more than one interval ahead of this node's clock is skew
+/// or a bad value, so it does not suppress replay.
+fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
+    let Some(last_started_ms) = last_started_ms else {
+        return true;
+    };
+    let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
+    now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
+}
+
+async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
+    let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
+        .await
+        .ok()?;
+    std::str::from_utf8(&data).ok()?.trim().parse().ok()
+}
+
+async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
+    if let Err(error) = ReplicationConfigStore::save(
+        storage.clone(),
+        ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
+        started_ms.to_string().into_bytes(),
+    )
+    .await
+    {
+        warn!(
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION,
+            error = %error,
+            "Failed to record the MRF replay start time"
+        );
+    }
+}
+
+/// One MRF journal replay pass. The recovery leader reads the journal,
+/// re-drives every entry, and acknowledges the replayed prefix; entries that
+/// could not be admitted, and entries appended meanwhile, stay for the next pass.
+///
+/// Every node runs the periodic loop, and the leader lock only serializes
+/// passes. With `min_interval`, a pass is skipped when a pass on any node
+/// started within that interval, read from the shared replay stamp, so a
+/// cluster replays once per interval rather than once per node. The startup
+/// pass passes `None` and always runs. A pass that finds entries records its
+/// start time. Returns whether the journal was read.
+/// Lock order: recovery leader lock -> replay stamp object lock.
+async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
+    let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
+        return false;
+    };
+    let started_ms = unix_millis_now();
+    if let Some(interval) = min_interval
+        && !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
+    {
+        return false;
+    }
+
+    let Some(entries) = load_mrf_recovery_entries(storage).await else {
+        return true;
+    };
+    if !entries.is_empty() {
+        write_mrf_replay_stamp(storage, started_ms).await;
+    }
+    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
+
+    let total = entries.len();
+    let mut queued_count = 0usize;
+    let mut retry_entries = Vec::new();
+
+    for entry in entries.iter() {
+        let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
+            continue;
+        };
+
+        if admission == ReplicationQueueAdmission::Missed {
+            retry_entries.push(entry.clone());
+        } else if admission == ReplicationQueueAdmission::Queued {
+            queued_count += 1;
+        }
+    }
+
+    let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
+    let retained_count = retained.len();
+    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
+
+    if queued_count > 0 {
+        info!(
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION,
+            recovered = queued_count,
+            total,
+            retained = retained_count,
+            "Replayed MRF entries admitted for retry"
+        );
+    }
+    true
+}
+
+/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
+/// one pass at a time across the cluster, including the startup pass, and the
+/// shared replay stamp spaces passes from different nodes by `period`.
+fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
+    tokio::spawn(async move {
+        loop {
+            tokio::time::sleep(period).await;
+            replay_mrf_journal(&storage, Some(period)).await;
+        }
+    })
+}
+
 #[derive(Debug, thiserror::Error)]
 #[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
 struct ResyncActiveConflictError {
@@ -1171,6 +1289,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
         // Start background tasks
         pool.start_mrf_persister().await;
         pool.start_mrf_processor().await;
+        pool.start_mrf_replayer().await;
         pool.start_force_delete_processor().await;
 
         pool
@@ -1462,7 +1581,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Queues a replica task
     pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
-        // A version that is already queued or being uploaded is not driven a
+        // A source generation already queued or being uploaded is not driven a
         // second time: the scanner heal pass sees it as PENDING until the
         // first upload lands and would otherwise re-queue it every cycle
         // (backlog#2362). The key is released when the worker finishes, or
@@ -1639,55 +1758,26 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Starts the MRF processor — one-shot at startup.
     ///
-    /// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
-    /// rewrites any entries that could not be admitted for a later startup retry.
+    /// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
+    /// the pass while the server runs.
     async fn start_mrf_processor(&self) {
         let storage = self.storage.clone();
-
         let handle = tokio::spawn(async move {
-            let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
-                return;
-            };
-
-            let Some(entries) = load_mrf_recovery_entries(&storage).await else {
-                return;
-            };
-            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
-
-            let total = entries.len();
-            let mut queued_count = 0usize;
-            let mut retry_entries = Vec::new();
-
-            for entry in entries.iter() {
-                let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
-                    continue;
-                };
-
-                if admission == ReplicationQueueAdmission::Missed {
-                    retry_entries.push(entry.clone());
-                } else if admission == ReplicationQueueAdmission::Queued {
-                    queued_count += 1;
-                }
-            }
-
-            let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
-            let retained_count = retained.len();
-            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
-
-            if queued_count > 0 {
-                info!(
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_REPLICATION,
-                    recovered = queued_count,
-                    total,
-                    retained = retained_count,
-                    "Replayed MRF entries admitted for retry"
-                );
-            }
+            replay_mrf_journal(&storage, None).await;
         });
         self.task_handles.lock().await.push(handle);
     }
 
+    /// Starts the runtime MRF replayer — ongoing background task.
+    ///
+    /// Entries reach the journal while the server runs (worker-queue overflow,
+    /// delete-marker purge intents whose watch window drained with targets still
+    /// dirty). Replaying only at startup left them waiting for a restart.
+    async fn start_mrf_replayer(&self) {
+        let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
+        self.task_handles.lock().await.push(handle);
+    }
+
     async fn start_force_delete_processor(&self) {
         let storage = self.storage.clone();
         let handle =
@@ -1755,7 +1845,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
     /// writes them to the on-disk MRF file every flush interval (default 10s,
     /// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
     /// entries accumulate. Runtime writers append under the journal lock; only the
-    /// startup recovery leader may remove an acknowledged prefix.
+    /// recovery leader of a replay pass may remove an acknowledged prefix.
     async fn start_mrf_persister(&self) {
         let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
             return;
@@ -2314,28 +2404,47 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 }
 
 /// Object versions currently queued or being uploaded, keyed by bucket,
-/// object name and version. `queue_replica_task` admits a version only once
+/// object name, version and source generation. `queue_replica_task` admits a generation only once
 /// while it is in flight; the scanner heal pass and MRF replays that arrive
 /// in the meantime are `Skipped` instead of driving a second complete upload
 /// (backlog#2362). Entries are removed when the worker finishes the task or
 /// when no worker accepted it.
 #[derive(Debug, Default)]
 pub(crate) struct ReplicationInFlight {
-    keys: std::sync::Mutex<std::collections::HashSet<(String, String, Option<uuid::Uuid>)>>,
+    keys: std::sync::Mutex<std::collections::HashSet<ReplicationInFlightKey>>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct ReplicationInFlightKey {
+    bucket: String,
+    object: String,
+    version_id: Option<uuid::Uuid>,
+    generation: ReplicationGenerationSnapshot,
+}
+
+impl From<&ReplicateObjectInfo> for ReplicationInFlightKey {
+    fn from(info: &ReplicateObjectInfo) -> Self {
+        Self {
+            bucket: info.bucket.clone(),
+            object: info.name.clone(),
+            version_id: info.version_id,
+            generation: info.replication_generation.clone(),
+        }
+    }
 }
 
 impl ReplicationInFlight {
-    fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<(String, String, Option<uuid::Uuid>)>> {
+    fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<ReplicationInFlightKey>> {
         self.keys.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
     }
 
-    /// Claim `ri`; `false` when the same version is already in flight.
+    /// Claim `ri`; a metadata mutation is independent of its earlier generation.
     fn try_begin(&self, ri: &ReplicateObjectInfo) -> bool {
-        self.lock().insert((ri.bucket.clone(), ri.name.clone(), ri.version_id))
+        self.lock().insert(ReplicationInFlightKey::from(ri))
     }
 
     fn finish(&self, ri: &ReplicateObjectInfo) {
-        self.lock().remove(&(ri.bucket.clone(), ri.name.clone(), ri.version_id));
+        self.lock().remove(&ReplicationInFlightKey::from(ri));
     }
 
     #[cfg(test)]
@@ -2348,26 +2457,21 @@ impl ReplicationInFlight {
 /// including when replication panics.
 struct ReplicationInFlightGuard {
     in_flight: Arc<ReplicationInFlight>,
-    key: ReplicateObjectInfo,
+    key: ReplicationInFlightKey,
 }
 
 impl ReplicationInFlightGuard {
     fn new(in_flight: Arc<ReplicationInFlight>, ri: &ReplicateObjectInfo) -> Self {
         Self {
             in_flight,
-            key: ReplicateObjectInfo {
-                bucket: ri.bucket.clone(),
-                name: ri.name.clone(),
-                version_id: ri.version_id,
-                ..Default::default()
-            },
+            key: ReplicationInFlightKey::from(ri),
         }
     }
 }
 
 impl Drop for ReplicationInFlightGuard {
     fn drop(&mut self) {
-        self.in_flight.finish(&self.key);
+        self.in_flight.lock().remove(&self.key);
     }
 }
 
@@ -3421,6 +3525,8 @@ mod tests {
         allow_first_read: Notify,
         read_count: AtomicUsize,
         delete_storage_calls: AtomicUsize,
+        delete_error: StdMutex<EcstoreError>,
+        replay_stamp: StdMutex<Option<Vec<u8>>>,
         reported_size: StdMutex<Option<i64>>,
         stream_read_bytes: Arc<AtomicUsize>,
         write_count: AtomicUsize,
@@ -3488,6 +3594,24 @@ mod tests {
             _h: Self::HeaderMap,
             _opts: &Self::ObjectOptions,
         ) -> Result<Self::GetObjectReader, Self::Error> {
+            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
+                let stamp = self
+                    .shared
+                    .replay_stamp
+                    .lock()
+                    .expect("test replay stamp lock should not be poisoned")
+                    .clone()
+                    .ok_or(EcstoreError::FileNotFound)?;
+                return Ok(Self::GetObjectReader {
+                    object_info: ObjectInfo {
+                        size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
+                        ..Default::default()
+                    },
+                    stream: Box::new(Cursor::new(stamp)),
+                    buffered_body: None,
+                    body_source: Default::default(),
+                });
+            }
             if !object.ends_with("/.replication/resync.bin")
                 && !object.ends_with("config/replication/mrf.bin")
                 && !object.ends_with("config/replication/force-delete.bin")
@@ -3551,6 +3675,16 @@ mod tests {
                 let lock = self.new_ns_lock(bucket, object).await?;
                 Some(lock.get_write_lock(Duration::from_secs(10)).await?)
             };
+            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
+                let mut stamp = Vec::new();
+                data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
+                *self
+                    .shared
+                    .replay_stamp
+                    .lock()
+                    .expect("test replay stamp lock should not be poisoned") = Some(stamp);
+                return Ok(ObjectInfo::default());
+            }
             if opts.http_preconditions.is_some()
                 && let Some(replacement) = self
                     .shared
@@ -3672,7 +3806,12 @@ mod tests {
             _opts: Self::ObjectOptions,
         ) -> Result<Self::ObjectInfo, Self::Error> {
             self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
-            Err(EcstoreError::NotImplemented)
+            Err(self
+                .shared
+                .delete_error
+                .lock()
+                .expect("delete error lock should not be poisoned")
+                .clone())
         }
 
         async fn delete_objects(
@@ -3929,6 +4068,48 @@ mod tests {
         assert_eq!(current_queue(&pool, "in-flight-bucket").await, (3, 3 * 4096));
     }
 
+    #[tokio::test]
+    async fn queue_replica_task_admits_metadata_changes_during_an_initial_upload() {
+        let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
+        let (tx, mut rx) = mpsc::channel(4);
+        pool.workers.write().await.push(tx);
+        let initial = ReplicateObjectInfo {
+            bucket: "pending-tags-bucket".to_string(),
+            name: "object".to_string(),
+            version_id: Some(uuid::Uuid::new_v4()),
+            size: 4096,
+            replication_generation: ReplicationGenerationSnapshot {
+                mutation_id: Some(uuid::Uuid::new_v4().to_string()),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        let metadata = ReplicateObjectInfo {
+            op_type: ReplicationType::Metadata,
+            user_tags: String::new(),
+            replication_generation: ReplicationGenerationSnapshot {
+                mutation_id: Some(uuid::Uuid::new_v4().to_string()),
+                ..Default::default()
+            },
+            ..initial.clone()
+        };
+        assert_eq!(pool.queue_replica_task(initial.clone()).await, ReplicationQueueAdmission::Queued);
+        assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Queued);
+        assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
+        assert_eq!(pool.in_flight.len(), 2);
+        drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &initial));
+        assert_eq!(pool.in_flight.len(), 1);
+        assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
+        for expected in [&initial, &metadata] {
+            let ReplicationOperation::Object(queued) = rx.recv().await.expect("queued operation") else {
+                panic!("expected object replication");
+            };
+            assert_eq!(queued.replication_generation, expected.replication_generation);
+        }
+        drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &metadata));
+        assert_eq!(pool.in_flight.len(), 0);
+    }
+
     #[tokio::test]
     async fn queue_replica_task_releases_the_version_when_no_worker_accepts_it() {
         let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
@@ -4171,6 +4352,8 @@ mod tests {
             allow_first_read: Notify::new(),
             read_count: AtomicUsize::new(0),
             delete_storage_calls: AtomicUsize::new(0),
+            delete_error: StdMutex::new(EcstoreError::NotImplemented),
+            replay_stamp: StdMutex::new(None),
             reported_size: StdMutex::new(None),
             stream_read_bytes: Arc::new(AtomicUsize::new(0)),
             write_count: AtomicUsize::new(0),
@@ -4221,6 +4404,103 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn delete_worker_already_purged_source_completes_without_retry() {
+        let arn = "arn:test";
+        for delete_marker_version in [false, true] {
+            for version_id in [Uuid::new_v4(), Uuid::nil()] {
+                for error in [
+                    EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
+                    EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+                    EcstoreError::FileVersionNotFound,
+                    EcstoreError::FileNotFound,
+                ] {
+                    let shared = empty_resync_shared_state();
+                    *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
+                    let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
+                    let delete = DeletedObjectReplicationInfo {
+                        bucket: "source".to_string(),
+                        delete_object: ReplicationDeletedObject {
+                            object_name: "purged".to_string(),
+                            version_id: (!delete_marker_version).then_some(version_id),
+                            delete_marker_version_id: delete_marker_version.then_some(version_id),
+                            replication_state: Some(ReplicationState {
+                                replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
+                                purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
+                                ..Default::default()
+                            }),
+                            ..Default::default()
+                        },
+                        op_type: ReplicationType::Delete,
+                        ..Default::default()
+                    };
+
+                    assert!(
+                        replicate_delete_with_outcome(delete, storage).await,
+                        "a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
+                    );
+                    assert_eq!(
+                        shared.delete_storage_calls.load(Ordering::SeqCst),
+                        2 + usize::from(!delete_marker_version && !version_id.is_nil()),
+                        "the worker must lock and attempt writeback, reading a non-null data version first"
+                    );
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
+        let arn = "arn:test";
+        for (version_purge, target_status, error) in [
+            (
+                true,
+                VersionPurgeStatusType::Pending,
+                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+            ),
+            (true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
+            (
+                false,
+                VersionPurgeStatusType::Empty,
+                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+            ),
+        ] {
+            let shared = empty_resync_shared_state();
+            *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
+            let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
+            let delete = DeletedObjectReplicationInfo {
+                bucket: "source".to_string(),
+                delete_object: ReplicationDeletedObject {
+                    object_name: "purged".to_string(),
+                    version_id: version_purge.then(Uuid::new_v4),
+                    delete_marker: !version_purge,
+                    replication_state: Some(ReplicationState {
+                        replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
+                        targets: if version_purge {
+                            HashMap::new()
+                        } else {
+                            HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
+                        },
+                        purge_targets: if version_purge {
+                            HashMap::from([(arn.to_string(), target_status)])
+                        } else {
+                            HashMap::new()
+                        },
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                op_type: ReplicationType::Delete,
+                ..Default::default()
+            };
+
+            assert!(
+                !replicate_delete_with_outcome(delete, storage).await,
+                "unfinished target work, source I/O failures, and marker creation must remain retryable"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn delete_worker_preserves_persisted_admission_without_live_config() {
         let arn = "arn:test";
@@ -5304,6 +5584,8 @@ mod tests {
                 allow_first_read: Notify::new(),
                 read_count: AtomicUsize::new(0),
                 delete_storage_calls: AtomicUsize::new(0),
+                delete_error: StdMutex::new(EcstoreError::NotImplemented),
+                replay_stamp: StdMutex::new(None),
                 reported_size: StdMutex::new(None),
                 stream_read_bytes: Arc::new(AtomicUsize::new(0)),
                 write_count: AtomicUsize::new(0),
@@ -6268,6 +6550,108 @@ mod tests {
         assert_eq!(retained.target_arns, entry.target_arns);
     }
 
+    #[tokio::test]
+    async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
+        let shared = empty_resync_shared_state();
+        let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
+        // A force-delete intent from a superseded generation replays as
+        // Skipped, so a pass that reaches it acknowledges it off the journal.
+        let stale_intent = |object: &str| MrfReplicateEntry {
+            bucket: "mrf-runtime-replay".to_string(),
+            object: object.to_string(),
+            op: MrfOpKind::Delete,
+            force_delete_id: Some(uuid::Uuid::new_v4()),
+            force_delete_local_commit: false,
+            target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
+            ..Default::default()
+        };
+        let journal_drained = || async {
+            tokio::time::timeout(Duration::from_secs(5), async {
+                while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
+                    tokio::time::sleep(Duration::from_millis(10)).await;
+                }
+            })
+            .await
+            .is_ok()
+        };
+
+        let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
+        for object in ["first", "second"] {
+            *shared.data.lock().expect("test data lock should not be poisoned") =
+                encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
+            assert!(
+                journal_drained().await,
+                "an MRF entry persisted while the server runs must be replayed without a restart ({object})"
+            );
+        }
+        handle.abort();
+    }
+
+    #[tokio::test]
+    async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
+        let shared = empty_resync_shared_state();
+        let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
+        let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
+        let interval = Duration::from_secs(1);
+        let persist_backlog = |object: &str| {
+            let entry = MrfReplicateEntry {
+                bucket: "mrf-node-replay".to_string(),
+                object: object.to_string(),
+                op: MrfOpKind::Delete,
+                force_delete_id: Some(uuid::Uuid::new_v4()),
+                force_delete_local_commit: false,
+                target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
+                ..Default::default()
+            };
+            *shared.data.lock().expect("test data lock should not be poisoned") =
+                encode_mrf_file(&[entry]).expect("MRF entry should encode");
+        };
+        let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
+
+        persist_backlog("first");
+        assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
+        assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
+
+        // Node B's timer fires within the same interval with a backlog present.
+        persist_backlog("second");
+        assert!(
+            !replay_mrf_journal(&node_b, Some(interval)).await,
+            "a node must not replay within one interval of another node's pass"
+        );
+        assert!(!journal_is_empty(), "the backlog should wait for the next interval");
+
+        assert!(
+            replay_mrf_journal(&node_b, None).await,
+            "the startup pass should replay regardless of the shared stamp"
+        );
+        assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
+
+        persist_backlog("third");
+        tokio::time::sleep(interval).await;
+        assert!(
+            replay_mrf_journal(&node_a, Some(interval)).await,
+            "a periodic pass should replay once the interval has elapsed"
+        );
+        assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
+    }
+
+    #[test]
+    fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
+        let interval = Duration::from_secs(300);
+        let now = 1_000_000_000;
+        assert!(mrf_replay_due(None, now, interval));
+        assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
+        assert!(mrf_replay_due(Some(now - 300_000), now, interval));
+        assert!(
+            !mrf_replay_due(Some(now + 1_000), now, interval),
+            "small clock skew should still space passes"
+        );
+        assert!(
+            mrf_replay_due(Some(now + 300_000), now, interval),
+            "a stamp an interval ahead must not suppress replay"
+        );
+    }
+
     #[test]
     fn mrf_entry_delete_marker_roundtrip() {
         let dm_vid = Uuid::new_v4();

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -2809,17 +2809,22 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
             true
         }
         Err(e) => {
-            error!(
-                event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
-                bucket = %bucket,
-                arn = %dobj.target_arn,
-                object = %dobj.delete_object.object_name,
-                operation = "apply_replication_delete_state",
-                error = %e,
-                "Replication target operation failed"
-            );
+            // A batch purge or another worker may have removed this version
+            // already. Remote failures still control the final outcome below.
+            let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
+            if !source_already_purged {
+                error!(
+                    event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+                    bucket = %bucket,
+                    arn = %dobj.target_arn,
+                    object = %dobj.delete_object.object_name,
+                    operation = "apply_replication_delete_state",
+                    error = %e,
+                    "Replication target operation failed"
+                );
+            }
             send_event(EventArgs {
                 event_name,
                 bucket_name: bucket.clone(),
@@ -2832,7 +2837,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
                 },
                 ..Default::default()
             });
-            false
+            source_already_purged
         }
     };
 
@@ -3010,7 +3015,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
 /// If the marker disappears (deleted before or while the replica landed),
 /// purge the replicated marker from the targets, retrying failed targets on
 /// later rounds. When the window drains with targets still dirty, persist the
-/// purge intent as a durable MRF entry so the next startup replays it through
+/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
 /// `purge_stale_delete_marker_targets`.
 async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
     bucket: String,
@@ -3101,7 +3106,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
 /// Heal callers retry for the full watch window because the startup MRF
 /// processor runs before bucket metadata (and thus target clients) finishes
 /// initializing — the first pass can see `target_client_missing` and a later
-/// round resolves the client; the replay loop is serial and startup-only, so
+/// round resolves the client; replay passes are serial background work, so
 /// blocking it for up to the window per dirty entry is acceptable. Live
 /// callers run on replication workers where a down target would pin a worker
 /// for the whole window, so they attempt once and lean on the durable intent
@@ -6604,6 +6609,7 @@ mod tests {
             health_check_duration: std::time::Duration::from_secs(5),
             replicate_sync: false,
             client: Arc::new(aws_sdk_s3::Client::from_conf(config)),
+            timeouts: Default::default(),
         })
     }
 

crates/ecstore/src/bucket/utils.rs

@@ -259,6 +259,10 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
 pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) -> Result<()> {
     check_put_object_args(bucket, object)?;
 
+    if !data_movement && !is_meta_bucketname(bucket) && has_xml_invalid_control_characters(object) {
+        return Err(StorageError::ObjectNameInvalid(bucket.to_owned(), object.to_owned()));
+    }
+
     if cfg!(target_os = "windows") && !data_movement && !is_meta_bucketname(bucket) && object.contains('\\') {
         return Err(StorageError::InvalidArgument(
             bucket.to_owned(),
@@ -270,6 +274,14 @@ pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) ->
     Ok(())
 }
 
+/// New public keys must remain representable in an unencoded XML listing.
+/// Existing keys still use the read/delete validators so they can be removed.
+pub fn has_xml_invalid_control_characters(object: &str) -> bool {
+    object
+        .chars()
+        .any(|character| character < ' ' && !matches!(character, '\t' | '\n' | '\r'))
+}
+
 pub fn check_copy_obj_args(bucket: &str, object: &str) -> Result<()> {
     check_bucket_and_object_names(bucket, object)
 }

crates/ecstore/src/cache_value/metacache_set.rs

@@ -34,7 +34,7 @@ use tokio::sync::Mutex as TokioMutex;
 use tokio::task::JoinSet;
 use tokio::time::timeout;
 use tokio_util::sync::CancellationToken;
-use tracing::{debug, error, warn};
+use tracing::{Instrument, debug, error, warn};
 
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
@@ -217,6 +217,8 @@ impl Drop for NotifyOnDrop {
 
 #[derive(Default)]
 pub struct ListPathRawOptions {
+    /// Retain native listing admission through producer cancellation/draining.
+    pub read_budget: Option<Arc<tokio::sync::OwnedSemaphorePermit>>,
     pub disks: Vec<Option<DiskStore>>,
     pub fallback_disks: Vec<Option<DiskStore>>,
     pub bucket: String,
@@ -224,6 +226,8 @@ pub struct ListPathRawOptions {
     pub recursive: bool,
     pub incl_deleted: bool,
     pub skip_hidden_prefix_check: bool,
+    pub hidden_entry_budget: usize,
+    pub pending_prefix: Option<String>,
     pub filter_prefix: Option<String>,
     pub forward_to: Option<String>,
     pub min_disks: usize,
@@ -253,6 +257,7 @@ pub struct ListPathRawOptions {
 impl Clone for ListPathRawOptions {
     fn clone(&self) -> Self {
         Self {
+            read_budget: self.read_budget.clone(),
             disks: self.disks.clone(),
             fallback_disks: self.fallback_disks.clone(),
             bucket: self.bucket.clone(),
@@ -260,6 +265,8 @@ impl Clone for ListPathRawOptions {
             recursive: self.recursive,
             incl_deleted: self.incl_deleted,
             skip_hidden_prefix_check: self.skip_hidden_prefix_check,
+            hidden_entry_budget: self.hidden_entry_budget,
+            pending_prefix: self.pending_prefix.clone(),
             filter_prefix: self.filter_prefix.clone(),
             forward_to: self.forward_to.clone(),
             min_disks: self.min_disks,
@@ -289,6 +296,8 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
         recursive: opts.recursive,
         incl_deleted: opts.incl_deleted,
         skip_hidden_prefix_check: opts.skip_hidden_prefix_check,
+        hidden_entry_budget: opts.hidden_entry_budget,
+        pending_prefix: opts.pending_prefix.clone(),
         report_notfound: opts.report_not_found,
         filter_prefix: opts.filter_prefix.clone(),
         forward_to: opts.forward_to.clone(),
@@ -301,13 +310,15 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
 }
 
 /// How a merged raw listing stream ended.
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
 pub struct ListPathRawOutcome {
     /// At least one producer stopped at its per-disk limit instead of at the
     /// end of its namespace, so the closed stream is not proof of exhaustion.
     /// Carried in-band by the metacache stream trailer, so remote producers
     /// report it as well.
     pub limit_reached: bool,
+    /// Inclusive sorted boundary proved by every reader before a bounded stop.
+    pub scan_cut: Option<String>,
 }
 
 pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
@@ -363,10 +374,15 @@ async fn list_path_raw_inner(
     let producer_errs: Arc<[OnceLock<DiskError>]> = (0..opts.disks.len()).map(|_| OnceLock::new()).collect::<Vec<_>>().into();
 
     let cancel_rx = CancellationToken::new();
+    let walk_options = Arc::new(walk_dir_options(&opts));
 
     for (disk_idx, disk) in opts.disks.iter().enumerate() {
         let opdisk = disk.clone();
-        let opts_clone = opts.clone();
+        let walk_options = walk_options.clone();
+        #[cfg(test)]
+        let test_reader_behavior = opts.test_reader_behaviors.get(disk_idx).cloned();
+        #[cfg(test)]
+        let test_disk_io_context_observed = opts.test_disk_io_context_observed.clone();
         let fallback_claim_tracker = fallback_claim_tracker.clone();
         let fds_clone = fds.clone();
         #[cfg(test)]
@@ -376,17 +392,19 @@ async fn list_path_raw_inner(
         let disk_io_context_clone = disk_io_context.clone();
         let (rd, wr) = tokio::io::duplex(64 * 1024);
         readers.push(MetacacheReader::new(rd));
-        jobs.spawn(with_optional_disk_io_context(disk_io_context_clone, async move {
+        let read_budget = opts.read_budget.clone();
+        let producer = with_optional_disk_io_context(disk_io_context_clone, async move {
+            let _read_budget = read_budget;
             let mut wr = PublishedBytesWriter::new(wr);
             #[cfg(test)]
-            if let Some(observer) = &opts_clone.test_disk_io_context_observed {
+            if let Some(observer) = &test_disk_io_context_observed {
                 observer.store(
                     crate::disk::current_disk_io_context().is_some_and(|context| context.class == DiskIoClass::Background),
                     std::sync::atomic::Ordering::Release,
                 );
             }
             #[cfg(test)]
-            let test_primary_error = if let Some(behavior) = opts_clone.test_reader_behaviors.get(disk_idx).cloned() {
+            let test_primary_error = if let Some(behavior) = test_reader_behavior {
                 match behavior {
                     TestReaderBehavior::Eof => return Ok(()),
                     TestReaderBehavior::Entries(entries) => {
@@ -443,8 +461,6 @@ async fn list_path_raw_inner(
                 None
             };
 
-            let wakl_opts = walk_dir_options(&opts_clone);
-
             let mut need_fallback = false;
             let mut last_err = None;
             #[cfg(test)]
@@ -459,7 +475,7 @@ async fn list_path_raw_inner(
 
             if !need_fallback && let Some(disk) = opdisk {
                 let primary_walk_started = std::time::Instant::now();
-                match disk.walk_dir(wakl_opts, &mut wr).await {
+                match disk.walk_dir((*walk_options).clone(), &mut wr).await {
                     Ok(_res) => {
                         rustfs_io_metrics::record_stage_duration(
                             "metacache_walk_dir_primary",
@@ -483,8 +499,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %opts_clone.bucket,
-                                path = %opts_clone.path,
+                                bucket = %walk_options.bucket,
+                                path = %walk_options.base_dir,
                                 disk_index = disk_idx,
                                 state = "walk_dir_missing_path",
                                 error = ?err,
@@ -495,8 +511,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %opts_clone.bucket,
-                                path = %opts_clone.path,
+                                bucket = %walk_options.bucket,
+                                path = %walk_options.base_dir,
                                 disk_index = disk_idx,
                                 state = "walk_dir_failed",
                                 error = ?err,
@@ -598,8 +614,8 @@ async fn list_path_raw_inner(
                             event = EVENT_METACACHE_LISTING,
                             component = LOG_COMPONENT_ECSTORE,
                             subsystem = LOG_SUBSYSTEM_METACACHE,
-                            bucket = %opts_clone.bucket,
-                            path = %opts_clone.path,
+                            bucket = %walk_options.bucket,
+                            path = %walk_options.base_dir,
                             disk_index = disk_idx,
                             state = "fallback_disk_missing_for_path",
                             error = ?err,
@@ -610,8 +626,8 @@ async fn list_path_raw_inner(
                             event = EVENT_METACACHE_LISTING,
                             component = LOG_COMPONENT_ECSTORE,
                             subsystem = LOG_SUBSYSTEM_METACACHE,
-                            bucket = %opts_clone.bucket,
-                            path = %opts_clone.path,
+                            bucket = %walk_options.bucket,
+                            path = %walk_options.base_dir,
                             disk_index = disk_idx,
                             state = "fallback_disk_missing",
                             error = ?err,
@@ -626,7 +642,7 @@ async fn list_path_raw_inner(
                 }
 
                 let fallback_walk_started = std::time::Instant::now();
-                match disk.as_ref().walk_dir(walk_dir_options(&opts_clone), &mut wr).await {
+                match disk.as_ref().walk_dir((*walk_options).clone(), &mut wr).await {
                     Ok(_r) => {
                         rustfs_io_metrics::record_stage_duration(
                             "metacache_walk_dir_fallback",
@@ -652,8 +668,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %opts_clone.bucket,
-                                path = %opts_clone.path,
+                                bucket = %walk_options.bucket,
+                                path = %walk_options.base_dir,
                                 disk_index = disk_idx,
                                 state = "fallback_walk_dir_missing_path",
                                 error = ?err,
@@ -664,8 +680,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %opts_clone.bucket,
-                                path = %opts_clone.path,
+                                bucket = %walk_options.bucket,
+                                path = %walk_options.base_dir,
                                 disk_index = disk_idx,
                                 state = "fallback_walk_dir_failed",
                                 error = ?err,
@@ -687,12 +703,13 @@ async fn list_path_raw_inner(
 
             // warn!("list_path_raw: while need_fallback done");
             Ok(())
-        }));
+        });
+        jobs.spawn(producer.in_current_span());
     }
 
     let revjob_rx = rx.clone();
     let mut revjobs = JoinSet::new();
-    revjobs.spawn(async move {
+    let merge = async move {
         // Consumer-side peek timeout, and a caveat worth understanding
         // (rustfs/backlog#1217).
         //
@@ -722,6 +739,8 @@ async fn list_path_raw_inner(
         // Sticky across rounds: once any reader's trailer says its producer
         // stopped at the per-disk limit, the merged stream may hide more keys.
         let mut limit_reached = false;
+        let mut scan_cut = None;
+        let mut last_scanned_key: Option<String> = None;
 
         loop {
             let mut current = MetaCacheEntry::default();
@@ -733,7 +752,7 @@ async fn list_path_raw_inner(
             // );
 
             if revjob_rx.is_cancelled() {
-                return Ok(ListPathRawOutcome { limit_reached });
+                return Ok(ListPathRawOutcome { limit_reached, scan_cut });
             }
 
             let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
@@ -747,6 +766,16 @@ async fn list_path_raw_inner(
             // Start every missing head read in the same round so one stalled
             // disk cannot multiply the wait budget by the erasure-set width.
             // Outcomes are still consumed below in stable disk-index order.
+            if !revjob_rx.is_cancelled() {
+                for (i, reader) in readers.iter_mut().enumerate() {
+                    if errs[i].is_none()
+                        && pending_entries[i].is_none()
+                        && let Some(Ok(Some(entry))) = reader.try_peek_buffered()
+                    {
+                        pending_entries[i] = Some(entry);
+                    }
+                }
+            }
             let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
                 if errs[i].is_some() || pending_entries[i].is_some() {
                     return None;
@@ -783,7 +812,12 @@ async fn list_path_raw_inner(
                                     continue;
                                 }
                                 // eof
-                                limit_reached |= r.limit_reached() == Some(true);
+                                if r.limit_reached() == Some(true) {
+                                    limit_reached = true;
+                                    scan_cut = Some(last_scanned_key.clone().ok_or_else(|| {
+                                        DiskError::other("bounded listing reader ended without a scan boundary")
+                                    })?);
+                                }
                                 at_eof += 1;
                                 // warn!("list_path_raw: peek eof, disk: {}", i);
                                 continue;
@@ -857,7 +891,7 @@ async fn list_path_raw_inner(
                             *r = MetacacheReader::new(detached_rd);
                             continue;
                         }
-                        PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
+                        PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                     }
                 };
 
@@ -878,7 +912,7 @@ async fn list_path_raw_inner(
                 }
                 if entry.name == current.name {
                     // If exact match, we agree.
-                    if current.matches_strict(&entry) {
+                    if current.matches_strict_borrowed(&entry) {
                         top_entries[i] = Some(entry);
                         agree += 1;
                     } else {
@@ -916,7 +950,7 @@ async fn list_path_raw_inner(
                 if let Some(finished_fn) = opts.finished.as_ref() {
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                         _ = finished_fn(&errs) => {}
                     }
                 }
@@ -966,6 +1000,14 @@ async fn list_path_raw_inner(
                 return Err(classify_listing_quorum_failure(&failures));
             }
 
+            // The next head from a bounded reader is unknown. The previous
+            // globally consumed group is a safe cut; consuming another reader's
+            // later head would turn this absence of evidence into a skipped key.
+            // Check quorum failures above before accepting the bounded result.
+            if limit_reached {
+                break;
+            }
+
             // Break if all at EOF or error.
             if at_eof + has_err == readers.len() {
                 if has_err > 0
@@ -973,7 +1015,7 @@ async fn list_path_raw_inner(
                 {
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                         _ = finished_fn(&errs) => {}
                     }
                 }
@@ -984,16 +1026,19 @@ async fn list_path_raw_inner(
                 break;
             }
 
+            last_scanned_key = Some(current.name.clone());
             if agree == readers.len() && !opts.preserve_replica_metadata {
                 for r in readers.iter_mut() {
-                    let _ = r.skip(1).await;
+                    if !r.skip_current() {
+                        let _ = r.skip(1).await;
+                    }
                 }
 
                 if let Some(agreed_fn) = opts.agreed.as_ref() {
                     // warn!("list_path_raw: agreed_fn start, current: {:?}", &current.name);
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                         result = agreed_fn(current) => result?
                     }
                     // warn!("list_path_raw: agreed_fn done");
@@ -1009,7 +1054,7 @@ async fn list_path_raw_inner(
             }
 
             for (i, r) in readers.iter_mut().enumerate() {
-                if top_entries[i].is_some() {
+                if top_entries[i].is_some() && !r.skip_current() {
                     let _ = r.skip(1).await;
                 }
             }
@@ -1017,13 +1062,14 @@ async fn list_path_raw_inner(
             if let Some(partial_fn) = opts.partial.as_ref() {
                 tokio::select! {
                     biased;
-                    _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                    _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                     _ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
                 }
             }
         }
-        Ok(ListPathRawOutcome { limit_reached })
-    });
+        Ok(ListPathRawOutcome { limit_reached, scan_cut })
+    };
+    revjobs.spawn(merge.in_current_span());
 
     let merge_started = std::time::Instant::now();
     let Some(revjob) = revjobs.join_next().await else {
@@ -1187,6 +1233,24 @@ mod tests {
         assert!(!options.skip_total_timeout);
     }
 
+    #[test]
+    fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
+        let request = ListPathRawOptions {
+            bucket: "versioned".to_owned(),
+            hidden_entry_budget: 100,
+            pending_prefix: Some("pending/".to_owned()),
+            forward_to: Some("pending/0099".to_owned()),
+            ..Default::default()
+        };
+        // Both walks are built once from the shared options; the per-job clone
+        // must carry every field that the walk derives from the request.
+        let job = request.clone();
+        let walk = walk_dir_options(&job);
+        assert_eq!(walk.hidden_entry_budget, 100);
+        assert_eq!(walk.pending_prefix.as_deref(), request.pending_prefix.as_deref());
+        assert_eq!(walk.forward_to.as_deref(), Some("pending/0099"));
+    }
+
     #[tokio::test]
     async fn list_path_raw_empty_disks_returns_read_quorum() {
         let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
@@ -1237,6 +1301,163 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn list_path_raw_fanout_retains_only_linear_disk_references() {
+        let root = tempfile::tempdir().expect("create disk root");
+        let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
+            .expect("parse disk endpoint");
+        let local = Arc::new(
+            crate::disk::local::LocalDisk::new(&endpoint, false)
+                .await
+                .expect("open local disk"),
+        );
+        let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
+        let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
+        let baseline = Arc::strong_count(&disk);
+
+        for primary_count in [1, 4, 16] {
+            for fallback_count in [0, 2] {
+                let entered: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
+                let stopped: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
+                let behaviors = entered
+                    .iter()
+                    .zip(&stopped)
+                    .map(|(entered, stopped)| TestReaderBehavior::StallWithSignals {
+                        entered: entered.clone(),
+                        stopped: stopped.clone(),
+                    })
+                    .collect();
+                let cancel = CancellationToken::new();
+                let listing = tokio::spawn(list_path_raw(
+                    cancel.clone(),
+                    ListPathRawOptions {
+                        disks: vec![Some(disk.clone()); primary_count],
+                        fallback_disks: vec![Some(disk.clone()); fallback_count],
+                        min_disks: primary_count,
+                        test_reader_behaviors: behaviors,
+                        peek_timeout: Some(Duration::from_secs(60)),
+                        ..Default::default()
+                    },
+                ));
+                for signal in &entered {
+                    timeout(Duration::from_secs(2), signal.notified())
+                        .await
+                        .expect("every producer must enter the real fanout before counting references");
+                }
+
+                // The merge owns the configured primary/fallback slots, each producer
+                // owns its primary, and the shared fallback queue owns each fallback.
+                let expected = baseline + 2 * (primary_count + fallback_count);
+                let retained = Arc::strong_count(&disk);
+                cancel.cancel();
+                timeout(Duration::from_secs(2), listing)
+                    .await
+                    .expect("cancellation must stop the listing")
+                    .expect("listing task must join")
+                    .expect("external cancellation must finish cleanly");
+                for signal in &stopped {
+                    timeout(Duration::from_secs(2), signal.notified())
+                        .await
+                        .expect("cancellation must drop every producer");
+                }
+                assert_eq!(Arc::strong_count(&disk), baseline, "cancellation must release every fanout reference");
+                assert_eq!(
+                    retained, expected,
+                    "fanout must retain linear references for {primary_count} primary and {fallback_count} fallback slots"
+                );
+            }
+        }
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn list_path_raw_producer_and_merge_preserve_request_span() {
+        use tracing_subscriber::{Layer, layer::SubscriberExt};
+
+        struct ProducerSpan(Arc<AtomicBool>);
+        #[derive(Default)]
+        struct StateField(bool);
+        impl tracing::field::Visit for StateField {
+            fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
+
+            fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
+                self.0 |= field.name() == "state" && value == "walk_dir_missing_path";
+            }
+        }
+        impl<S> Layer<S> for ProducerSpan
+        where
+            S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
+        {
+            fn on_event(&self, event: &tracing::Event<'_>, context: tracing_subscriber::layer::Context<'_, S>) {
+                let mut state = StateField::default();
+                event.record(&mut state);
+                if state.0
+                    && context
+                        .lookup_current()
+                        .is_some_and(|span| span.name() == "flink_list_request")
+                {
+                    self.0.store(true, std::sync::atomic::Ordering::Release);
+                }
+            }
+        }
+
+        // The subscriber is thread-local; keep the spawned tasks on this runtime thread.
+        let producer_observed = Arc::new(AtomicBool::new(false));
+        let subscriber = tracing_subscriber::registry().with(ProducerSpan(producer_observed.clone()));
+        let _subscriber = tracing::subscriber::set_default(subscriber);
+        let request = tracing::info_span!("flink_list_request", request_id = "list-span-regression");
+        let root = tempfile::tempdir().expect("create disk root");
+        let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
+            .expect("parse disk endpoint");
+        let local = Arc::new(
+            crate::disk::local::LocalDisk::new(&endpoint, false)
+                .await
+                .expect("open local disk"),
+        );
+        let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
+        let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
+        let error = list_path_raw(
+            CancellationToken::new(),
+            ListPathRawOptions {
+                disks: vec![Some(disk)],
+                bucket: "missing-bucket".to_owned(),
+                min_disks: 1,
+                ..Default::default()
+            },
+        )
+        .instrument(request.clone())
+        .await
+        .expect_err("missing bucket must fail the real disk producer");
+        assert_eq!(error, DiskError::VolumeNotFound);
+        assert!(
+            producer_observed.load(std::sync::atomic::Ordering::Acquire),
+            "disk error must retain the request span"
+        );
+
+        let merge_observed = Arc::new(AtomicBool::new(false));
+        let merge_sink = merge_observed.clone();
+        let request_id = request.id();
+        list_path_raw(
+            CancellationToken::new(),
+            ListPathRawOptions {
+                disks: vec![None],
+                min_disks: 1,
+                test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![fallback_test_entry()])],
+                agreed: Some(Box::new(move |_| {
+                    merge_sink.store(tracing::Span::current().id() == request_id, std::sync::atomic::Ordering::Release);
+                    Box::pin(async { Ok(()) })
+                })),
+                ..Default::default()
+            },
+        )
+        .instrument(request)
+        .await
+        .expect("merge should deliver the test entry");
+        assert!(
+            merge_observed.load(std::sync::atomic::Ordering::Acquire),
+            "merge callback must retain the request span"
+        );
+    }
+
     #[tokio::test]
     async fn scanner_joinset_walk_uses_the_local_disk_background_gate() {
         let root = tempfile::tempdir().expect("local disk root should be created");
@@ -1304,6 +1525,7 @@ mod tests {
             outcome.limit_reached,
             "a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
         );
+        assert_eq!(outcome.scan_cut.as_deref(), Some("bucket/object"));
     }
 
     #[tokio::test]
@@ -1680,6 +1902,42 @@ mod tests {
         assert_eq!(seen.lock().expect("seen mutex poisoned").as_slice(), &["bucket/object".to_string()]);
     }
 
+    #[tokio::test]
+    async fn list_read_budget_is_retained_by_raw_producer_until_cancellation() {
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(4));
+        let grant = Arc::new(semaphore.clone().acquire_many_owned(4).await.expect("listing grant"));
+        let weak_grant = Arc::downgrade(&grant);
+        let entered = Arc::new(tokio::sync::Notify::new());
+        let stopped = Arc::new(tokio::sync::Notify::new());
+        let worker = tokio::spawn(list_path_raw(
+            CancellationToken::new(),
+            ListPathRawOptions {
+                read_budget: Some(grant),
+                disks: vec![None],
+                min_disks: 1,
+                test_reader_behaviors: vec![TestReaderBehavior::StallWithSignals {
+                    entered: entered.clone(),
+                    stopped: stopped.clone(),
+                }],
+                ..Default::default()
+            },
+        ));
+        timeout(Duration::from_secs(5), entered.notified())
+            .await
+            .expect("producer entered");
+        assert!(weak_grant.strong_count() >= 2, "producer must retain a grant independently of the parent");
+        assert_eq!(semaphore.available_permits(), 0);
+        worker.abort();
+        assert!(worker.await.expect_err("parent cancelled").is_cancelled());
+        timeout(Duration::from_secs(5), stopped.notified())
+            .await
+            .expect("producer stopped");
+        let _next = timeout(Duration::from_secs(5), semaphore.acquire_many(4))
+            .await
+            .expect("cancelled listing releases all permits")
+            .expect("budget stays open");
+    }
+
     #[tokio::test]
     async fn list_path_raw_aborts_unresponsive_producer_after_quorum_eof() {
         let result = timeout(

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -32,7 +32,6 @@ use crate::storage_api_contracts::internode::{
     PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
 };
 
-use hmac::{Hmac, KeyInit, Mac};
 use http::uri::Authority;
 use http::{HeaderMap, HeaderValue, Method, Uri};
 #[cfg(test)]
@@ -47,6 +46,7 @@ use rustfs_io_metrics::internode_metrics::{
     global_internode_metrics,
 };
 use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
+use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
 use rustfs_utils::get_env_bool;
 use sha2::Digest as _;
 use sha2::Sha256;
@@ -58,8 +58,6 @@ use time::OffsetDateTime;
 use tracing::{error, info, warn};
 use uuid::Uuid;
 
-type HmacSha256 = Hmac<Sha256>;
-
 const SIGNATURE_HEADER: &str = "x-rustfs-signature";
 pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
 const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
@@ -368,10 +366,84 @@ fn resolve_replay_cache_capacity() -> usize {
     decision.capacity
 }
 
+const RPC_NONCE_EXPIRATIONS_PER_PAGE: usize = 1_024;
+
+struct RpcNonceExpiration {
+    nonce: Uuid,
+    expiry_offset_ns: u32,
+    valid_until_offset_secs: i32,
+}
+
+struct RpcNonceExpirationPage {
+    expires_at: Instant,
+    valid_until: i64,
+    entries: VecDeque<RpcNonceExpiration>,
+}
+
+#[derive(Default)]
+struct RpcNonceExpirations {
+    pages: VecDeque<RpcNonceExpirationPage>,
+}
+
+impl RpcNonceExpirations {
+    fn push_back(&mut self, expires_at: Instant, valid_until: i64, nonce: Uuid) {
+        // Use exact offsets and start another page if either offset is too large.
+        // Fully drained pages can be freed while newer nonces are still valid.
+        if let Some(page) = self.pages.back_mut()
+            && page.entries.len() < RPC_NONCE_EXPIRATIONS_PER_PAGE
+            && let Some(offset) = expires_at.checked_duration_since(page.expires_at)
+            && let Ok(expiry_offset_ns) = u32::try_from(offset.as_nanos())
+            && let Some(offset) = valid_until.checked_sub(page.valid_until)
+            && let Ok(valid_until_offset_secs) = i32::try_from(offset)
+        {
+            page.entries.push_back(RpcNonceExpiration {
+                nonce,
+                expiry_offset_ns,
+                valid_until_offset_secs,
+            });
+        } else {
+            self.pages.push_back(RpcNonceExpirationPage {
+                expires_at,
+                valid_until,
+                entries: VecDeque::from([RpcNonceExpiration {
+                    nonce,
+                    expiry_offset_ns: 0,
+                    valid_until_offset_secs: 0,
+                }]),
+            });
+        }
+    }
+
+    fn pop_expired(&mut self, now: Instant, wall_time: i64) -> Option<Uuid> {
+        let page = self.pages.front_mut()?;
+        let entry = page.entries.front()?;
+        let elapsed = now.checked_duration_since(page.expires_at)?;
+        let valid_until = page.valid_until + i64::from(entry.valid_until_offset_secs);
+        if elapsed <= Duration::from_nanos(u64::from(entry.expiry_offset_ns)) || wall_time <= valid_until {
+            return None;
+        }
+
+        let nonce = entry.nonce;
+        page.entries.pop_front();
+        if page.entries.is_empty() {
+            self.pages.pop_front();
+            if self.pages.is_empty() {
+                self.pages.shrink_to_fit();
+            }
+        }
+        Some(nonce)
+    }
+
+    #[cfg(test)]
+    fn capacity(&self) -> usize {
+        self.pages.iter().map(|page| page.entries.capacity()).sum()
+    }
+}
+
 #[derive(Default)]
 struct RpcNonceCache {
     nonces: HashSet<Uuid>,
-    expirations: VecDeque<(Instant, i64, Uuid)>,
+    expirations: RpcNonceExpirations,
     max_wall_time: i64,
 }
 
@@ -427,16 +499,13 @@ fn publish_nonce_cache_metrics(metrics: Option<RpcNonceCacheMetrics<'_>>) {
 impl RpcNonceCache {
     fn remove_expired(&mut self, now: Instant, wall_time: i64) -> usize {
         let mut removed = 0;
-        while matches!(
-            self.expirations.front(),
-            Some((expires_at, valid_until, _)) if *expires_at < now && *valid_until < wall_time
-        ) {
-            let Some((_, _, nonce)) = self.expirations.pop_front() else {
-                break;
-            };
+        while let Some(nonce) = self.expirations.pop_expired(now, wall_time) {
             self.nonces.remove(&nonce);
             removed += 1;
         }
+        if removed > 0 && self.nonces.is_empty() {
+            self.nonces.shrink_to_fit();
+        }
         removed
     }
 
@@ -470,7 +539,7 @@ impl RpcNonceCache {
         }
         self.nonces.insert(record.nonce);
         self.expirations
-            .push_back((record.expires_at, record.signed_at.saturating_add(SIGNATURE_VALID_DURATION), record.nonce));
+            .push_back(record.expires_at, record.signed_at.saturating_add(SIGNATURE_VALID_DURATION), record.nonce);
         (
             Ok(()),
             Some(RpcNonceCacheMetrics {
@@ -1590,7 +1659,13 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
     if boot_epoch != tonic_rpc_boot_epoch() {
         return Err(std::io::Error::other("RPC boot epoch is stale"));
     }
-    check_and_record_tonic_nonce(nonce, signed_at, path)
+    check_and_record_tonic_nonce(nonce, signed_at, path)?;
+    // A body-bound request also carries a usable v2 signature. Consume both
+    // nonces so stripping the replay-scope headers cannot authorize it again.
+    if content_sha256 != UNSIGNED_PAYLOAD {
+        verify_tonic_rpc_signature_with_strictness(audience, path, headers, true)?;
+    }
+    Ok(())
 }
 
 /// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
@@ -2581,6 +2656,43 @@ mod tests {
         assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
     }
 
+    #[test]
+    fn replay_scope_consumes_the_body_bound_v2_nonce() {
+        ensure_test_rpc_secret();
+        let audience = "node-a:9000";
+        let path = "/node_service.NodeService/RecoverDeleteTransaction";
+        let digest = "a".repeat(64);
+        for v3_first in [true, false] {
+            let mut headers =
+                gen_tonic_signature_headers(audience, "node_service.NodeService", "RecoverDeleteTransaction", Some(&digest))
+                    .expect("body-bound v2 headers should build");
+            let timestamp = headers[TIMESTAMP_HEADER].to_str().expect("timestamp").to_string();
+            headers.extend(
+                gen_tonic_replay_scope_headers(audience, path, &timestamp, &digest, tonic_rpc_boot_epoch())
+                    .expect("replay-scope headers should build"),
+            );
+            let mut downgraded = headers.clone();
+            for header in [
+                RPC_REPLAY_SCOPE_VERSION_HEADER,
+                RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
+                RPC_REPLAY_SCOPE_NONCE_HEADER,
+                RPC_BOOT_EPOCH_HEADER,
+            ] {
+                downgraded.remove(header);
+            }
+            let (first, replay) = if v3_first {
+                (&headers, &downgraded)
+            } else {
+                (&downgraded, &headers)
+            };
+            verify_tonic_rpc_signature_with_policy(audience, path, first, true, false, false)
+                .expect("the first delivery must be accepted");
+            let error = verify_tonic_rpc_signature_with_policy(audience, path, replay, true, false, false)
+                .expect_err("a mutation cannot replay through the other signature generation");
+            assert_eq!(error.to_string(), "RPC request replay detected");
+        }
+    }
+
     #[test]
     fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
         ensure_test_rpc_secret();
@@ -3137,6 +3249,51 @@ mod tests {
         assert!(cache.nonces.contains(&nonce_b));
     }
 
+    #[test]
+    fn nonce_cache_releases_expiration_storage_after_a_burst() {
+        let start = Instant::now();
+        let expiry = start + REPLAY_CACHE_RETENTION;
+        let mut cache = RpcNonceCache::default();
+
+        for nonce in 1..=4_096 {
+            let record = test_nonce_record(Uuid::from_u128(nonce), 100, start, 100, expiry, 4_096);
+            cache.check_and_record(record).0.expect("first use of nonce");
+        }
+        assert!(cache.expirations.capacity() >= 4_096);
+        assert_eq!(cache.remove_expired(expiry, 701), 0);
+        assert_eq!(cache.remove_expired(expiry + Duration::from_nanos(1), 400), 0);
+        assert_eq!(cache.remove_expired(expiry + Duration::from_nanos(1), 701), 4_096);
+        assert!(cache.nonces.is_empty());
+        assert_eq!(cache.nonces.capacity(), 0);
+        assert_eq!(cache.expirations.capacity(), 0, "expired entries must release their expiration storage");
+    }
+
+    #[test]
+    fn nonce_expirations_preserve_exact_deadlines_across_pages() {
+        let expiry = Instant::now();
+        let first = Uuid::from_u128(1);
+        let second = Uuid::from_u128(2);
+        let third = Uuid::from_u128(3);
+        let fourth = Uuid::from_u128(4);
+        let mut queue = super::RpcNonceExpirations::default();
+        let next = expiry + Duration::from_nanos(1);
+        let later = expiry + Duration::from_secs(5);
+
+        queue.push_back(expiry, 501, first);
+        queue.push_back(next, 500, second);
+        queue.push_back(later, 505, third);
+        queue.push_back(later, i64::MAX, fourth);
+        assert_eq!(queue.pages.len(), 3);
+        assert_eq!(queue.pop_expired(next, 501), None);
+        assert_eq!(queue.pop_expired(next, 502), Some(first));
+        assert_eq!(queue.pop_expired(next, 502), None);
+        assert_eq!(queue.pop_expired(later, 502), Some(second));
+        assert_eq!(queue.pages.len(), 2);
+        assert_eq!(queue.pop_expired(later, 506), None);
+        assert_eq!(queue.pop_expired(later + Duration::from_nanos(1), 506), Some(third));
+        assert_eq!(queue.pop_expired(later + Duration::from_nanos(1), i64::MAX), None);
+    }
+
     #[test]
     fn nonce_cache_metrics_mark_successful_records_only() {
         let now = Instant::now();

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -2014,9 +2014,21 @@ impl PeerRestClient {
             }
             result
         })
-        .await
-        .unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
-        self.finalize_result(result).await
+        .await;
+        self.settle_bucket_metadata_reload(bucket, result).await
+    }
+
+    async fn settle_bucket_metadata_reload(
+        &self,
+        bucket: &str,
+        result: std::result::Result<Result<()>, tokio::time::error::Elapsed>,
+    ) -> Result<()> {
+        match result {
+            Ok(result) => self.finalize_result(result).await,
+            // A local operation deadline does not establish that the peer is
+            // unreachable. In particular, it must not gate IAM revocations.
+            Err(_) => Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))),
+        }
     }
 
     async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
@@ -4753,6 +4765,31 @@ mod tests {
         assert!(client.offline.load(Ordering::Acquire));
     }
 
+    #[tokio::test]
+    async fn bucket_metadata_deadline_does_not_gate_healthy_peer_notifications() {
+        let client = test_peer_client();
+        let elapsed = tokio::time::timeout(std::time::Duration::ZERO, std::future::pending::<Result<()>>()).await;
+        let error = client
+            .settle_bucket_metadata_reload("slow-policy", elapsed)
+            .await
+            .expect_err("local budget expires");
+        assert!(error.to_string().contains("timed out"));
+        assert!(
+            !client.offline.load(Ordering::Acquire),
+            "local work must not suppress subsequent IAM RPCs"
+        );
+        client
+            .settle_bucket_metadata_reload("healthy", Ok(Ok(())))
+            .await
+            .expect("peer reply");
+        let error = client
+            .settle_bucket_metadata_reload("unreachable", Ok(Err(Error::from(tonic::Status::unavailable("connection refused")))))
+            .await
+            .expect_err("real transport failure");
+        assert!(error.to_string().contains("connection refused"));
+        assert!(client.offline.load(Ordering::Acquire), "transport failures must still enter recovery");
+    }
+
     #[tokio::test]
     async fn peer_rest_client_finalize_result_keeps_online_for_business_errors() {
         let client = test_peer_client();

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -27,13 +27,13 @@ use crate::disk::multipart_source::{
     MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartSourceOperation, MultipartSourceRequest, MultipartSourceResponse,
 };
 use crate::disk::{
-    BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, DELETE_RECOVERY_PROTOCOL_VERSION,
-    DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection,
-    DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
-    DiskOption, FileInfoVersions, FileReader, FileWriter, GuardedDeleteVersionsPayload, GuardedFileInfoVersions,
-    PartTransactionAction, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse, ReadMultipleReq, ReadMultipleResp,
-    ReadOptions, RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
-    batch_read_version_one_by_one,
+    BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION, BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate,
+    DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
+    DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DiskAPI, DiskInfo,
+    DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader, FileWriter, GuardedDeleteVersionsPayload,
+    GuardedFileInfoVersions, PartTransactionAction, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
+    ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo,
+    WalkDirOptions, batch_read_version_one_by_one,
     disk_store::{
         DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
         get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
@@ -996,21 +996,32 @@ fn internode_idempotent_read_retries() -> usize {
 static PREWARMED_PEERS: std::sync::LazyLock<std::sync::Mutex<std::collections::HashSet<String>>> =
     std::sync::LazyLock::new(|| std::sync::Mutex::new(std::collections::HashSet::new()));
 
+fn claim_control_channel_prewarm(addr: &str) -> bool {
+    let Ok(mut prewarmed) = PREWARMED_PEERS.lock() else {
+        return false;
+    };
+    prewarmed.insert(addr.to_owned())
+}
+
+fn release_control_channel_prewarm(addr: &str) {
+    if let Ok(mut prewarmed) = PREWARMED_PEERS.lock() {
+        prewarmed.remove(addr);
+    }
+}
+
 /// Best-effort background prewarm of a peer's control channel (grpc-optimization P3-1). Deduped per
-/// peer; a dial failure just falls through to the existing lazy connect and recovery monitor.
+/// peer; a dial failure releases the marker so a later disk construction can retry the prewarm.
 fn spawn_control_channel_prewarm(addr: String) {
-    {
-        let Ok(mut prewarmed) = PREWARMED_PEERS.lock() else {
-            return;
-        };
-        if !prewarmed.insert(addr.clone()) {
-            return;
-        }
+    if !claim_control_channel_prewarm(&addr) {
+        return;
     }
     tokio::spawn(async move {
         match node_service_time_out_client_no_auth(&addr).await {
             Ok(_) => debug!(addr = %addr, "internode control channel prewarmed"),
-            Err(err) => debug!(addr = %addr, error = %err, "internode control channel prewarm failed (best-effort)"),
+            Err(err) => {
+                release_control_channel_prewarm(&addr);
+                debug!(addr = %addr, error = %err, "internode control channel prewarm failed (best-effort)");
+            }
         }
     });
 }
@@ -1021,6 +1032,7 @@ impl RemoteDisk {
         volume: &str,
         payload: Vec<u8>,
         probe_only: bool,
+        bounded_prepare: bool,
     ) -> Result<DeleteVersionsResponse> {
         if payload.len() > rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES {
             return Err(DiskError::FileCorrupt);
@@ -1045,6 +1057,8 @@ impl RemoteDisk {
                     .map_err(|status| {
                         if status.code() == Code::Unimplemented {
                             DiskError::GuardedDeleteUnsupported
+                        } else if bounded_prepare && status.code() == Code::InvalidArgument {
+                            DiskError::UnsupportedDisk
                         } else {
                             status.into()
                         }
@@ -1078,11 +1092,22 @@ impl RemoteDisk {
         volume: &str,
         operation: DeleteRecoveryRpcOperation,
     ) -> Result<DeleteRecoveryRpcReply> {
+        let protocol_version = match &operation {
+            DeleteRecoveryRpcOperation::ApplyVersion { opts, .. } | DeleteRecoveryRpcOperation::ApplyVersions { opts, .. }
+                if opts.requires_bounded_delete_recovery =>
+            {
+                BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION
+            }
+            _ => DELETE_RECOVERY_PROTOCOL_VERSION,
+        };
         let payload = rmp_serde::to_vec_named(&operation)?;
         if payload.len() > operation.max_wire_payload_bytes() {
             return Err(Error::other("delete recovery operation exceeds payload limit"));
         }
-        let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) {
+        let settle_timeout = if matches!(
+            &operation,
+            DeleteRecoveryRpcOperation::Settle { .. } | DeleteRecoveryRpcOperation::SettleBounded(_)
+        ) {
             Some(crate::disk::get_delete_recovery_timeout()?)
         } else {
             None
@@ -1105,7 +1130,7 @@ impl RemoteDisk {
                 let mut request = Request::new(RecoverDeleteTransactionRequest {
                     disk: self.endpoint.to_string(),
                     volume: volume.to_owned(),
-                    protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
+                    protocol_version,
                     operation: payload.into(),
                 });
                 let canonical_body = rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref())
@@ -1123,7 +1148,7 @@ impl RemoteDisk {
                         }
                     })?
                     .into_inner();
-                if response.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION {
+                if response.protocol_version != protocol_version {
                     return Err(DiskError::UnsupportedDisk);
                 }
                 if !response.success {
@@ -1637,6 +1662,7 @@ impl RemoteDisk {
         let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
         if initial_probe_ok {
             health.record_operation_success(&endpoint, "connectivity_probe_success");
+            rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
         } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
             warn!(
                 event = EVENT_REMOTE_DISK_HEALTH,
@@ -1708,6 +1734,7 @@ impl RemoteDisk {
                     // Perform basic connectivity check
                     if Self::perform_connectivity_check(&addr).await.is_ok() {
                         health.record_operation_success(&endpoint, "connectivity_probe_success");
+                        rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
                     } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
                         warn!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -1778,6 +1805,7 @@ impl RemoteDisk {
                 }
                 _ = interval.tick() => {
                     if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
+                        rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
                         let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
                         debug!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -2009,6 +2037,7 @@ impl RemoteDisk {
             let operation_result = operation().await;
             if operation_result.is_ok() {
                 self.health.record_operation_success(&self.endpoint, "operation_success");
+                rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
             }
             self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                 .await;
@@ -2023,6 +2052,7 @@ impl RemoteDisk {
                 // Log success; the waiting guard balances every exit path.
                 if operation_result.is_ok() {
                     self.health.record_operation_success(&self.endpoint, "operation_success");
+                    rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
                 }
                 self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                     .await;
@@ -2711,8 +2741,56 @@ impl RemoteDisk {
 
 #[async_trait::async_trait]
 impl DiskAPI for RemoteDisk {
+    async fn list_multipart_index_roots(
+        &self,
+        bucket: &str,
+        marker: Option<&str>,
+    ) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
+        use crate::disk::multipart_index::{MULTIPART_INDEX_RPC_MAX_BYTES, MultipartIndexRpcReply, MultipartIndexRpcRequest};
+        let expected_disk = self
+            .id
+            .lock()
+            .await
+            .filter(|id| !id.is_nil())
+            .ok_or(DiskError::DiskNotFound)?;
+        let query = MultipartIndexRpcRequest::new(bucket, marker);
+        let endpoint = self.endpoint.to_string();
+        let canonical = query.canonical_request(&endpoint)?;
+        let opts = serde_json::to_string(&query)?;
+        self.execute_with_timeout(
+            || async {
+                let mut client = self.get_client().await?;
+                let mut request = Request::new(DiskInfoRequest {
+                    disk: endpoint.clone(),
+                    opts: opts.clone(),
+                });
+                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
+                let response = client.disk_info(request).await?.into_inner();
+                if !response.success {
+                    return Err(response.error.unwrap_or_default().into());
+                }
+                if response.disk_info.len() > MULTIPART_INDEX_RPC_MAX_BYTES {
+                    return Err(DiskError::FileCorrupt);
+                }
+                // An old server ignores the additive query and returns ordinary DiskInfo.
+                // It cannot satisfy the dedicated reply schema or the fresh challenge.
+                let reply: MultipartIndexRpcReply =
+                    serde_json::from_str(&response.disk_info).map_err(|_| DiskError::MethodNotAllowed)?;
+                reply.validate(&query, expected_disk)?;
+                crate::cluster::rpc::verify_tonic_rpc_response_proof(
+                    &query.canonical_response(&endpoint, &reply.page)?,
+                    &reply.proof,
+                )
+                .map_err(Error::other)?;
+                Ok(reply.page)
+            },
+            get_drive_disk_info_timeout(),
+        )
+        .await
+    }
+
     async fn supports_guarded_delete_versions(&self) -> Result<()> {
-        let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
+        let response = self.guarded_delete_versions_rpc("", Vec::new(), true, false).await?;
         if !response.errors.is_empty() || !response.item_errors.is_empty() {
             return Err(DiskError::FileCorrupt);
         }
@@ -2725,7 +2803,12 @@ impl DiskAPI for RemoteDisk {
         opts: DeleteOptions,
     ) -> Vec<Option<DiskError>> {
         let count = versions.len();
-        let payload = GuardedDeleteVersionsPayload { versions, opts };
+        let bounded_prepare = opts.requires_bounded_delete_recovery;
+        let payload = GuardedDeleteVersionsPayload {
+            versions,
+            opts,
+            bounded_recovery_protocol: bounded_prepare.then_some(BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION),
+        };
         if let Err(error) = payload.validate() {
             return vec![Some(error); count];
         }
@@ -2733,7 +2816,10 @@ impl DiskAPI for RemoteDisk {
             Ok(encoded) => encoded,
             Err(error) => return vec![Some(error.into()); count],
         };
-        match self.guarded_delete_versions_rpc(volume, encoded, false).await {
+        match self
+            .guarded_delete_versions_rpc(volume, encoded, false, bounded_prepare)
+            .await
+        {
             Ok(response) => {
                 // New guarded peers must return typed errors, never lossy strings.
                 if response.item_errors.len() != count {
@@ -3624,7 +3710,13 @@ impl DiskAPI for RemoteDisk {
                         );
                     }
                     let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled);
-                    let response = match client.batch_read_version(request).await {
+                    let response_result = client.batch_read_version(request).await;
+                    if let Some(rpc_started) = rpc_started {
+                        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_duration(
+                            rpc_started.elapsed(),
+                        );
+                    }
+                    let response = match response_result {
                         Ok(response) => {
                             record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
                             response.into_inner()
@@ -4294,6 +4386,38 @@ impl DiskAPI for RemoteDisk {
         }
     }
 
+    async fn discover_delete_transactions_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        max_entries: u32,
+    ) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
+        // Older peers reject this variant. Do not retry through the unbounded protocol;
+        // they are reported as unsupported participants (rolling-upgrade contract).
+        match self
+            .recover_delete_transaction(
+                volume,
+                DeleteRecoveryRpcOperation::DiscoverBounded {
+                    object: object.to_owned(),
+                    max_entries,
+                },
+            )
+            .await
+        {
+            Ok(DeleteRecoveryRpcReply::DiscoveredBounded(page)) => Ok(page),
+            Ok(_) => Err(Error::FileCorrupt),
+            Err(error)
+                if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
+                    || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
+                    .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
+            {
+                // No paginated fallback through the unbounded protocol; the peer
+                // is an unsupported participant until it is upgraded.
+                Err(Error::UnsupportedDisk)
+            }
+            Err(error) => Err(error),
+        }
+    }
     async fn discover_delete_transactions(
         &self,
         volume: &str,
@@ -4329,6 +4453,62 @@ impl DiskAPI for RemoteDisk {
             _ => Err(Error::other("unexpected delete discovery reply")),
         }
     }
+    async fn settle_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        request: crate::disk::BoundedDeleteRecoverySettlement,
+        _namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
+        match self
+            .recover_delete_transaction(volume, DeleteRecoveryRpcOperation::SettleBounded(request))
+            .await
+        {
+            Ok(DeleteRecoveryRpcReply::SettledBounded(reply)) => Ok(reply),
+            Ok(_) => Err(Error::FileCorrupt),
+            Err(error)
+                if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
+                    || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
+                    .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
+            {
+                // No paginated fallback through the unbounded protocol; the peer
+                // is an unsupported participant until it is upgraded.
+                Err(Error::UnsupportedDisk)
+            }
+            Err(error) => Err(error),
+        }
+    }
+    async fn inspect_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        transaction: Uuid,
+        max_entries: u32,
+    ) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
+        match self
+            .recover_delete_transaction(
+                volume,
+                DeleteRecoveryRpcOperation::InspectBounded {
+                    object: object.to_owned(),
+                    transaction,
+                    max_entries,
+                },
+            )
+            .await
+        {
+            Ok(DeleteRecoveryRpcReply::InspectedBounded(page)) => Ok(page),
+            Ok(_) => Err(Error::FileCorrupt),
+            Err(error)
+                if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
+                    || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
+                    .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
+            {
+                // No paginated fallback through the unbounded protocol; the peer
+                // is an unsupported participant until it is upgraded.
+                Err(Error::UnsupportedDisk)
+            }
+            Err(error) => Err(error),
+        }
+    }
     async fn inspect_delete_transaction(
         &self,
         volume: &str,
@@ -4551,7 +4731,10 @@ impl DiskAPI for RemoteDisk {
                     read_multiple_req_bin: read_multiple_req_bin.into(),
                 });
 
-                let response = client.read_multiple(request).await?.into_inner();
+                let rpc_started = Instant::now();
+                let response = client.read_multiple(request).await;
+                crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_multiple_duration(rpc_started.elapsed());
+                let response = response?.into_inner();
 
                 if !response.success {
                     return Err(response.error.unwrap_or_default().into());
@@ -4602,6 +4785,7 @@ impl DiskAPI for RemoteDisk {
                 attach_mutation_body_digest(&mut request, canonical_body, "write_all")?;
 
                 crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_request();
+                let rpc_started = Instant::now();
                 let response = match client.write_all(request).await {
                     Ok(response) => response.into_inner(),
                     Err(err) => {
@@ -4610,6 +4794,7 @@ impl DiskAPI for RemoteDisk {
                     }
                 };
 
+                crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_duration(rpc_started.elapsed());
                 crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_sent_bytes(data_len);
 
                 if !response.success {
@@ -4707,6 +4892,7 @@ impl DiskAPI for RemoteDisk {
                 });
 
                 crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_request();
+                let rpc_started = Instant::now();
                 let response = match client.read_all(request).await {
                     Ok(response) => response.into_inner(),
                     Err(err) => {
@@ -4715,6 +4901,8 @@ impl DiskAPI for RemoteDisk {
                     }
                 };
 
+                crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_duration(rpc_started.elapsed());
+
                 if !response.success {
                     crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_error();
                     return Err(response.error.unwrap_or_default().into());
@@ -4849,6 +5037,113 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn bounded_delete_discovery_lost_remote_reply_leaves_no_cursor() {
+        runtime_sources::ensure_test_rpc_secret();
+        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
+            .await
+            .expect("test peer must bind");
+        let directory = tempfile::tempdir().expect("temporary disk");
+        let endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("endpoint");
+        let disk = Arc::new(
+            crate::disk::local::LocalDisk::new(&endpoint, false)
+                .await
+                .expect("local disk"),
+        );
+        disk.make_volume("bucket").await.expect("bucket");
+        let prefix = directory.path().join("bucket/prefix");
+        std::fs::create_dir(&prefix).expect("prefix");
+        for index in 0..64 {
+            std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
+        }
+        let fixture = Arc::new(TestBoundedDiscovery {
+            disk: Arc::clone(&disk),
+            response_gate: TestFormatReadGate::default(),
+        });
+        *peer.peer.bounded_discovery.lock().expect("bounded fixture") = Some(Arc::clone(&fixture));
+        let remote = remote_disk_for_addr(&peer.addr).await;
+        let task = tokio::spawn(async move { remote.discover_delete_transactions_bounded("bucket", "prefix", 8).await });
+        tokio::time::timeout(Duration::from_secs(5), fixture.response_gate.entered.notified())
+            .await
+            .expect("disk completed bounded scan");
+        task.abort();
+        assert!(task.await.expect_err("caller cancelled").is_cancelled());
+        assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
+        fixture.response_gate.release.cancel();
+        peer.stop().await;
+        assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
+        assert!(prefix.join("child-63").is_dir());
+    }
+
+    #[tokio::test]
+    async fn multipart_index_old_disk_info_handler_is_not_a_ready_empty_index() {
+        runtime_sources::ensure_test_rpc_secret();
+        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
+            .await
+            .expect("old peer must bind");
+        let remote = remote_disk_for_addr(&peer.addr).await;
+        remote
+            .set_disk_id(Some(Uuid::new_v4()))
+            .await
+            .expect("expected disk identity");
+        assert!(matches!(
+            remote.list_multipart_index_roots("bucket", None).await,
+            Err(DiskError::MethodNotAllowed)
+        ));
+        assert_eq!(
+            peer.peer.disk_info_calls.load(Ordering::Acquire),
+            1,
+            "exercise the actual legacy JSON handler"
+        );
+        peer.stop().await;
+    }
+
+    #[tokio::test]
+    async fn bounded_delete_discovery_old_peer_rejects_without_paginated_fallback() {
+        runtime_sources::ensure_test_rpc_secret();
+        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
+            .await
+            .expect("old peer must bind");
+        let remote = remote_disk_for_addr(&peer.addr).await;
+        assert_eq!(
+            remote
+                .discover_delete_transactions_bounded("bucket", "prefix", 4096)
+                .await
+                .expect_err("old peer cannot prove a bounded inventory"),
+            DiskError::UnsupportedDisk
+        );
+        let transaction = Uuid::new_v4();
+        assert_eq!(
+            remote
+                .inspect_delete_transaction_bounded("bucket", "prefix", transaction, 65536)
+                .await
+                .expect_err("old peer cannot inspect with a budget"),
+            DiskError::UnsupportedDisk
+        );
+        assert_eq!(
+            remote
+                .settle_delete_transaction_bounded(
+                    "bucket",
+                    crate::disk::BoundedDeleteRecoverySettlement {
+                        object: "prefix".to_owned(),
+                        transaction,
+                        expected_fingerprint: [0; 32],
+                        action: DeleteRecoveryAction::PublishCommit,
+                        max_entries: 65536,
+                    },
+                    None
+                )
+                .await
+                .expect_err("old peer cannot settle with a budget"),
+            DiskError::UnsupportedDisk
+        );
+        assert_eq!(
+            *peer.peer.unsupported_methods.lock().expect("observed methods"),
+            vec!["/node_service.NodeService/RecoverDeleteTransaction"; 3]
+        );
+        peer.stop().await;
+    }
+
     #[tokio::test]
     async fn multipart_source_recovery_old_peer_rejects_without_generic_fallback() {
         runtime_sources::ensure_test_rpc_secret();
@@ -5740,10 +6035,18 @@ mod tests {
         object_read_all_disks: Arc<StdMutex<Vec<String>>>,
         format_data: Bytes,
         read_all_data: Bytes,
+        fail_format_reads: Arc<AtomicBool>,
         next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
+        bounded_discovery: Arc<StdMutex<Option<Arc<TestBoundedDiscovery>>>>,
         unsupported_methods: Arc<StdMutex<Vec<String>>>,
     }
 
+    #[derive(Debug)]
+    struct TestBoundedDiscovery {
+        disk: Arc<crate::disk::local::LocalDisk>,
+        response_gate: TestFormatReadGate,
+    }
+
     #[derive(Debug, Default)]
     struct TestFormatReadGate {
         entered: tokio::sync::Notify,
@@ -5759,7 +6062,9 @@ mod tests {
                 object_read_all_disks: Arc::default(),
                 format_data,
                 read_all_data,
+                fail_format_reads: Arc::default(),
                 next_format_read_gate: Arc::default(),
+                bounded_discovery: Arc::default(),
                 unsupported_methods: Arc::default(),
             }
         }
@@ -5859,6 +6164,56 @@ mod tests {
                         Ok(grpc.unary(method, request).await)
                     })
                 }
+                "/node_service.NodeService/RecoverDeleteTransaction"
+                    if self.peer.bounded_discovery.lock().expect("bounded fixture").is_some() =>
+                {
+                    #[derive(Clone)]
+                    struct BoundedSvc(Arc<TestBoundedDiscovery>);
+                    impl tonic::server::UnaryService<RecoverDeleteTransactionRequest> for BoundedSvc {
+                        type Response = rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse;
+                        type Future = Pin<Box<dyn Future<Output = std::result::Result<Response<Self::Response>, Status>> + Send>>;
+                        fn call(&mut self, request: Request<RecoverDeleteTransactionRequest>) -> Self::Future {
+                            let fixture = Arc::clone(&self.0);
+                            Box::pin(async move {
+                                let request = request.into_inner();
+                                let operation: DeleteRecoveryRpcOperation =
+                                    rustfs_protos::decode_delete_recovery_payload(&request.operation)
+                                        .map_err(|_| Status::invalid_argument("invalid bounded request"))?;
+                                let DeleteRecoveryRpcOperation::DiscoverBounded { object, max_entries } = operation else {
+                                    return Err(Status::invalid_argument("unexpected fallback"));
+                                };
+                                let page = fixture
+                                    .disk
+                                    .discover_delete_transactions_bounded(&request.volume, &object, max_entries)
+                                    .await
+                                    .map_err(|_| Status::internal("bounded discovery failed"))?;
+                                // Lose the response only after the real disk operation has completed.
+                                fixture.response_gate.entered.notify_one();
+                                fixture.response_gate.release.cancelled().await;
+                                let result = rmp_serde::to_vec_named(&DeleteRecoveryRpcReply::DiscoveredBounded(page))
+                                    .map_err(|_| Status::internal("encode bounded reply"))?;
+                                Ok(Response::new(rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse {
+                                    success: true,
+                                    protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
+                                    result: result.into(),
+                                    error: None,
+                                }))
+                            })
+                        }
+                    }
+                    let fixture = self
+                        .peer
+                        .bounded_discovery
+                        .lock()
+                        .expect("bounded fixture")
+                        .clone()
+                        .expect("configured fixture");
+                    Box::pin(async move {
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec);
+                        Ok(grpc.unary(BoundedSvc(fixture), request).await)
+                    })
+                }
                 "/node_service.NodeService/ReadAll" => {
                     #[derive(Clone)]
                     struct ReadAllSvc(AuthenticatedReadPeer);
@@ -5882,6 +6237,9 @@ mod tests {
                                         gate.entered.notify_one();
                                         gate.release.cancelled().await;
                                     }
+                                    if peer.fail_format_reads.load(Ordering::Acquire) {
+                                        return Err(Status::unavailable("format read temporarily unavailable"));
+                                    }
                                     peer.format_data.clone()
                                 } else {
                                     peer.object_read_all_disks
@@ -8847,6 +9205,124 @@ mod tests {
         peer.stop().await;
     }
 
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn endpoint_reconnect_failure_preserves_registered_remote_recovery() {
+        temp_env::async_with_vars(
+            [
+                (crate::disk::disk_store::ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, Some("true")),
+                (rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD, Some("1")),
+            ],
+            async {
+                runtime_sources::ensure_test_rpc_secret();
+                let mut reference = crate::layout::format::FormatV3::new(1, 1);
+                reference.erasure.this = reference.erasure.sets[0][0];
+                let peer = TestGrpcPeer::spawn(
+                    Bytes::from(reference.to_json().expect("format should serialize")),
+                    Bytes::from_static(b"recovered-peer-data"),
+                )
+                .await
+                .expect("authenticated regression peer is required");
+                let set = missing_read_set(&peer, reference).await;
+                let disk = crate::disk::new_disk(
+                    &set.set_endpoints[0],
+                    &DiskOption {
+                        cleanup: false,
+                        health_check: true,
+                    },
+                )
+                .await
+                .expect("registered remote disk should construct");
+                let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
+                    panic!("fixture must create a remote disk");
+                };
+                assert!(remote.health_check);
+                remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
+                set.disks.write().await[0] = Some(Arc::clone(&disk));
+
+                peer.peer.fail_format_reads.store(true, Ordering::Release);
+                set.connect_disks().await;
+                peer.peer.fail_format_reads.store(false, Ordering::Release);
+                let cancelled_after_failure = remote.cancel_token.is_cancelled();
+                assert!(peer.peer.read_all_calls() > 0, "reconnect must attempt the failing format RPC");
+                assert!(
+                    Arc::ptr_eq(&disk, set.disks.read().await[0].as_ref().expect("existing slot must remain")),
+                    "a failed reconnect must not publish its unvalidated candidate"
+                );
+                remote.spawn_recovery_monitor_if_needed();
+                let recovery = time::timeout(Duration::from_secs(5), async {
+                    while !disk.is_online().await {
+                        tokio::task::yield_now().await;
+                    }
+                    disk.read_all("bucket", "object").await
+                })
+                .await;
+                disk.close().await.expect("stop the registered disk's monitoring");
+                peer.stop().await;
+                assert!(!cancelled_after_failure, "failed replacement must retain the registered monitor");
+                assert_eq!(
+                    recovery
+                        .expect("failed reconnect must leave the registered disk able to recover without another sweep")
+                        .expect("the recovered registered disk must serve real I/O"),
+                    Bytes::from_static(b"recovered-peer-data")
+                );
+            },
+        )
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn endpoint_reconnect_stops_old_remote_monitor_only_after_publication() {
+        runtime_sources::ensure_test_rpc_secret();
+        let mut reference = crate::layout::format::FormatV3::new(1, 1);
+        reference.erasure.this = reference.erasure.sets[0][0];
+        let peer = TestGrpcPeer::spawn(
+            Bytes::from(reference.to_json().expect("format should serialize")),
+            Bytes::from_static(b"replacement-peer-data"),
+        )
+        .await
+        .expect("authenticated regression peer is required");
+        let set = missing_read_set(&peer, reference).await;
+        let disk = crate::disk::new_disk(
+            &set.set_endpoints[0],
+            &DiskOption {
+                cleanup: false,
+                health_check: true,
+            },
+        )
+        .await
+        .expect("registered remote disk should construct");
+        let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
+            panic!("fixture must create a remote disk");
+        };
+        remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
+        set.disks.write().await[0] = Some(Arc::clone(&disk));
+        let gate = peer.peer.pause_next_format_read();
+        let pending_set = Arc::clone(&set);
+        let pending = tokio::spawn(async move { pending_set.connect_disks().await });
+        time::timeout(Duration::from_secs(5), gate.entered.notified())
+            .await
+            .expect("replacement must reach the format validation barrier");
+        let cancelled_before_publication = remote.cancel_token.is_cancelled();
+        gate.release.cancel();
+        pending.await.expect("validated replacement must finish publication");
+        let replacement = set.disks.read().await[0].clone().expect("replacement must occupy the slot");
+        let readback = replacement.read_all("bucket", "object").await;
+        replacement.close().await.expect("stop replacement monitoring");
+        peer.stop().await;
+        assert!(
+            !cancelled_before_publication,
+            "the registered monitor must survive pending format validation"
+        );
+        assert!(!Arc::ptr_eq(&disk, &replacement), "validated replacement must be published");
+        assert!(remote.cancel_token.is_cancelled(), "the displaced monitor must stop after publication");
+        assert_eq!(
+            readback.expect("replacement must serve real I/O"),
+            Bytes::from_static(b"replacement-peer-data")
+        );
+    }
+
     #[tokio::test]
     #[serial(remote_disk_recovery_probe)]
     async fn endpoint_reconnect_admits_ready_drive_while_another_format_probe_is_blocked() {
@@ -11543,4 +12019,15 @@ mod tests {
 
         accept_task.abort();
     }
+
+    #[test]
+    fn prewarm_claim_is_deduplicated_and_released_after_failure() {
+        let addr = "http://prewarm-retry-test.invalid:9000";
+        release_control_channel_prewarm(addr);
+        assert!(claim_control_channel_prewarm(addr));
+        assert!(!claim_control_channel_prewarm(addr));
+        release_control_channel_prewarm(addr);
+        assert!(claim_control_channel_prewarm(addr));
+        release_control_channel_prewarm(addr);
+    }
 }

crates/ecstore/src/cluster/rpc/runtime_sources.rs

@@ -481,6 +481,14 @@ pub(crate) fn record_remote_disk_grpc_write_all_request() {
         .record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
 }
 
+pub(crate) fn record_remote_disk_grpc_write_all_duration(duration: Duration) {
+    global_internode_metrics().record_duration_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_WRITE_ALL,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        duration,
+    );
+}
+
 pub(crate) fn record_remote_disk_grpc_write_all_sent_bytes(bytes: usize) {
     global_internode_metrics().record_sent_bytes_for_operation_and_backend(
         INTERNODE_OPERATION_GRPC_WRITE_ALL,
@@ -521,6 +529,14 @@ pub(crate) fn record_remote_disk_grpc_read_all_request() {
         .record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
 }
 
+pub(crate) fn record_remote_disk_grpc_read_all_duration(duration: Duration) {
+    global_internode_metrics().record_duration_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_READ_ALL,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        duration,
+    );
+}
+
 #[cfg(feature = "diagnostic-read-version-caller")]
 fn read_version_request_mode(opts: &ReadOptions) -> ReadVersionRequestMode {
     match (opts.read_data, opts.healing, opts.incl_free_versions) {
@@ -598,6 +614,17 @@ pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static s
     );
 }
 
+pub(crate) fn record_remote_disk_grpc_batch_read_version_duration(duration: Duration) {
+    if !rustfs_io_metrics::get_stage_metrics_enabled() {
+        return;
+    }
+    global_internode_metrics().record_duration_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        duration,
+    );
+}
+
 pub(crate) fn record_remote_disk_grpc_batch_read_version_error() {
     if !rustfs_io_metrics::get_stage_metrics_enabled() {
         return;
@@ -690,6 +717,14 @@ pub(crate) fn record_remote_disk_grpc_read_multiple_recv_bytes(bytes: usize) {
     record_grpc_payload_size(INTERNODE_OPERATION_GRPC_READ_MULTIPLE, bytes);
 }
 
+pub(crate) fn record_remote_disk_grpc_read_multiple_duration(duration: Duration) {
+    global_internode_metrics().record_duration_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        duration,
+    );
+}
+
 /// Payload-size threshold (bytes) above which a unary internode gRPC response is counted as a
 /// "large payload" for alerting. Env-overridable via `RUSTFS_INTERNODE_RPC_LARGE_PAYLOAD_WARN_BYTES`.
 fn internode_rpc_large_payload_warn_bytes() -> usize {

crates/ecstore/src/config/com.rs

@@ -3506,6 +3506,7 @@ mod tests {
                 replication_decision: String::new(),
                 checksum: None,
                 multipart_completion_replayed: false,
+                backfilled_old_current_size: None,
             }
         }
     }

crates/ecstore/src/core/pool_meta_native_integration_tests.rs

@@ -18,6 +18,7 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
 use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
 use std::path::PathBuf;
 use tokio::fs;
+use uuid::Uuid;
 
 struct PhysicalFixture {
     _directories: Vec<tempfile::TempDir>,
@@ -216,6 +217,264 @@ async fn pool_meta_native_physical_split_without_ordinary_commit_performs_no_mut
         "all current heads, identities and complete backups must remain unchanged"
     );
     assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
+    state
+        .ensure_write_safe("ordinary authority temporarily unavailable")
+        .expect("missing ordinary authority alone must not install a permanent corruption latch");
+}
+
+async fn interrupted_first_commit_fixture() -> (PhysicalFixture, PoolMetaCommittedCandidate) {
+    let fixture = fixture(3).await;
+    let PoolMetaReplica::Valid {
+        previous: Some(previous),
+        committed: false,
+        ..
+    } = decode_pool_meta_replica(fixture.pending.clone())
+    else {
+        panic!("fixture must retain its acknowledged predecessor");
+    };
+    let physical = disks(&fixture.store).await;
+    let prepared = replicas(&fixture.pending, OffsetDateTime::now_utc()).await;
+    for (index, disk) in physical[0].iter().enumerate() {
+        fs::write(object_path(disk).join("xl.meta"), &prepared[index])
+            .await
+            .expect("one pool must retain an ordinary-readable PREPARE");
+    }
+    (fixture, *previous)
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_interrupted_first_commit_restores_proven_predecessor() {
+    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
+    let before = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+    assert!(
+        before[1..]
+            .iter()
+            .all(|read| matches!(read.replica, PoolMetaReplica::Unreadable(_)))
+    );
+    assert!(matches!(before[0].replica, PoolMetaReplica::Valid { committed: false, .. }));
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    assert!(
+        repair(&fixture.store, &mut state, &fence)
+            .await
+            .expect("ordinary PREPARE must authorize restoring only its acknowledged predecessor")
+    );
+    state
+        .ensure_write_safe("first commit rollback")
+        .expect("rollback must finish its transaction");
+    assert_ne!(
+        previous.canonical, fixture.canonical,
+        "the interrupted COMMIT must not become authoritative"
+    );
+    fixture.canonical = previous.canonical;
+    fixture.revision = previous.revision;
+    assert_committed(&fixture).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_interrupted_rollback_retains_authority_and_resumes() {
+    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
+    let pause = Arc::new(NativeApplyPause::default());
+    let reached = pause.reached.notified();
+    tokio::pin!(reached);
+    reached.as_mut().enable();
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    let mut state = clean_state(&fixture);
+    let recovery = tokio::spawn({
+        let store = Arc::clone(&fixture.store);
+        let pause = Arc::clone(&pause);
+        async move {
+            AFTER_NATIVE_POOL_APPLY
+                .scope(pause, async move { repair(&store, &mut state, &fence).await })
+                .await
+        }
+    });
+    tokio::time::timeout(std::time::Duration::from_secs(10), reached)
+        .await
+        .expect("real rollback should pause after one pool is durably restored");
+    let partial = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+    assert!(matches!(partial[0].replica, PoolMetaReplica::Valid { committed: false, .. }));
+    assert!(matches!(&partial[1].replica, PoolMetaReplica::Valid { canonical, committed: true, .. }
+        if canonical == &previous.canonical));
+    assert!(matches!(partial[2].replica, PoolMetaReplica::Unreadable(_)));
+    recovery.abort();
+    assert!(
+        recovery
+            .await
+            .expect_err("the paused repair must be cancelled")
+            .is_cancelled()
+    );
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    assert!(
+        repair(&fixture.store, &mut state, &fence)
+            .await
+            .expect("restart must resume from the retained ordinary PREPARE")
+    );
+    fixture.canonical = previous.canonical;
+    fixture.revision = previous.revision;
+    assert_committed(&fixture).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_rollback_resumes_after_last_prepare_loses_read_quorum() {
+    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    let inspected = inspect(&fixture.store.pools, &fence)
+        .await
+        .expect("physical rollback inventory");
+    let replacements = plan(
+        &inspected,
+        &fixture.canonical,
+        fixture.revision,
+        &fixture.identity,
+        Some(&previous.canonical),
+    )
+    .await
+    .expect("the ordinary PREPARE must authorize its predecessor");
+    for (pool, count) in [(1, 4), (2, 4), (0, 2)] {
+        for (index, replacement) in replacements[pool].iter().enumerate().take(count) {
+            let snapshot = &inspected[pool].snapshots[index];
+            assert_eq!(
+                inspected[pool].disks[index]
+                    .pool_meta_native_recovery(Request::Apply {
+                        object: PoolMetaNativeObject::PoolBin,
+                        expected_disk_id: snapshot.disk_id,
+                        expected_current_sha256: snapshot.current.as_ref().map(|raw| Sha256::digest(raw).into()),
+                        expected_backup_inventory_sha256: snapshot.inventory_sha256().expect("original inventory hash"),
+                        replacement: replacement.clone(),
+                    })
+                    .await
+                    .expect("partial rollback must use the real conditional publication and sync path"),
+                Response::Applied
+            );
+        }
+    }
+    let partial = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+    assert!(matches!(partial[0].replica, PoolMetaReplica::Unreadable(_)));
+    assert!(partial[1..].iter().all(|read| matches!(&read.replica,
+        PoolMetaReplica::Valid { canonical, committed: true, .. } if canonical == &previous.canonical)));
+    let mut state = clean_state(&fixture);
+    assert!(
+        repair(&fixture.store, &mut state, &fence)
+            .await
+            .expect("restored ordinary COMMIT must finish rollback after the last PREPARE becomes split")
+    );
+    fixture.canonical = previous.canonical;
+    fixture.revision = previous.revision;
+    assert_committed(&fixture).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_rollback_backup_does_not_poison_reused_generation() {
+    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    assert!(repair(&fixture.store, &mut state, &fence).await.expect("initial rollback"));
+    let revision = PoolMetaRevision {
+        transaction_id: Some(Uuid::new_v4()),
+        ..fixture.revision
+    };
+    let mut meta = previous.meta.clone();
+    meta.pools[0].last_update += Duration::seconds(1);
+    let canonical = encode_pool_meta_v3_envelope(&meta, revision, true, None).expect("later COMMIT");
+    let pending = encode_pool_meta_v3_envelope(&meta, revision, false, Some(&previous)).expect("later PREPARE");
+    let modified = OffsetDateTime::now_utc() + Duration::seconds(1);
+    let split = split(&pending, &canonical, modified).await;
+    let commit = replicas(&canonical, modified).await;
+    let prepared = replicas(&pending, modified).await;
+    let directory = Uuid::new_v4();
+    let retained = snapshots(&fixture.store).await;
+    for (pool_index, pool) in disks(&fixture.store).await.into_iter().enumerate() {
+        for (disk_index, disk) in pool.iter().enumerate() {
+            assert!(
+                !retained[pool_index][disk_index].backups.is_empty(),
+                "the abandoned PREPARE must remain as real on-disk evidence"
+            );
+            let object = object_path(disk);
+            fs::write(object.join("xl.meta"), &if pool_index < 2 { &split } else { &commit }[disk_index])
+                .await
+                .expect("install later partial COMMIT without clearing old backups");
+            fs::create_dir(object.join(directory.to_string()))
+                .await
+                .expect("later backup directory");
+            fs::write(object.join(directory.to_string()).join("xl.meta.bkp"), &prepared[disk_index])
+                .await
+                .expect("later PREPARE backup");
+        }
+    }
+    assert!(
+        repair(&fixture.store, &mut state, &fence)
+            .await
+            .expect("an ordinary later COMMIT must survive the abandoned sibling PREPARE")
+    );
+    fixture.canonical = canonical;
+    fixture.revision = revision;
+    assert_committed(&fixture).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_rollback_rejects_retained_commit_without_mutation() {
+    let (fixture, _) = interrupted_first_commit_fixture().await;
+    let physical = disks(&fixture.store).await;
+    let commit = replicas(&fixture.canonical, OffsetDateTime::now_utc()).await;
+    let directory = Uuid::new_v4();
+    let object = object_path(&physical[0][0]);
+    fs::create_dir(object.join(directory.to_string()))
+        .await
+        .expect("retained COMMIT directory");
+    fs::write(object.join(directory.to_string()).join("xl.meta.bkp"), &commit[0])
+        .await
+        .expect("retain evidence that the interrupted transaction may already have committed");
+    let before = snapshots(&fixture.store).await;
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    assert!(repair(&fixture.store, &mut state, &fence).await.is_err());
+    assert_eq!(snapshots(&fixture.store).await, before, "ambiguous COMMIT evidence must remain untouched");
+    assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_rollback_rejects_competing_prepares_without_mutation() {
+    let (fixture, previous) = interrupted_first_commit_fixture().await;
+    let pending = encode_pool_meta_v3_envelope(
+        &previous.meta,
+        PoolMetaRevision {
+            transaction_id: Some(Uuid::new_v4()),
+            ..fixture.revision
+        },
+        false,
+        Some(&previous),
+    )
+    .expect("competing PREPARE must remain structurally valid");
+    let physical = disks(&fixture.store).await;
+    let prepared = replicas(&pending, OffsetDateTime::now_utc()).await;
+    for (index, disk) in physical[1].iter().enumerate() {
+        fs::write(object_path(disk).join("xl.meta"), &prepared[index])
+            .await
+            .expect("second pool must expose the competing transaction through ordinary reads");
+    }
+    let before = snapshots(&fixture.store).await;
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    let error = repair(&fixture.store, &mut state, &fence)
+        .await
+        .expect_err("two distinct prepared transactions must not authorize a guessed rollback");
+    assert_eq!(
+        error
+            .pool_metadata_failure()
+            .expect("transaction conflict must be typed")
+            .kind,
+        crate::error::PoolMetadataFailure::RecoveryRequired
+    );
+    assert_eq!(snapshots(&fixture.store).await, before, "competing transactions must remain untouched");
+    assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
 }
 
 struct IdentityEpochFork {
@@ -467,7 +726,7 @@ async fn pool_meta_native_physical_partial_apply_resumes_after_conditional_rejec
     let inspected = inspect(&fixture.store.pools, &fence)
         .await
         .expect("real native snapshots should be obtained");
-    let replacements = plan(&inspected, &fixture.canonical, fixture.revision, &fixture.identity)
+    let replacements = plan(&inspected, &fixture.canonical, fixture.revision, &fixture.identity, None)
         .await
         .expect("physical recovery plan should validate");
     let snapshot = &inspected[0].snapshots[0];
@@ -628,3 +887,559 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
         );
     }
 }
+
+async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
+    let fixture = fixture(0).await;
+    for pool in disks(&fixture.store).await {
+        for disk in pool {
+            let object = object_path(&disk);
+            let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
+            while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
+                if entry.file_type().await.expect("fixture entry type").is_dir() {
+                    fs::remove_dir_all(entry.path())
+                        .await
+                        .expect("remove task-owned fixture rollback directory");
+                }
+            }
+        }
+    }
+    fixture
+}
+
+async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
+    let mut all = Vec::new();
+    for pool in &store.pools {
+        let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
+        let members = set
+            .disks
+            .read()
+            .await
+            .iter()
+            .cloned()
+            .map(|disk| disk.expect("identity fixture disk must be online"))
+            .collect::<Vec<_>>();
+        let mut heads = Vec::new();
+        for disk in members {
+            match disk
+                .pool_meta_native_recovery(Request::InspectIdentity {})
+                .await
+                .expect("physical identity inspection")
+            {
+                Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
+                _ => panic!("identity inspection must return its physical head"),
+            }
+        }
+        all.push(heads);
+    }
+    all
+}
+
+async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
+    let fixture = owner_repair_no_backups_fixture().await;
+    let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
+        PoolMetaReplica::Valid {
+            meta, committed: true, ..
+        } => meta,
+        _ => panic!("fixture must provide valid committed pool metadata"),
+    };
+    meta.version = version;
+    let raw = meta
+        .encode_config_data_for_v2_gate(version == POOL_META_VERSION)
+        .expect("legacy fixture should encode in its original version");
+    assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
+    let all = disks(&fixture.store).await;
+    let now = OffsetDateTime::now_utc();
+    for pool in &all {
+        let heads = replicas(&raw, now).await;
+        for (index, disk) in pool.iter().enumerate() {
+            fs::write(object_path(disk).join("xl.meta"), &heads[index])
+                .await
+                .expect("install exact legacy head");
+        }
+    }
+    if stale {
+        let old = replicas(&raw, now - Duration::seconds(10)).await;
+        fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
+            .await
+            .expect("install stale same-raw legacy head");
+    }
+    (fixture, raw)
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
+            )
+            .await
+            .expect("legacy native heal must terminate");
+            assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
+            drop(state);
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            for read in &reads {
+                assert!(
+                    matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
+                    if bytes == &raw && revision.version == POOL_META_V1_VERSION),
+                    "native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
+                );
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
+                let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
+                let mut source_raw = raw.clone();
+                // A second MessagePack value is readable by legacy decoding but
+                // must not be copied as part of an unverified repair payload.
+                source_raw.push(0xc0);
+                assert!(
+                    matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
+                    "ordinary legacy reads must remain compatible with trailing bytes"
+                );
+                let all_disks = disks(&fixture.store).await;
+                let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+                for (index, disk) in all_disks[0].iter().enumerate() {
+                    fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
+                        .await
+                        .expect("install readable legacy source with trailing bytes");
+                }
+                let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
+                    .await
+                    .expect("legacy semantic selection must remain readable");
+                assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
+                let before = snapshots(&fixture.store).await;
+                let identities = owner_repair_identity_heads(&fixture.store).await;
+                let guard = owner(&fixture.store).await;
+                let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                let result = tokio::time::timeout(
+                    std::time::Duration::from_secs(30),
+                    fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+                )
+                .await
+                .expect("legacy native heal must terminate");
+                drop(state);
+                assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
+                assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
+                assert_eq!(
+                    owner_repair_identity_heads(&fixture.store).await,
+                    identities,
+                    "version {version}: no identity shard may change"
+                );
+                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
+                assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
+                for target_pool in [0, 1] {
+                    let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+                    let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
+                        PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
+                        _ => panic!("initial V1 metadata must decode"),
+                    };
+                    meta.version = POOL_META_VERSION;
+                    let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
+                    assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
+                    assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
+                    let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
+                        PoolMetaReplica::Valid { canonical, .. } => canonical,
+                        _ => panic!("V1 payload must remain valid"),
+                    };
+                    let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
+                        PoolMetaReplica::Valid { canonical, .. } => canonical,
+                        _ => panic!("V2 payload must remain valid"),
+                    };
+                    assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
+                    let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
+                    // Pool two follows pool zero; pools zero and one cover both
+                    // orderings and both target formats in separate runs.
+                    let versions = [first_version, other_version, first_version];
+                    let physical = disks(&fixture.store).await;
+                    let timestamp = OffsetDateTime::now_utc();
+                    for (pool_index, pool) in physical.iter().enumerate() {
+                        let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
+                        let heads = replicas(raw, timestamp).await;
+                        for (index, disk) in pool.iter().enumerate() {
+                            fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
+                        }
+                    }
+                    let before = snapshots(&fixture.store).await;
+                    let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
+                        .await.expect("equivalent legacy pools should be selectable");
+                    assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
+                    assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
+                    let guard = owner(&fixture.store).await;
+                    let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                    tokio::time::timeout(std::time::Duration::from_secs(30),
+                        fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
+                        .await.expect("mixed legacy heal must terminate")
+                        .expect("equivalent legacy metadata should heal without downgrading the format floor");
+                    drop(state);
+                    let after = snapshots(&fixture.store).await;
+                    let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                    for (pool_index, read) in reads.iter().enumerate() {
+                        let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
+                        let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
+                        assert!(matches!(&read.replica,
+                            PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
+                                if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
+                        ), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
+                        for (index, snapshot) in after[pool_index].iter().enumerate() {
+                            codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
+                                .await.expect("every physical shard must match its selected legacy payload");
+                        }
+                        if pool_index != target_pool {
+                            assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
+                        }
+                    }
+                }
+            }
+        },
+    ).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for prepare_pool in [0, 1] {
+                let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+                let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
+                    PoolMetaReplica::Valid {
+                        meta,
+                        canonical,
+                        revision,
+                        committed: true,
+                        ..
+                    } if revision.version == POOL_META_V1_VERSION => (meta, canonical),
+                    _ => panic!("baseline V1 metadata must be committed and readable"),
+                };
+                let previous = PoolMetaCommittedCandidate {
+                    canonical: legacy_canonical,
+                    meta: legacy_meta.clone(),
+                    revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
+                };
+                let mut next = legacy_meta;
+                next.version = POOL_META_GENERATION_VERSION;
+                next.pools[0].last_update += Duration::seconds(1);
+                let revision = PoolMetaRevision {
+                    version: POOL_META_GENERATION_VERSION,
+                    cluster_id: fixture.revision.cluster_id,
+                    epoch: fixture.revision.epoch,
+                    generation: 1,
+                    transaction_id: Some(Uuid::new_v4()),
+                };
+                let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
+                    .expect("V3 PREPARE with V1 predecessor must encode");
+                assert!(matches!(decode_pool_meta_replica(pending.clone()),
+                    PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
+                let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+                let physical = disks(&fixture.store).await;
+                for (index, disk) in physical[prepare_pool].iter().enumerate() {
+                    fs::write(object_path(disk).join("xl.meta"), &heads[index])
+                        .await
+                        .expect("install task-owned V3 PREPARE evidence");
+                }
+                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                assert!(matches!(&reads[prepare_pool].replica,
+                    PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
+                        if raw == &pending && *actual == revision));
+                let other_pool = 1 - prepare_pool;
+                assert!(matches!(&reads[other_pool].replica,
+                    PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
+                        if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
+                let before = snapshots(&fixture.store).await;
+                let identities = owner_repair_identity_heads(&fixture.store).await;
+                let guard = owner(&fixture.store).await;
+                let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                let result = tokio::time::timeout(
+                    std::time::Duration::from_secs(30),
+                    fixture
+                        .store
+                        .heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
+                )
+                .await
+                .expect("mixed V1/V3 PREPARE heal must terminate");
+                drop(state);
+                assert!(
+                    result.is_err(),
+                    "pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
+                );
+                assert_eq!(
+                    snapshots(&fixture.store).await,
+                    before,
+                    "pool {prepare_pool}: rejection must retain all physical head and backup evidence"
+                );
+                assert_eq!(
+                    owner_repair_identity_heads(&fixture.store).await,
+                    identities,
+                    "pool {prepare_pool}: rejection must not mutate identity evidence"
+                );
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    meta, committed: true, ..
+                } => meta,
+                _ => panic!("baseline V1 metadata must decode"),
+            };
+            let statuses = meta
+                .pools
+                .iter()
+                .map(|pool| {
+                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
+                    LegacyPoolStatus {
+                        id: pool.id,
+                        cmd_line: pool.cmd_line.clone(),
+                        last_update: pool.last_update,
+                        decommission: None,
+                    }
+                })
+                .collect();
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: statuses,
+                dont_save: true,
+            };
+            let mut dangerous_raw = Vec::new();
+            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
+            let old_reader: LegacyPoolMeta =
+                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
+            assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
+            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
+            let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
+                PoolMetaReplica::Valid {
+                    canonical,
+                    revision,
+                    committed: true,
+                    ..
+                } => {
+                    assert_eq!(revision.version, POOL_META_V1_VERSION);
+                    canonical
+                }
+                _ => panic!("current reader must recognize the legacy source"),
+            };
+            assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
+                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
+                    if canonical == source && revision.version == POOL_META_V1_VERSION));
+            let physical = disks(&fixture.store).await;
+            let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+            for (index, disk) in physical[0].iter().enumerate() {
+                fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
+                    .await
+                    .expect("install legacy source on pool zero");
+            }
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
+            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
+            let before = snapshots(&fixture.store).await;
+            let identities = owner_repair_identity_heads(&fixture.store).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+            )
+            .await
+            .expect("unsafe legacy heal must terminate");
+            drop(state);
+            let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
+            assert!(
+                error.to_string().contains("dont_save"),
+                "rejection must identify the unsafe legacy field: {error}"
+            );
+            assert_eq!(
+                snapshots(&fixture.store).await,
+                before,
+                "rejection must preserve every original head and backup"
+            );
+            assert_eq!(
+                owner_repair_identity_heads(&fixture.store).await,
+                identities,
+                "rejection must not write identity shards"
+            );
+            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
+            assert!(
+                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
+                "the dangerous source bytes must not be copied into the target pool"
+            );
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    meta, committed: true, ..
+                } => meta,
+                _ => panic!("baseline V1 metadata must decode"),
+            };
+            let statuses = meta
+                .pools
+                .iter()
+                .map(|pool| {
+                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
+                    LegacyPoolStatus {
+                        id: pool.id,
+                        cmd_line: pool.cmd_line.clone(),
+                        last_update: pool.last_update,
+                        decommission: None,
+                    }
+                })
+                .collect();
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: statuses,
+                dont_save: true,
+            };
+            let mut dangerous_raw = Vec::new();
+            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
+            let old_reader: LegacyPoolMeta =
+                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
+            assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
+            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
+            let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    canonical,
+                    revision,
+                    committed: true,
+                    ..
+                } => {
+                    assert_eq!(revision.version, POOL_META_V1_VERSION);
+                    canonical
+                }
+                _ => panic!("safe pool-zero source must decode"),
+            };
+            assert!(
+                matches!(decode_pool_meta_replica(dangerous_raw.clone()),
+                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
+                    if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
+                "the dangerous target must remain semantically equivalent to the safe source"
+            );
+            let physical = disks(&fixture.store).await;
+            let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+            for (index, disk) in physical[1].iter().enumerate() {
+                fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
+                    .await
+                    .expect("install legacy dangerous target head");
+            }
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
+            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
+            let before = snapshots(&fixture.store).await;
+            let identities = owner_repair_identity_heads(&fixture.store).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+            )
+            .await
+            .expect("unsafe legacy target heal must terminate");
+            drop(state);
+            let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
+            assert!(
+                error.to_string().contains("dont_save"),
+                "rejection must identify the unsafe legacy field: {error}"
+            );
+            assert_eq!(
+                snapshots(&fixture.store).await,
+                before,
+                "rejection must preserve all physical heads and backups"
+            );
+            assert_eq!(
+                owner_repair_identity_heads(&fixture.store).await,
+                identities,
+                "rejection must not write identity shards"
+            );
+            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
+            assert!(
+                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
+                "unsafe target evidence must not be overwritten by the safe source"
+            );
+        },
+    )
+    .await;
+}

crates/ecstore/src/core/pool_meta_native_recovery.rs

@@ -30,6 +30,7 @@ pub(super) struct NativeApplyPause {
 #[cfg(test)]
 tokio::task_local! {
     pub(super) static AFTER_NATIVE_APPLY: Arc<NativeApplyPause>;
+    pub(super) static AFTER_NATIVE_POOL_APPLY: Arc<NativeApplyPause>;
 }
 
 struct InspectedPool {
@@ -168,6 +169,7 @@ async fn plan(
     canonical: &[u8],
     revision: PoolMetaRevision,
     identity: &[u8],
+    rollback: Option<&[u8]>,
 ) -> Result<Vec<Vec<Bytes>>> {
     let mut pending = Vec::<Vec<u8>>::new();
     let mut predecessors = Vec::<Vec<u8>>::new();
@@ -223,23 +225,83 @@ async fn plan(
             }
         }
     }
-    let mut obsolete = Vec::<Vec<u8>>::new();
-    for payload in &decoded {
-        match decode_pool_meta_replica(payload.clone()) {
-            PoolMetaReplica::Valid {
+    // The last ordinary PREPARE can itself become split during rollback. In
+    // that case another pool already exposes the restored ordinary COMMIT.
+    // Its authority permits continuing the rollback, never promoting a child
+    // transaction reconstructed from disk evidence.
+    let mut continuation: Option<(Vec<u8>, PoolMetaRevision)> = None;
+    if rollback.is_none() {
+        let mut outstanding = Vec::new();
+        for pool in inspected {
+            for (index, snapshot) in pool.snapshots.iter().enumerate() {
+                let raw = snapshot.current.as_ref().ok_or_else(invalid)?;
+                if codec::validate_inline_pool_meta_shard(raw, index, canonical).await.is_err() {
+                    outstanding.push((index, raw));
+                }
+            }
+        }
+        for payload in &decoded {
+            if let PoolMetaReplica::Valid {
                 canonical: candidate,
                 revision: candidate_revision,
                 committed: false,
                 previous: Some(previous),
                 ..
-            } if candidate_revision == revision && candidate == canonical => {
-                if !pending.contains(payload) {
-                    pending.push(payload.clone());
+            } = decode_pool_meta_replica(payload.clone())
+                && previous.revision == revision
+                && previous.canonical == canonical
+            {
+                let mut matches = true;
+                for (index, raw) in &outstanding {
+                    matches &= codec::validate_inline_pool_meta_shard(raw, *index, payload).await.is_ok()
+                        || codec::validate_inline_pool_meta_shard(raw, *index, &candidate).await.is_ok();
                 }
-                if !predecessors.contains(&previous.canonical) {
-                    predecessors.push(previous.canonical);
+                if matches {
+                    if !outstanding.is_empty()
+                        && continuation.as_ref().is_some_and(|(selected, selected_revision)| {
+                            selected != &candidate || *selected_revision != candidate_revision
+                        })
+                    {
+                        return Err(known_validation(invalid(), "native_transaction_validation"));
+                    }
+                    continuation.get_or_insert((candidate, candidate_revision));
                 }
             }
+        }
+    }
+    let (canonical, revision, rollback) = match &continuation {
+        Some((transaction, next_revision)) => (transaction.as_slice(), *next_revision, Some(canonical)),
+        None => (canonical, revision, rollback),
+    };
+    for payload in &decoded {
+        if let PoolMetaReplica::Valid {
+            canonical: candidate,
+            revision: candidate_revision,
+            committed: false,
+            previous: Some(previous),
+            ..
+        } = decode_pool_meta_replica(payload.clone())
+            && candidate_revision == revision
+            && candidate == canonical
+        {
+            pending.push(payload.clone());
+            if !predecessors.contains(&previous.canonical) {
+                predecessors.push(previous.canonical);
+            }
+        }
+    }
+    if pending.len() > 1 {
+        return Err(known_validation(
+            Error::other("native V3 prepare records disagree on the predecessor"),
+            "native_transaction_validation",
+        ));
+    }
+    let mut obsolete = Vec::<Vec<u8>>::new();
+    for payload in &decoded {
+        if pending.contains(payload) {
+            continue;
+        }
+        match decode_pool_meta_replica(payload.clone()) {
             PoolMetaReplica::Valid { revision: older, .. }
                 if older.is_generation_protocol()
                     && older.cluster_id == revision.cluster_id
@@ -251,6 +313,23 @@ async fn plan(
                 // fully reconstructed and bound into the native CAS.
                 obsolete.push(payload.clone());
             }
+            PoolMetaReplica::Valid {
+                revision: abandoned,
+                committed: false,
+                previous: Some(previous),
+                ..
+            } if abandoned.is_generation_protocol()
+                && abandoned.cluster_id == revision.cluster_id
+                && abandoned.epoch == revision.epoch
+                && abandoned.generation == revision.generation
+                && abandoned.transaction_id != revision.transaction_id
+                && predecessors.contains(&previous.canonical) =>
+            {
+                // A rollback retains the failed PREPARE as evidence. A later
+                // transaction may reuse that generation, but only a backup of
+                // an uncommitted sibling with the same predecessor is obsolete.
+                obsolete.push(payload.clone());
+            }
             PoolMetaReplica::Valid {
                 canonical: candidate,
                 revision: candidate_revision,
@@ -272,13 +351,7 @@ async fn plan(
             _ => {}
         }
     }
-    if pending.len() > 1 {
-        return Err(known_validation(
-            Error::other("native V3 prepare records disagree on the predecessor"),
-            "native_transaction_validation",
-        ));
-    }
-    if pending.is_empty() {
+    if pending.is_empty() || rollback.is_some_and(|previous| !predecessors.iter().any(|candidate| candidate == previous)) {
         return Err(invalid());
     }
     for payload in &decoded {
@@ -294,13 +367,25 @@ async fn plan(
         for (index, snapshot) in pool.snapshots.iter().enumerate() {
             let raw = snapshot.current.as_ref().ok_or_else(invalid)?;
             let mut matched = false;
-            for payload in std::iter::once(canonical).chain(pending.iter().map(Vec::as_slice)) {
+            for payload in std::iter::once(canonical)
+                .chain(pending.iter().map(Vec::as_slice))
+                .chain(rollback)
+            {
                 matched |= codec::validate_inline_pool_meta_shard(raw, index, payload).await.is_ok();
             }
             if !matched {
                 return Err(invalid());
             }
             for backup in &snapshot.backups {
+                if rollback.is_some()
+                    && codec::validate_inline_pool_meta_shard(&backup.data, index, canonical)
+                        .await
+                        .is_ok()
+                {
+                    // A retained COMMIT might have been authoritative before
+                    // this interruption. Never roll it back from PREPARE alone.
+                    return Err(invalid());
+                }
                 let mut matched = false;
                 for payload in std::iter::once(canonical).chain(
                     pending
@@ -323,7 +408,7 @@ async fn plan(
     let mut replacements = Vec::with_capacity(inspected.len());
     for pool in inspected {
         replacements.push(
-            codec::encode_inline_pool_meta_replicas(&current(pool)?, canonical, modified)
+            codec::encode_inline_pool_meta_replicas(&current(pool)?, rollback.unwrap_or(canonical), modified)
                 .await
                 .map_err(|error| codec_validation(error, "native_metadata_validation"))?,
         );
@@ -338,10 +423,12 @@ pub(super) async fn repair(
 ) -> Result<bool> {
     fence.ensure_held()?;
     let reads = read_pool_meta_replicas(store.pools.clone(), true).await;
-    if !reads
+    let unreadable = reads
         .iter()
-        .any(|read| matches!(read.replica, PoolMetaReplica::Unreadable(_)))
-    {
+        .enumerate()
+        .filter_map(|(index, read)| matches!(read.replica, PoolMetaReplica::Unreadable(_)).then_some(index))
+        .collect::<Vec<_>>();
+    if unreadable.is_empty() {
         return Ok(false);
     }
     state.ensure_write_safe("native pool metadata recovery")?;
@@ -360,8 +447,9 @@ pub(super) async fn repair(
     {
         return Err(invalid());
     }
-    // Only an ordinary, successfully read COMMIT grants authority. Rebuilding
-    // two erasure shards below proves integrity, never a commit decision.
+    // Ordinary reads grant authority: a COMMIT selects its own state, while a
+    // PREPARE can select only its acknowledged predecessor. Reconstruction
+    // below proves integrity and never promotes an interrupted transaction.
     let ordinary_commits = reads
         .iter()
         .filter_map(|read| match &read.replica {
@@ -375,7 +463,22 @@ pub(super) async fn repair(
             _ => None,
         })
         .collect::<Vec<_>>();
-    if ordinary_commits.is_empty() {
+    let ordinary_prepares = reads
+        .iter()
+        .filter_map(|read| match &read.replica {
+            PoolMetaReplica::Valid {
+                canonical,
+                revision,
+                committed: false,
+                previous: Some(previous),
+                ..
+            } if revision.is_generation_protocol() && previous.revision.is_generation_protocol() => {
+                Some((canonical.clone(), *revision, previous.canonical.clone()))
+            }
+            _ => None,
+        })
+        .collect::<Vec<_>>();
+    if ordinary_commits.is_empty() && ordinary_prepares.is_empty() {
         return Err(invalid());
     }
     let selection = select_pool_meta_replica_reads(reads)
@@ -395,10 +498,20 @@ pub(super) async fn repair(
         return Err(invalid());
     }
     let canonical = selection.canonical.as_ref().ok_or_else(invalid)?;
+    let mut rollback = None;
+    for (pending, revision, previous) in &ordinary_prepares {
+        if previous == canonical {
+            if rollback.is_some_and(|candidate| candidate != (pending, *revision)) {
+                return Err(latch_validation(state, known_validation(invalid(), "native_transaction_validation")));
+            }
+            rollback = Some((pending, *revision));
+        }
+    }
     if !selection.revision.is_generation_protocol()
-        || !ordinary_commits
-            .iter()
-            .any(|(raw, revision)| raw == canonical && *revision == selection.revision)
+        || (rollback.is_none()
+            && !ordinary_commits
+                .iter()
+                .any(|(raw, revision)| raw == canonical && *revision == selection.revision))
     {
         return Err(invalid());
     }
@@ -445,13 +558,18 @@ pub(super) async fn repair(
         .await
         .map_err(|error| latch_validation(state, error))?;
     let inspected = inspect(&store.pools, fence).await?;
-    let replacements = plan(&inspected, canonical, selection.revision, &identity_bytes)
+    let (transaction, revision) = rollback.unwrap_or((canonical, selection.revision));
+    let replacements = plan(&inspected, transaction, revision, &identity_bytes, rollback.map(|_| canonical.as_slice()))
         .await
         .map_err(|error| latch_validation(state, error))?;
     fence.ensure_held()?;
     let mut arm = state.arm_transaction();
     arm.phase = Some("native_repair_apply");
-    for (pool, replacements) in inspected.iter().zip(replacements) {
+    let mut apply_order = inspected.iter().zip(replacements).enumerate().collect::<Vec<_>>();
+    // Preserve ordinary authority until all unreadable pools are repaired.
+    // Cancellation can then resume from a PREPARE or a restored COMMIT.
+    apply_order.sort_by_key(|(index, _)| !unreadable.contains(index));
+    for (_, (pool, replacements)) in apply_order {
         for ((disk, snapshot), replacement) in pool.disks.iter().zip(&pool.snapshots).zip(replacements) {
             fence.ensure_held()?;
             let request = Request::Apply {
@@ -466,6 +584,11 @@ pub(super) async fn repair(
                 _ => return Err(invalid()),
             }
         }
+        #[cfg(test)]
+        if let Ok(pause) = AFTER_NATIVE_POOL_APPLY.try_with(Arc::clone) {
+            pause.reached.notify_one();
+            pause.resume.notified().await;
+        }
     }
     #[cfg(test)]
     if let Ok(pause) = AFTER_NATIVE_APPLY.try_with(Arc::clone) {
@@ -627,7 +750,7 @@ mod tests {
     #[tokio::test]
     async fn pool_meta_native_plan_restores_physical_split_with_fresh_common_identity() {
         let f = fixture().await;
-        let repaired = plan(&f.pools, &f.commit, f.revision, &f.identity).await.unwrap();
+        let repaired = plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.unwrap();
         assert_eq!(repaired.len(), 4);
         for pool in repaired {
             assert_eq!(
@@ -648,7 +771,7 @@ mod tests {
         let mut f = fixture().await;
         let wrong = replicas(b"different identity", f.modified).await;
         f.pools[0].snapshots[0].identity = Some(wrong[0].clone());
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
     }
 
     #[tokio::test]
@@ -671,10 +794,10 @@ mod tests {
         .unwrap();
         let wrong = replicas(&higher, f.modified + Duration::seconds(2)).await;
         f.pools[0].snapshots[0].current = Some(wrong[0].clone());
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
         f.pools[0].snapshots[0].current = Some(replicas(&f.pending, f.modified).await[0].clone());
         f.pools[0].snapshots[0].backups[0].data = wrong[0].clone();
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
     }
 
     #[tokio::test]
@@ -717,17 +840,17 @@ mod tests {
             }
         }
         assert!(older_pending != f.pending);
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_ok());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_ok());
     }
 
     #[tokio::test]
     async fn pool_meta_native_plan_rejects_missing_or_corrupt_evidence() {
         let mut f = fixture().await;
         let before = f.pools[1].snapshots[2].current.take();
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
         f.pools[1].snapshots[2].current = before;
         f.pools[1].snapshots[2].backups[0].data = Bytes::from_static(b"invalid metadata");
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
     }
 }
 

crates/ecstore/src/core/pool_meta_recovery_codec.rs

@@ -315,7 +315,7 @@ pub(super) async fn encode_inline_pool_meta_replicas(
         .encode_with_shard_integrity(
             Cursor::new(Bytes::copy_from_slice(canonical)),
             &mut writers,
-            raw.len(),
+            crate::erasure::coding::encode::ShardWritePolicy::object(raw.len()),
             1,
             IntegrityEncodeMode::Inline(canonical.len()),
             protected,

crates/ecstore/src/core/pools_test.rs

@@ -1616,7 +1616,7 @@ mod decommission_lock_order_tests {
             run_routine_to_a_deferral_pause(
                 &store,
                 1,
-                "target pool 1 no longer fits the selected mutation: source pool 0 reservation estimate is exhausted",
+                "target pool 1 no longer fits the selected mutation: target pool 1 reservation requires",
             )
             .await;
         });
@@ -1647,7 +1647,7 @@ mod decommission_lock_order_tests {
             run_routine_to_a_deferral_pause(
                 &store,
                 1,
-                "target pool 1 no longer fits the selected mutation: source pool 0 reservation estimate is exhausted",
+                "target pool 1 no longer fits the selected mutation: target pool 1 reservation requires",
             )
             .await;
         });
@@ -6062,6 +6062,12 @@ mod decommission_lock_order_tests {
             )
             .await
             .expect("read the next source object for the multipart capacity oracle");
+        // Reconciled bytes remain charged. With no physical room for a new
+        // reservation, a distinct mutation must fail instead of reusing them.
+        let mut exhausted_capacity = capacity_snapshot();
+        exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total);
+        set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
+
         let next_result = tokio::time::timeout(
             Duration::from_secs(30),
             data_movement::migrate_decommission_object(
@@ -7827,6 +7833,12 @@ mod decommission_lock_order_tests {
             .expect("same-mutation retry should publish a readable target");
         assert_eq!(retry_info.size, i64::try_from(body.len()).expect("test body size fits i64"));
 
+        // Reconciled bytes remain charged. With no physical room for a new
+        // reservation, a distinct mutation must fail instead of reusing them.
+        let mut exhausted_capacity = capacity_snapshot();
+        exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total);
+        set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
+
         let next_object = "data-movement-put-after-retry.bin";
         let next_version_id = uuid::Uuid::new_v4().to_string();
         let mut next_opts = ObjectOptions {
@@ -7841,7 +7853,11 @@ mod decommission_lock_order_tests {
         let next_result = other_store
             .put_object_for_data_movement(RUSTFS_META_BUCKET, next_object, &mut next_data, &next_opts, None)
             .await;
-        assert!(next_result.is_err(), "a different mutation must not reuse A's recovered capacity");
+        let next_error = match next_result {
+            Err(error) | Ok((_, Err(error))) => error,
+            Ok((_, Ok(_))) => panic!("a different mutation must not reuse A's recovered capacity"),
+        };
+        assert!(next_error.to_string().contains("decommission_capacity_blocked"), "{next_error}");
         let next_target_err = other_store.pools[2]
             .get_object_info(
                 RUSTFS_META_BUCKET,
@@ -9929,6 +9945,12 @@ mod decommission_lock_order_tests {
             )
             .await
             .expect("read the next source object for the capacity oracle");
+        // Reconciled bytes remain charged. With no physical room for a new
+        // reservation, a distinct mutation must fail instead of reusing them.
+        let mut exhausted_capacity = capacity_snapshot();
+        exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total);
+        set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
+
         let next_result = tokio::time::timeout(
             Duration::from_secs(30),
             data_movement::migrate_decommission_object(
@@ -10190,11 +10212,20 @@ mod decommission_lock_order_tests {
         assert_eq!(reconciled_reservation.pending_target_physical_bytes, 0);
         assert_eq!(reconciled_reservation.consumed_target_physical_bytes, 1);
 
+        // Reconciled bytes remain charged. With no physical room for a new
+        // reservation, a distinct mutation must fail instead of reusing them.
+        let mut exhausted_capacity = capacity_snapshot();
+        exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 1, 2, 1);
+        set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
+
         let next_after_reconcile = other_store
             .delete_object(&bucket, next_object, next_opts)
             .await
             .expect_err("the consumed exact-fit capacity must reject the next marker");
-        assert!(next_after_reconcile.to_string().contains("decommission_capacity_blocked"));
+        assert!(
+            next_after_reconcile.to_string().contains("decommission_capacity_blocked"),
+            "{next_after_reconcile}"
+        );
         let next_target_err = other_store.pools[2]
             .get_object_info(
                 &bucket,

crates/ecstore/src/core/sets.rs

@@ -317,8 +317,9 @@ impl Sets {
         &self.ctx
     }
 
-    #[cfg(test)]
-    pub(crate) async fn stop_endpoint_monitor_for_test(&self) {
+    /// Stop endpoint reconnection and wait for any in-flight pass before a test owns disk probes.
+    #[cfg(any(test, feature = "test-util"))]
+    pub async fn stop_endpoint_monitor_for_test(&self) {
         if let Some(exit_signal) = &self.exit_signal {
             let _ = exit_signal.send(());
             // Wait for an in-flight reconnect pass as well as its receiver to exit.

crates/ecstore/src/data_movement/e2e_barrier.rs

@@ -31,6 +31,8 @@ use tokio::fs;
 use tokio::time::Instant;
 
 pub(crate) const BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
+/// Rebalance holds its admission fence; no version of this entry has moved.
+pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
 /// Every target part is staged and the target version is not yet published.
 pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
 /// The target commit is durable and the source preflight matched; the source

crates/ecstore/src/data_movement/mod.rs

@@ -605,6 +605,7 @@ fn resolve_data_movement_abort_result(
 #[derive(Debug)]
 struct DataMovementStageError {
     rendered: String,
+    stage: Option<String>,
     source: Box<dyn std::error::Error + Send + Sync>,
 }
 
@@ -626,6 +627,7 @@ where
 {
     Error::other(DataMovementStageError {
         rendered,
+        stage: None,
         source: Box::new(err),
     })
 }
@@ -634,7 +636,21 @@ fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, objec
 where
     E: std::error::Error + Send + Sync + 'static,
 {
-    data_movement_context_error(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"), err)
+    Error::other(DataMovementStageError {
+        rendered: format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"),
+        stage: Some(stage.to_owned()),
+        source: Box::new(err),
+    })
+}
+
+pub(crate) fn data_movement_error_is_stage(err: &Error, stage: &str) -> bool {
+    let Error::Io(io_err) = err else {
+        return false;
+    };
+    io_err
+        .get_ref()
+        .and_then(|source| source.downcast_ref::<DataMovementStageError>())
+        .is_some_and(|source| source.stage.as_deref() == Some(stage))
 }
 
 #[cfg(test)]

crates/ecstore/src/data_usage/mod.rs

@@ -34,7 +34,7 @@ pub use local_snapshot::{LocalUsageSnapshot, read_snapshot as read_local_snapsho
 use rustfs_data_usage::{
     BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
     DataUsageCache, DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, VersionsHistogram,
-    observed_data_usage_is_newer,
+    observed_data_usage_is_newer, observed_data_usage_is_newer_than_identity,
 };
 use rustfs_io_metrics::record_system_path_failure;
 use rustfs_utils::path::SLASH_SEPARATOR;
@@ -227,7 +227,10 @@ pub async fn lookup_degraded_bucket_usage_baseline(store: Arc<ECStore>, bucket:
 /// endpoints for up to `DATA_USAGE_CACHE_TTL_SECS` between backend reads.
 #[derive(Debug, Clone)]
 struct CachedDataUsageSnapshot {
+    /// Last successful load, retained through failures for lineage comparison.
     info: Option<DataUsageInfo>,
+    /// Controls serving independently from the retained ordering evidence.
+    last_load_failed: bool,
     loaded_at: tokio::time::Instant,
     /// Pre-discard per-bucket sizes from the same load, retained even when the
     /// snapshot is incomplete and its bucket data is discarded. Consumed only
@@ -240,6 +243,7 @@ impl CachedDataUsageSnapshot {
     fn result<T>(&self, project: impl FnOnce(&DataUsageInfo) -> T) -> Result<T, Error> {
         self.info
             .as_ref()
+            .filter(|_| !self.last_load_failed)
             .map(project)
             .ok_or_else(|| Error::other("data usage snapshot load recently failed"))
     }
@@ -275,6 +279,17 @@ fn cache_data_usage_snapshot_result<T>(
             mut degraded_baseline,
             observed_unavailable,
         }) => {
+            // A primary read can fall back to an older backup. Refresh the
+            // retry age without replacing a newer snapshot this process has
+            // already read; snapshot size itself is not an ordering signal.
+            if let Some(previous) = cache.as_mut()
+                && let Some(cached_info) = previous.info.as_ref()
+                && snapshot_predates(&info, cached_info)
+            {
+                previous.loaded_at = loaded_at;
+                previous.last_load_failed = false;
+                return Some(Ok(project(cached_info)));
+            }
             // A flaky observed read must not shrink quota coverage for a TTL
             // window: carry the previous refresh's baseline entries forward,
             // letting the fresh (authoritative) values win where they exist.
@@ -286,21 +301,23 @@ fn cache_data_usage_snapshot_result<T>(
             let projected = project(&info);
             *cache = Some(CachedDataUsageSnapshot {
                 info: Some(info),
+                last_load_failed: false,
                 loaded_at,
                 degraded_baseline,
             });
             Ok(projected)
         }
         Err(e) => {
-            // Keep the previous baseline through a failed refresh: quota
-            // admission must not lose its last grounded values because one
-            // backend read errored.
-            let degraded_baseline = cache.take().map(|cached| cached.degraded_baseline).unwrap_or_default();
-            *cache = Some(CachedDataUsageSnapshot {
+            // A failed load is negatively cached, but cannot erase the last
+            // good snapshot's ordering proof or grounded quota baseline.
+            let cached = cache.get_or_insert_with(|| CachedDataUsageSnapshot {
                 info: None,
+                last_load_failed: true,
                 loaded_at,
-                degraded_baseline,
+                degraded_baseline: HashMap::new(),
             });
+            cached.loaded_at = loaded_at;
+            cached.last_load_failed = true;
             Err(e)
         }
     })
@@ -1245,7 +1262,7 @@ async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Resu
     let (loaded_snapshot, source) = load_data_usage_snapshot(store.clone()).await?;
     // The observed-newness gate below compares against the snapshot as
     // persisted, before normalization demotes or discards anything.
-    let authoritative_as_persisted = loaded_snapshot.clone();
+    let authoritative_as_persisted = loaded_snapshot.snapshot_identity();
     let (info, mut degraded_baseline) = normalize_loaded_data_usage(loaded_snapshot, source.is_authoritative()).await;
 
     // A bucket without a converged scanner cycle behind it — a freshly joined
@@ -1263,7 +1280,7 @@ async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Resu
     // read per snapshot refresh.
     let mut observed_unavailable = false;
     match load_observed_data_usage_snapshot(store).await {
-        Ok(Some(observed)) if observed_data_usage_is_newer(&observed, &authoritative_as_persisted) => {
+        Ok(Some(observed)) if observed_data_usage_is_newer_than_identity(&observed, authoritative_as_persisted) => {
             backfill_degraded_baseline_from_observed(&mut degraded_baseline, &observed);
         }
         Ok(_) => {}
@@ -2367,6 +2384,9 @@ fn unknown_bucket_reload_due(cache: &Option<CachedDataUsageSnapshot>, bucket: &s
     let Some(cached) = cache.as_ref() else {
         return false;
     };
+    if cached.last_load_failed {
+        return false;
+    }
     let Some(info) = cached.info.as_ref() else {
         return false;
     };
@@ -2382,11 +2402,35 @@ fn unknown_bucket_reload_due(cache: &Option<CachedDataUsageSnapshot>, bucket: &s
         .is_none_or(|attempted_at| now.duration_since(attempted_at) >= refresh_age)
 }
 
-/// Whether a freshly loaded snapshot is older than the cached one, as when a
-/// primary read fails and the loader falls back to the previous `.bkp` copy.
-fn snapshot_predates(loaded: Option<SystemTime>, cached: Option<SystemTime>) -> bool {
+/// Compare authoritative lineage by the writer's epoch, cycle, then timestamp
+/// order. Within one lineage, compare the observation's coordinated position
+/// before its timestamp; a later observation cannot revive an older lineage.
+fn snapshot_predates(loaded: &DataUsageInfo, cached: &DataUsageInfo) -> bool {
+    let loaded_baseline = loaded
+        .usage_snapshot_authoritative_baseline
+        .unwrap_or_else(|| loaded.snapshot_identity());
+    let cached_baseline = cached
+        .usage_snapshot_authoritative_baseline
+        .unwrap_or_else(|| cached.snapshot_identity());
+    let loaded_position = (loaded_baseline.scanner_epoch, loaded_baseline.scanner_cycle);
+    let cached_position = (cached_baseline.scanner_epoch, cached_baseline.scanner_cycle);
+    if loaded_position != cached_position {
+        return loaded_position < cached_position;
+    }
+    let (loaded, cached) = if loaded_baseline == cached_baseline {
+        let loaded_observation_position = loaded.scanner_epoch.zip(loaded.scanner_cycle);
+        let cached_observation_position = cached.scanner_epoch.zip(cached.scanner_cycle);
+        if loaded_observation_position != cached_observation_position {
+            return loaded_observation_position < cached_observation_position;
+        }
+        (loaded.last_update, cached.last_update)
+    } else {
+        (loaded_baseline.last_update, cached_baseline.last_update)
+    };
     match (loaded, cached) {
-        (Some(loaded), Some(cached)) => loaded < cached,
+        (Some(loaded), Some(cached)) => {
+            loaded < cached && !rustfs_data_usage::usage_last_update_is_untrusted_future(cached, SystemTime::now())
+        }
         (None, Some(_)) => true,
         _ => false,
     }
@@ -2427,22 +2471,15 @@ where
         return false;
     };
     let mut cache = data_usage_snapshot_cache().write().await;
-    let cached_last_update = cache
-        .as_ref()
-        .and_then(|cached| cached.info.as_ref())
-        .and_then(|info| info.last_update);
-    if snapshot_predates(loaded.info.last_update, cached_last_update) {
-        return false;
-    }
     cache_data_usage_snapshot_result(
         &mut cache,
         Ok(loaded),
         tokio::time::Instant::now(),
         refresh_generation,
         data_usage_snapshot_generation(),
-        |_| (),
+        |info| info.buckets_usage.contains_key(bucket),
     )
-    .is_some()
+    .is_some_and(|result| matches!(result, Ok(true)))
 }
 
 // Calls of `reload_unknown_bucket_usage` per bucket. Unit tests do not run in
@@ -4224,6 +4261,7 @@ mod tests {
         let degraded_baseline = info.bucket_sizes.clone();
         *data_usage_snapshot_cache().write().await = Some(CachedDataUsageSnapshot {
             info: Some(info),
+            last_load_failed: false,
             loaded_at: tokio::time::Instant::now()
                 .checked_sub(age)
                 .expect("the monotonic clock must reach back by the requested age"),
@@ -4700,6 +4738,355 @@ mod tests {
         assert_snapshot_bucket(&cached, "bucket");
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn expired_usage_snapshot_refresh_keeps_newer_counts_over_an_older_backup() {
+        use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
+        use crate::config::com::save_config;
+
+        let (_dirs, store) = isolated_store_over_temp_disks().await;
+        let mut current = usage_snapshot_with_new_bucket(SystemTime::UNIX_EPOCH + Duration::from_secs(2), 100);
+        current.scanner_epoch = Some(4);
+        current.scanner_cycle = Some(11);
+        current.usage_snapshot_converged = Some(true);
+        let mut backup = data_usage_info_for_test("existing", 1, 5, SystemTime::UNIX_EPOCH + Duration::from_secs(1));
+        backup.scanner_epoch = Some(4);
+        backup.scanner_cycle = Some(10);
+        backup.usage_snapshot_converged = Some(true);
+        for (path, info) in [
+            (DATA_USAGE_OBJ_NAME_PATH.as_str(), &current),
+            (DATA_USAGE_OBJ_BACKUP_PATH.as_str(), &backup),
+        ] {
+            save_config(store.clone(), path, serde_json::to_vec(info).expect("encode usage snapshot"))
+                .await
+                .expect("persist initial usage snapshot");
+        }
+        invalidate_data_usage_snapshot_cache().await;
+        for loaded in [
+            load_data_usage_from_backend_cached(store.clone()).await,
+            load_admin_data_usage_from_backend_cached(store.clone()).await,
+        ] {
+            assert_eq!(loaded.expect("prime usage cache"), current);
+        }
+
+        // Bucket cleanup can reduce usage without advancing the scanner
+        // position. Only its publication timestamp advances.
+        let mut after_delete = data_usage_info_for_test("existing", 1, 3, SystemTime::UNIX_EPOCH + Duration::from_secs(3));
+        after_delete.scanner_epoch = current.scanner_epoch;
+        after_delete.scanner_cycle = current.scanner_cycle;
+        after_delete.usage_snapshot_converged = Some(true);
+        let mut stale_observation =
+            data_usage_info_for_test("retired-incarnation", 99, 999, SystemTime::UNIX_EPOCH + Duration::from_secs(4));
+        stale_observation.scanner_epoch = Some(4);
+        stale_observation.scanner_cycle = Some(12);
+        stale_observation.usage_snapshot_converged = Some(false);
+        stale_observation.usage_snapshot_authoritative_baseline = Some(backup.snapshot_identity());
+        assert!(observed_data_usage_is_newer(&stale_observation, &backup));
+        assert!(!observed_data_usage_is_newer(&stale_observation, &current));
+        save_config(
+            store.clone(),
+            DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
+            serde_json::to_vec(&stale_observation).expect("encode observation of an older namespace"),
+        )
+        .await
+        .expect("persist an observation anchored to the old backup");
+        for (payload, expected) in [
+            (b"not a usage snapshot".to_vec(), &current),
+            (serde_json::to_vec(&after_delete).expect("encode post-delete usage"), &after_delete),
+        ] {
+            save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), payload)
+                .await
+                .expect("replace primary usage snapshot");
+            for cache in [data_usage_snapshot_cache(), admin_data_usage_snapshot_cache()] {
+                cache
+                    .write()
+                    .await
+                    .as_mut()
+                    .expect("usage cache should be populated")
+                    .loaded_at = tokio::time::Instant::now() - Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
+            }
+            for loaded in [
+                load_data_usage_from_backend_cached(store.clone()).await,
+                load_admin_data_usage_from_backend_cached(store.clone()).await,
+            ] {
+                assert_eq!(loaded.expect("refresh an expired usage cache"), *expected);
+            }
+            assert!(
+                !data_usage_snapshot_cache()
+                    .read()
+                    .await
+                    .as_ref()
+                    .expect("quota cache should remain populated")
+                    .degraded_baseline
+                    .contains_key("retired-incarnation"),
+                "a newer observation anchored to an older namespace must not backfill ghost usage"
+            );
+        }
+        invalidate_data_usage_snapshot_cache().await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn expired_admin_usage_refresh_orders_observations_by_generation_before_time() {
+        use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
+        use crate::config::com::save_config;
+
+        let (_dirs, store) = isolated_store_over_temp_disks().await;
+        let mut authoritative = data_usage_info_for_test("bucket", 1, 10, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
+        authoritative.scanner_epoch = Some(7);
+        authoritative.scanner_cycle = Some(10);
+        authoritative.usage_snapshot_converged = Some(true);
+        save_config(
+            store.clone(),
+            DATA_USAGE_OBJ_NAME_PATH.as_str(),
+            serde_json::to_vec(&authoritative).expect("encode authoritative usage"),
+        )
+        .await
+        .expect("persist the common authoritative baseline");
+        let observation = |epoch, cycle, seconds, objects, size| {
+            let mut info =
+                data_usage_info_for_test("bucket", objects, size, SystemTime::UNIX_EPOCH + Duration::from_secs(seconds));
+            info.scanner_epoch = Some(epoch);
+            info.scanner_cycle = Some(cycle);
+            info.usage_snapshot_converged = Some(false);
+            info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
+            info
+        };
+        let first = observation(7, 11, 200, 11, 1_100);
+        let newer_cycle = observation(7, 12, 150, 12, 1_200);
+        let older_cycle = observation(7, 11, 300, 99, 9_900);
+        let newer_epoch = observation(8, 1, 50, 1, 100);
+        invalidate_data_usage_snapshot_cache().await;
+
+        for (observed, expected) in [
+            (&first, &first),
+            (&newer_cycle, &newer_cycle),
+            (&older_cycle, &newer_cycle),
+            (&newer_epoch, &newer_epoch),
+        ] {
+            assert!(
+                observed_data_usage_is_newer(observed, &authoritative),
+                "the persisted observation must pass the real loader's selection gate"
+            );
+            save_config(
+                store.clone(),
+                DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
+                serde_json::to_vec(observed).expect("encode scanner observation"),
+            )
+            .await
+            .expect("persist scanner observation");
+            if let Some(cached) = admin_data_usage_snapshot_cache().write().await.as_mut() {
+                cached.loaded_at = tokio::time::Instant::now() - Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
+            }
+            let loaded = load_admin_data_usage_from_backend_cached(store.clone())
+                .await
+                .expect("refresh admin usage through the real backend loader");
+            assert_eq!(
+                &loaded,
+                expected,
+                "same-baseline observation ordering must follow epoch/cycle before wall clock: incoming={:?}",
+                observed.snapshot_identity()
+            );
+        }
+        invalidate_data_usage_snapshot_cache().await;
+    }
+
+    #[test]
+    fn expired_usage_snapshot_refresh_accepts_new_authority_over_a_later_old_observation() {
+        let loaded_at = tokio::time::Instant::now();
+        let baseline = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(1));
+        let mut observed = data_usage_info_for_test("bucket", 5, 60, SystemTime::UNIX_EPOCH + Duration::from_secs(4));
+        observed.usage_snapshot_converged = Some(false);
+        observed.usage_snapshot_authoritative_baseline = Some(baseline.snapshot_identity());
+        let mut cache = Some(CachedDataUsageSnapshot {
+            info: Some(observed),
+            last_load_failed: false,
+            loaded_at,
+            degraded_baseline: HashMap::new(),
+        });
+        let after_delete = data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH + Duration::from_secs(3));
+        let refreshed = cache_data_usage_snapshot_result(
+            &mut cache,
+            Ok(loaded_usage_for_test(after_delete.clone())),
+            loaded_at,
+            1,
+            1,
+            DataUsageInfo::clone,
+        )
+        .expect("matching refresh generation")
+        .expect("a newer authority must retire observations of its predecessor");
+        assert_eq!(refreshed, after_delete);
+    }
+
+    #[test]
+    fn expired_usage_snapshot_refresh_orders_lineage_by_epoch_cycle_then_time() {
+        let loaded_at = tokio::time::Instant::now();
+        for (cached_position, loaded_position, loaded_time, accept) in [
+            ((Some(2), Some(10)), (Some(1), Some(99)), 110, false),
+            ((Some(2), Some(10)), (Some(2), Some(9)), 110, false),
+            ((Some(2), Some(10)), (Some(3), Some(1)), 90, true),
+            ((Some(2), Some(10)), (Some(2), Some(11)), 90, true),
+            ((Some(2), Some(10)), (Some(2), Some(10)), 110, true),
+            ((Some(3), None), (Some(2), Some(10)), 110, false),
+            ((Some(2), Some(10)), (Some(3), None), 90, true),
+            ((None, None), (None, None), 110, true),
+            ((None, None), (None, None), 90, false),
+        ] {
+            let mut previous = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
+            (previous.scanner_epoch, previous.scanner_cycle) = cached_position;
+            let mut loaded = data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH + Duration::from_secs(loaded_time));
+            (loaded.scanner_epoch, loaded.scanner_cycle) = loaded_position;
+            let expected = if accept { &loaded } else { &previous };
+            let mut cache = Some(CachedDataUsageSnapshot {
+                info: Some(previous.clone()),
+                last_load_failed: false,
+                loaded_at,
+                degraded_baseline: HashMap::new(),
+            });
+            let refreshed = cache_data_usage_snapshot_result(
+                &mut cache,
+                Ok(loaded_usage_for_test(loaded.clone())),
+                loaded_at,
+                1,
+                1,
+                DataUsageInfo::clone,
+            )
+            .expect("matching refresh generation")
+            .expect("a valid usage snapshot remains available");
+            assert_eq!(
+                &refreshed, expected,
+                "lineage ordering must match scanner publication: cached={cached_position:?}, loaded={loaded_position:?}, loaded_time={loaded_time}"
+            );
+        }
+    }
+
+    #[test]
+    fn expired_usage_snapshot_refresh_preserves_rejection_backoff_and_baseline() {
+        let loaded_at = tokio::time::Instant::now();
+        let expected = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(2));
+        let mut cache = Some(CachedDataUsageSnapshot {
+            info: Some(expected.clone()),
+            last_load_failed: false,
+            loaded_at,
+            degraded_baseline: HashMap::from([("bucket".to_string(), 42)]),
+        });
+        let refreshed_at = loaded_at + Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
+        let refreshed = cache_data_usage_snapshot_result(
+            &mut cache,
+            Ok(loaded_usage_for_test(data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH))),
+            refreshed_at,
+            1,
+            1,
+            DataUsageInfo::clone,
+        )
+        .expect("matching refresh generation")
+        .expect("the newer cached snapshot remains available");
+        assert_eq!(refreshed, expected);
+        let retained = cache.as_ref().expect("cache should remain populated");
+        assert_eq!(retained.degraded_baseline.get("bucket"), Some(&42));
+        assert_eq!(retained.loaded_at, refreshed_at, "an older fallback must retain the TTL backoff");
+
+        let result =
+            cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), refreshed_at, 1, 1, DataUsageInfo::clone)
+                .expect("matching refresh generation");
+        assert!(matches!(result, Err(Error::ErasureReadQuorum)));
+        let failed = cache.as_ref().expect("negative cache should be populated");
+        assert!(failed.last_load_failed);
+        assert_eq!(failed.info.as_ref(), Some(&expected), "failure must retain the last good ordering proof");
+    }
+
+    #[test]
+    fn expired_usage_snapshot_refresh_accepts_clock_recovery() {
+        let loaded_at = tokio::time::Instant::now();
+        let now = SystemTime::now();
+        let future = now + rustfs_data_usage::USAGE_LAST_UPDATE_FUTURE_TOLERANCE + Duration::from_secs(60);
+        let mut cache = Some(CachedDataUsageSnapshot {
+            info: Some(data_usage_info_for_test("bucket", 3, 42, future)),
+            last_load_failed: false,
+            loaded_at,
+            degraded_baseline: HashMap::new(),
+        });
+        let recovered = data_usage_info_for_test("bucket", 1, 7, now);
+        let refreshed = cache_data_usage_snapshot_result(
+            &mut cache,
+            Ok(loaded_usage_for_test(recovered.clone())),
+            loaded_at,
+            1,
+            1,
+            DataUsageInfo::clone,
+        )
+        .expect("matching refresh generation")
+        .expect("a clock recovery snapshot must not be frozen behind an untrusted timestamp");
+        assert_eq!(refreshed, recovered);
+    }
+
+    #[test]
+    fn expired_usage_snapshot_refresh_keeps_last_good_lineage_across_a_failed_load() {
+        let loaded_at = tokio::time::Instant::now();
+        let current = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(2));
+        let mut cache = None;
+        let first = cache_data_usage_snapshot_result(
+            &mut cache,
+            Ok(loaded_usage_for_test(current.clone())),
+            loaded_at,
+            1,
+            1,
+            DataUsageInfo::clone,
+        )
+        .expect("matching refresh generation")
+        .expect("prime the last successful snapshot");
+        assert_eq!(first, current);
+
+        let failed_at = loaded_at + Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
+        let failure =
+            cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), failed_at, 1, 1, DataUsageInfo::clone)
+                .expect("matching refresh generation");
+        assert!(matches!(failure, Err(Error::ErasureReadQuorum)));
+        assert!(
+            fresh_cached_data_usage_snapshot(
+                &cache,
+                failed_at + Duration::from_secs(DATA_USAGE_UNKNOWN_BUCKET_REFRESH_SECS),
+                Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS),
+                |_| panic!("a failed load must not serve its retained snapshot as a success"),
+            )
+            .expect("the failed load remains inside its negative TTL")
+            .is_err()
+        );
+        assert!(
+            !unknown_bucket_reload_due(
+                &cache,
+                "unknown",
+                failed_at + Duration::from_secs(DATA_USAGE_UNKNOWN_BUCKET_REFRESH_SECS),
+            ),
+            "an unknown bucket must not bypass the failed load's TTL"
+        );
+
+        let recovered_at = failed_at + Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
+        let recovered = cache_data_usage_snapshot_result(
+            &mut cache,
+            Ok(loaded_usage_for_test(data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH))),
+            recovered_at,
+            1,
+            1,
+            DataUsageInfo::clone,
+        )
+        .expect("matching refresh generation")
+        .expect("a readable older backup must preserve the last good lineage");
+        assert_eq!(recovered, current);
+        assert_eq!(
+            cache.as_ref().expect("recovered cache").degraded_baseline.get("bucket"),
+            Some(&42),
+            "a failed load must not erase the baseline's ordering proof"
+        );
+        assert_eq!(
+            fresh_cached_data_usage_snapshot(&cache, recovered_at, Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS), |info| info
+                .objects_total_size,)
+            .expect("the recovered load starts a fresh TTL")
+            .expect("the last successful snapshot is available again"),
+            42
+        );
+    }
+
     #[test]
     #[serial]
     fn unavailable_observed_read_keeps_previous_baseline_coverage() {
@@ -4707,6 +5094,7 @@ mod tests {
         let refresh_generation = data_usage_snapshot_generation();
         let mut cache = Some(CachedDataUsageSnapshot {
             info: Some(data_usage_info_for_test("bucket", 1, 42, SystemTime::UNIX_EPOCH)),
+            last_load_failed: false,
             loaded_at,
             degraded_baseline: HashMap::from([("observed-only".to_string(), 7_u64), ("covered".to_string(), 1)]),
         });
@@ -4740,6 +5128,7 @@ mod tests {
         let refresh_generation = data_usage_snapshot_generation();
         let mut cache = Some(CachedDataUsageSnapshot {
             info: Some(data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH)),
+            last_load_failed: false,
             loaded_at,
             degraded_baseline: HashMap::new(),
         });
@@ -5928,6 +6317,7 @@ mod tests {
         let mut snapshot_cache = data_usage_snapshot_cache().write().await;
         *snapshot_cache = Some(CachedDataUsageSnapshot {
             info: Some(successor),
+            last_load_failed: false,
             loaded_at: tokio::time::Instant::now(),
             degraded_baseline: HashMap::new(),
         });
@@ -5999,6 +6389,7 @@ mod tests {
         let mut snapshot_cache = data_usage_snapshot_cache().write().await;
         *snapshot_cache = Some(CachedDataUsageSnapshot {
             info: Some(successor),
+            last_load_failed: false,
             loaded_at: tokio::time::Instant::now(),
             degraded_baseline: HashMap::new(),
         });
@@ -6051,6 +6442,7 @@ mod tests {
         let stale = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now());
         *data_usage_snapshot_cache().write().await = Some(CachedDataUsageSnapshot {
             info: Some(stale),
+            last_load_failed: false,
             loaded_at: tokio::time::Instant::now(),
             degraded_baseline: HashMap::new(),
         });

crates/ecstore/src/data_usage/tests/snapshot_projection.rs

@@ -21,6 +21,7 @@ fn scalar_projection_borrows_snapshot_and_preserves_unknown_and_zero() {
     info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
     let cache = Some(CachedDataUsageSnapshot {
         info: Some(info),
+        last_load_failed: false,
         loaded_at,
         degraded_baseline: HashMap::new(),
     });
@@ -66,6 +67,7 @@ fn projection_is_skipped_for_missing_failed_expired_or_invalidated_snapshots() {
 
     cache = Some(CachedDataUsageSnapshot {
         info: Some(data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH)),
+        last_load_failed: false,
         loaded_at,
         degraded_baseline: HashMap::new(),
     });
@@ -127,6 +129,33 @@ fn projected_publication_moves_bucket_maps_into_cache() {
         cached.buckets_usage.get_key_value("known").expect("cached bucket").0.as_ptr(),
         original_key
     );
+
+    let failure = cache_data_usage_snapshot_result(
+        &mut cache,
+        Err(Error::ErasureReadQuorum),
+        tokio::time::Instant::now(),
+        1,
+        1,
+        |_: &DataUsageInfo| -> u64 { panic!("a failed load must not project its retained snapshot") },
+    )
+    .expect("matching refresh generation");
+    assert!(matches!(failure, Err(Error::ErasureReadQuorum)));
+    let retained = cache
+        .as_ref()
+        .expect("negative cache")
+        .info
+        .as_ref()
+        .expect("last good ordering proof");
+    assert_eq!(
+        retained
+            .buckets_usage
+            .get_key_value("known")
+            .expect("retained bucket")
+            .0
+            .as_ptr(),
+        original_key,
+        "a failed load must retain the snapshot without copying its bucket maps"
+    );
 }
 
 #[test]

crates/ecstore/src/diagnostics/object_lock.rs

@@ -167,6 +167,16 @@ impl ObjectLockAttempt {
             }
         }
     }
+
+    /// Correlate a raw guard's acquisition with its peer identities without
+    /// claiming ownership of the guard's lifetime. Peer release events retain
+    /// the identities needed to reconstruct when the actual owner releases it.
+    pub(crate) fn observe_acquisition(&mut self, result: &Result<NamespaceLockGuard, LockError>) {
+        self.observe(result);
+        if let Some(context) = self.0.as_mut() {
+            context.finished = true;
+        }
+    }
 }
 
 fn duration_ms(duration: Duration) -> u64 {
@@ -500,6 +510,45 @@ pub(crate) mod tests {
         )
     }
 
+    #[test]
+    #[serial_test::serial]
+    fn raw_guard_acquisition_diagnostic_does_not_claim_guard_lifetime() {
+        temp_env::with_vars([(rustfs_config::ENV_OBJECT_LOCK_DIAG_ENABLE, Some("true"))], || {
+            let capture = CapturedLockEvents::default();
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("raw guard diagnostic runtime");
+            tracing::subscriber::with_default(capture.subscriber(), || {
+                runtime.block_on(async {
+                    let lock = namespace();
+                    let mut attempt = ObjectLockAttempt::start(
+                        "bucket_incarnation_disk_read",
+                        "bucket",
+                        "transaction.lock",
+                        None,
+                        &lock,
+                        "read",
+                        Duration::from_secs(1),
+                    );
+                    let result = lock.get_read_lock(Duration::from_secs(1)).await;
+                    attempt.observe_acquisition(&result);
+                    drop(attempt);
+                    let guard = result.expect("retain the raw guard after diagnostic completion");
+                    assert!(!guard.is_lock_lost());
+                    let states = capture
+                        .rows()
+                        .iter()
+                        .map(|row| row["fields"]["state"].as_str().expect("diagnostic state string").to_owned())
+                        .collect::<Vec<_>>();
+                    assert_eq!(states, ["acquiring", "acquired"]);
+                    drop(guard);
+                    assert_eq!(capture.rows().len(), 2, "the acquisition observer must not invent a guard release");
+                });
+            });
+        });
+    }
+
     #[test]
     #[serial_test::serial]
     fn object_lock_diagnostics_disabled_does_not_allocate_context_or_emit() {

crates/ecstore/src/disk/disk_store.rs

@@ -55,6 +55,20 @@ use tokio_util::sync::CancellationToken;
 use tracing::{info, warn};
 use uuid::Uuid;
 
+/// Join the local disk's actual bootstrap worker for deterministic coverage tests.
+/// Completion may mean legacy fallback after a failed scan; callers must assert
+/// the expected index result separately. Remote disks have no local worker.
+#[cfg(any(test, feature = "test-util"))]
+pub async fn wait_for_multipart_index_bootstrap(disk: &crate::disk::DiskStore) -> Result<()> {
+    match disk.as_ref() {
+        crate::disk::Disk::Local(local) => {
+            local.disk.wait_for_multipart_index_bootstrap().await;
+            Ok(())
+        }
+        crate::disk::Disk::Remote(_) => Err(DiskError::MethodNotAllowed),
+    }
+}
+
 /// Disk health status constants
 const DISK_HEALTH_OK: u32 = 0;
 const DISK_HEALTH_FAULTY: u32 = 1;
@@ -853,6 +867,8 @@ pub(crate) struct DiskHealthMetricEpoch {
     fallback_operation_metrics: StdRwLock<HashMap<&'static str, Arc<DiskOperationMetrics>>>,
     #[cfg(test)]
     multipart_cleanup_observer: std::sync::Mutex<Option<std::sync::Weak<ScopedMultipartCleanupCompletions>>>,
+    #[cfg(test)]
+    operation_timeout_override: std::sync::Mutex<Option<Duration>>,
     /// Caller API operations currently executing through the disk health wrapper.
     api_waiting: AtomicU32,
     /// Operations rejected because the disk was unavailable for the caller.
@@ -880,6 +896,8 @@ impl Default for DiskHealthMetricEpoch {
             fallback_operation_metrics: StdRwLock::new(HashMap::new()),
             #[cfg(test)]
             multipart_cleanup_observer: std::sync::Mutex::new(None),
+            #[cfg(test)]
+            operation_timeout_override: std::sync::Mutex::new(None),
             api_waiting: AtomicU32::new(0),
             total_errors_availability: AtomicU64::new(0),
             total_errors_timeout: AtomicU64::new(0),
@@ -1573,6 +1591,20 @@ impl Default for DiskHealthTracker {
     }
 }
 
+#[cfg(test)]
+#[derive(Debug)]
+pub(crate) struct ScopedDiskOperationTimeout {
+    metrics: Arc<DiskHealthMetricEpoch>,
+    previous: Option<Duration>,
+}
+
+#[cfg(test)]
+impl Drop for ScopedDiskOperationTimeout {
+    fn drop(&mut self) {
+        *self.metrics.operation_timeout_override.lock().expect("test timeout lock") = self.previous;
+    }
+}
+
 /// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
 /// This is similar to Go's xlStorageDiskIDCheck.
 #[derive(Debug, Clone)]
@@ -1914,6 +1946,23 @@ impl LocalDiskWrapper {
         self.health_check
     }
 
+    #[cfg(test)]
+    pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout {
+        assert!(!timeout.is_zero(), "fault injection needs a positive timeout");
+        // The metrics epoch follows this drive across clones and reconnects.
+        // Its scoped override cannot shorten unrelated healthy drives' I/O.
+        let previous = self
+            .metrics
+            .operation_timeout_override
+            .lock()
+            .expect("test timeout lock")
+            .replace(timeout);
+        ScopedDiskOperationTimeout {
+            metrics: Arc::clone(&self.metrics),
+            previous,
+        }
+    }
+
     /// Enable health monitoring after disk creation.
     /// Used to defer health checks until after startup format loading completes.
     pub fn enable_health_check(&self) {
@@ -2398,12 +2447,17 @@ impl LocalDiskWrapper {
         .await
     }
 
+    pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
+        self.disk.has_clean_active_multipart_source(upload_or_part_path, part_path)
+    }
+
     pub(crate) async fn try_publish_prepared_delete_local(
         &self,
         volume: &str,
         object: &str,
         transaction: uuid::Uuid,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
+        work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
     ) -> Result<Option<(crate::disk::DeleteRecoveryJournal, bool, bool)>> {
         self.track_disk_health_mutation(
             "settle_delete_transaction",
@@ -2411,7 +2465,7 @@ impl LocalDiskWrapper {
             || {
                 Box::pin(async {
                     self.disk
-                        .try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
+                        .try_publish_prepared_delete_local_with_budget(volume, object, transaction, namespace_owner, work_budget)
                         .await
                 })
             },
@@ -2427,6 +2481,7 @@ impl LocalDiskWrapper {
         transaction: uuid::Uuid,
         journal: &crate::disk::DeleteRecoveryJournal,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
+        work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
     ) -> Result<bool> {
         self.track_disk_health_mutation(
             "settle_delete_transaction",
@@ -2434,7 +2489,14 @@ impl LocalDiskWrapper {
             || {
                 Box::pin(async {
                     self.disk
-                        .try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
+                        .try_finish_and_finalize_prepared_delete_local_with_budget(
+                            volume,
+                            object,
+                            transaction,
+                            journal,
+                            namespace_owner,
+                            work_budget,
+                        )
                         .await
                 })
             },
@@ -2634,6 +2696,13 @@ impl LocalDiskWrapper {
             }
             return result;
         }
+        #[cfg(test)]
+        let timeout_duration = self
+            .metrics
+            .operation_timeout_override
+            .lock()
+            .expect("test timeout lock")
+            .map_or(timeout_duration, |injected| timeout_duration.min(injected));
         // Execute the operation with timeout
         let result = tokio::time::timeout(timeout_duration, operation).await;
 
@@ -2682,6 +2751,18 @@ impl LocalDiskWrapper {
 
 #[async_trait::async_trait]
 impl DiskAPI for LocalDiskWrapper {
+    async fn list_multipart_index_roots(
+        &self,
+        bucket: &str,
+        marker: Option<&str>,
+    ) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
+        if self.health.is_faulty() {
+            return Err(DiskError::FaultyDisk);
+        }
+        self.check_disk_stale().await?;
+        self.disk.list_multipart_index_roots(bucket, marker).await
+    }
+
     async fn supports_guarded_delete_versions(&self) -> Result<()> {
         // The local implementation shares this binary's capability. Availability
         // and disk identity remain mutation checks and write-quorum inputs.
@@ -2735,6 +2816,62 @@ impl DiskAPI for LocalDiskWrapper {
         )
         .await
     }
+    async fn discover_delete_transactions_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        max_entries: u32,
+    ) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
+        self.track_disk_health_with_op(
+            "discover_delete_transactions",
+            || async {
+                self.disk
+                    .discover_delete_transactions_bounded(volume, object, max_entries)
+                    .await
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
+    async fn inspect_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        transaction: uuid::Uuid,
+        max_entries: u32,
+    ) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
+        self.track_disk_health_with_op(
+            "inspect_delete_transaction",
+            || async {
+                self.disk
+                    .inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
+                    .await
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
+    async fn settle_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        request: crate::disk::BoundedDeleteRecoverySettlement,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
+        self.track_disk_health_mutation(
+            "settle_delete_transaction",
+            DiskMetricMutation::Delete,
+            || async {
+                self.disk
+                    .settle_delete_transaction_bounded(volume, request, namespace_owner)
+                    .await
+            },
+            crate::disk::get_delete_recovery_timeout()?,
+        )
+        .await
+    }
+
     async fn discover_delete_transactions(
         &self,
         volume: &str,
@@ -3703,7 +3840,11 @@ mod tests {
         assert!(futures::poll!(&mut operation).is_pending());
         time::advance(SLOW_DISK_OPERATION_THRESHOLD - Duration::from_millis(1)).await;
         assert!(futures::poll!(&mut operation).is_pending());
-        assert!(logs.contents().is_empty(), "normal disk latency should not warn");
+        assert!(
+            !logs.contents().contains("disk_operation_slow") && !logs.contents().contains("WARN"),
+            "normal disk latency should not warn: {}",
+            logs.contents()
+        );
         time::advance(Duration::from_millis(1)).await;
         assert!(futures::poll!(&mut operation).is_pending());
         let warning = logs.contents();
@@ -6040,6 +6181,67 @@ mod tests {
         assert!(health.offline_duration().is_none());
     }
 
+    #[tokio::test]
+    async fn scoped_operation_timeout_is_per_disk_and_preserves_caller_policy() {
+        let directories = [
+            tempfile::tempdir().expect("faulted drive"),
+            tempfile::tempdir().expect("healthy drive"),
+        ];
+        let mut wrappers = Vec::new();
+        for directory in &directories {
+            let endpoint = Endpoint::try_from(directory.path().to_str().expect("UTF-8 drive path")).expect("drive endpoint");
+            let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local drive"));
+            wrappers.push(LocalDiskWrapper::new(disk, false));
+        }
+        let reconnected = LocalDiskWrapper::new_with_reconnect_state(
+            Arc::clone(&wrappers[0].disk),
+            false,
+            Some(wrappers[0].health_tracker_epoch_for_reconnect()),
+        );
+        let scope = wrappers[0].override_operation_timeout_for_test(Duration::from_secs(1));
+        tokio::time::pause();
+        let operation = || async {
+            tokio::time::sleep(Duration::from_secs(2)).await;
+            Ok(())
+        };
+        let (faulted, healthy) = tokio::join!(
+            reconnected.track_disk_health_with_op_and_timeout_action(
+                "read_metadata",
+                operation,
+                Duration::from_secs(3),
+                TimeoutHealthAction::IgnoreFailure,
+            ),
+            wrappers[1].track_disk_health_with_op_and_timeout_action(
+                "read_metadata",
+                operation,
+                Duration::from_secs(3),
+                TimeoutHealthAction::IgnoreFailure,
+            ),
+        );
+        assert_eq!(faulted.expect_err("only the selected drive must time out"), DiskError::Timeout);
+        healthy.expect("healthy drive keeps its caller's deadline");
+        let shorter_started = tokio::time::Instant::now();
+        let shorter = wrappers[0]
+            .track_disk_health_with_op_and_timeout_action(
+                "read_metadata",
+                operation,
+                Duration::from_millis(500),
+                TimeoutHealthAction::IgnoreFailure,
+            )
+            .await;
+        assert_eq!(shorter.expect_err("fault injection cannot extend a caller deadline"), DiskError::Timeout);
+        assert!(shorter_started.elapsed() < Duration::from_secs(1));
+        wrappers[0]
+            .track_disk_health_with_op("read_metadata", operation, Duration::ZERO)
+            .await
+            .expect("explicit unbounded physical ownership cannot acquire a test deadline");
+        drop(scope);
+        reconnected
+            .track_disk_health_with_op("read_metadata", operation, Duration::from_secs(3))
+            .await
+            .expect("dropping the scope restores reconnected callers' deadlines");
+    }
+
     #[tokio::test]
     async fn ignored_timeout_does_not_mark_drive_failure() {
         let dir = tempfile::tempdir().expect("temp dir should be created");

crates/ecstore/src/disk/error.rs

@@ -172,6 +172,8 @@ pub enum DiskError {
 
     #[error("guarded batch deletion is not supported by this disk")]
     GuardedDeleteUnsupported,
+    #[error("foreground delete recovery work limit reached")]
+    DeleteRecoveryWorkLimit,
 
     #[error("erasure write quorum")]
     ErasureWriteQuorum,
@@ -770,6 +772,7 @@ impl Clone for DiskError {
             DiskError::MethodNotAllowed => DiskError::MethodNotAllowed,
             DiskError::ConditionalDeleteConflict => DiskError::ConditionalDeleteConflict,
             DiskError::GuardedDeleteUnsupported => DiskError::GuardedDeleteUnsupported,
+            DiskError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
             DiskError::ErasureWriteQuorum => DiskError::ErasureWriteQuorum,
             DiskError::ErasureReadQuorum => DiskError::ErasureReadQuorum,
             DiskError::ShortWrite => DiskError::ShortWrite,
@@ -829,6 +832,7 @@ impl DiskError {
             DiskError::RemoteClientUnavailable(_) => 0x2B,
             DiskError::ConditionalDeleteConflict => 0x2C,
             DiskError::GuardedDeleteUnsupported => 0x2D,
+            DiskError::DeleteRecoveryWorkLimit => 0x2E,
         }
     }
 
@@ -879,6 +883,7 @@ impl DiskError {
             0x2B => Some(DiskError::RemoteClientUnavailable(String::new())),
             0x2C => Some(DiskError::ConditionalDeleteConflict),
             0x2D => Some(DiskError::GuardedDeleteUnsupported),
+            0x2E => Some(DiskError::DeleteRecoveryWorkLimit),
             _ => None,
         }
     }
@@ -1662,6 +1667,7 @@ mod tests {
         for (error, code) in [
             (DiskError::ConditionalDeleteConflict, 0x2C),
             (DiskError::GuardedDeleteUnsupported, 0x2D),
+            (DiskError::DeleteRecoveryWorkLimit, 0x2E),
         ] {
             assert_eq!(error.to_u32(), code);
             assert_eq!(DiskError::from_u32(code), Some(error.clone()));

crates/ecstore/src/disk/fd_cache_tests.rs

@@ -5,6 +5,7 @@ use super::{FdCache, FdCacheEntry, FdKey};
 use std::fs::File;
 use std::os::unix::fs::FileExt;
 use std::sync::Arc;
+use std::sync::atomic::AtomicBool;
 
 fn key(volume: &str, path: &str, direct: bool) -> FdKey {
     FdKey {
@@ -52,7 +53,11 @@ async fn uring_fd_cache_exact_invalidation_removes_both_modes_and_preserves_othe
 async fn uring_fd_cache_exact_invalidation_bumps_one_generation_and_rejects_old_epoch_inserts() {
     let (_directory, file) = fixture();
     let cache = FdCache::new();
-    let entry = Arc::new(FdCacheEntry { file, len: 3 });
+    let entry = Arc::new(FdCacheEntry {
+        file,
+        len: 3,
+        invalidated: AtomicBool::new(false),
+    });
     let old_generation = cache.generation();
     cache.invalidate_exact("v", "part").await;
     let current_generation = cache.generation();
@@ -125,7 +130,7 @@ async fn uring_fd_cache_volume_and_clear_invalidation_cover_both_modes() {
         cache.insert(key("v", "part", direct), Arc::clone(&file)).await;
         assert!(
             cache.get(&key("v", "part", direct)).await.is_some(),
-            "new entries must survive the earlier volume predicate"
+            "new entries must survive the earlier volume invalidation"
         );
     }
     cache.clear();
@@ -164,6 +169,7 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
     let replacement = Arc::new(FdCacheEntry {
         file: Arc::new(File::open(directory.path().join("part")).expect("open replacement inode")),
         len: 11,
+        invalidated: AtomicBool::new(false),
     });
     for direct in [false, true] {
         let key = key("v", "part", direct);
@@ -177,3 +183,459 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
         assert_eq!(fresh.len, 11);
     }
 }
+
+#[tokio::test]
+async fn fd_cache_tracker_skips_unmatched_invalidations_and_handles_deep_prefixes() {
+    let (_directory, file) = fixture();
+    let mut cache = FdCache::new();
+
+    // Unmatched invalidations on an empty cache must not spawn the maintenance task.
+    cache.invalidate_under("v", "missing/prefix");
+    cache.invalidate_volume("v");
+    cache.invalidate_exact("v", "missing/prefix/part.1").await;
+    cache.clear();
+    assert!(
+        cache.maintenance_task.get().is_none(),
+        "empty-cache invalidations must not spawn maintenance"
+    );
+
+    // Populate a normal entry and verify matched vs unmatched filter gates directly,
+    // and ensure unmatched invalidations do not enqueue pending removals.
+    let cached_key = key("v", "obj-1/0198a3f0/part.1", false);
+    cache.insert(cached_key.clone(), Arc::clone(&file)).await;
+    assert_eq!(cache.tracker.live_entries(), 1);
+    assert!(cache.tracker.may_match_volume("v"));
+    assert!(!cache.tracker.may_match_volume("other-volume"));
+    assert!(cache.tracker.may_match_prefix("v", "obj-1"));
+    assert!(cache.tracker.may_match_prefix("v", "obj-1/0198a3f0"));
+    assert!(!cache.tracker.may_match_prefix("v", "obj-2/0198a3f0"));
+    assert!(cache.tracker.may_match_exact("v", "obj-1/0198a3f0/part.1"));
+    assert!(!cache.tracker.may_match_exact("v", "obj-1/0198a3f0/xl.meta"));
+
+    cache.invalidate_volume("other-volume");
+    cache.invalidate_under("v", "obj-2/0198a3f0");
+    cache.invalidate_exact("v", "obj-1/0198a3f0/xl.meta").await;
+    assert!(
+        !cache
+            .tracker
+            .has_pending_invalidations
+            .load(std::sync::atomic::Ordering::SeqCst),
+        "unmatched invalidations must not enqueue maintenance removals"
+    );
+    assert!(cache.get(&cached_key).await.is_some());
+
+    // Deeply nested path (> FD_CACHE_MAX_INDEXED_PREFIXES slashes) must fall back
+    // to per-volume unindexed_counts without degrading other volumes.
+    let other_vol_key = key("v2", "shallow/part.1", false);
+    cache.insert(other_vol_key.clone(), Arc::clone(&file)).await;
+    let deep_path = "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13/c14/part.1";
+    let deep_key = key("v", deep_path, false);
+    cache.insert(deep_key.clone(), Arc::clone(&file)).await;
+    assert_eq!(
+        cache
+            .tracker
+            .unindexed_counts
+            .iter()
+            .map(|c| c.load(std::sync::atomic::Ordering::Acquire))
+            .sum::<u32>(),
+        1
+    );
+    assert!(
+        !cache.tracker.may_match_prefix("v2", "unmatched/prefix"),
+        "deep key in volume v must not disable prefix filtering in volume v2"
+    );
+    cache.invalidate_under("v", "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13");
+    assert!(cache.get(&deep_key).await.is_none());
+    cache.invalidate_volume("v2");
+    assert!(cache.get(&other_vol_key).await.is_none());
+
+    // Replacing an invalidated key before maintenance drains pending_invalidations
+    // must release the old descriptor immediately and keep the replacement live
+    // even when a strong reference to the old CachedFdEntry shell survives until
+    // maintenance runs its Arc::ptr_eq check.
+    let old_file = Arc::new(tempfile::tempfile().expect("create replaced descriptor"));
+    let old_weak = Arc::downgrade(&old_file);
+    cache.insert(cached_key.clone(), old_file).await;
+    let held_old_shell = cache.cache.get(&cached_key).expect("cached shell must exist");
+    cache.invalidate_under("v", "obj-1/0198a3f0");
+    assert!(cache.get(&cached_key).await.is_none());
+    cache.insert(cached_key.clone(), Arc::clone(&file)).await;
+    assert!(
+        old_weak.upgrade().is_none(),
+        "replacing an entry must retire and drop the previous descriptor immediately"
+    );
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while cache
+            .tracker
+            .has_pending_invalidations
+            .load(std::sync::atomic::Ordering::SeqCst)
+        {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("maintenance should drain pending invalidations");
+    drop(held_old_shell);
+    assert!(
+        cache.get(&cached_key).await.is_some(),
+        "stale pending invalidation must not evict a newer replacement"
+    );
+    assert_eq!(
+        cache
+            .tracker
+            .unindexed_counts
+            .iter()
+            .map(|c| c.load(std::sync::atomic::Ordering::Acquire))
+            .sum::<u32>(),
+        0
+    );
+
+    // Dropping a cache while a live non-invalidated entry is present (even if a
+    // cloned Moka handle outlives FdCache and the maintenance task is aborted)
+    // must retire the descriptor and release FdCacheTracker without cycles.
+    let live_drop_file = Arc::new(tempfile::tempfile().expect("create live drop descriptor"));
+    let live_drop_weak = Arc::downgrade(&live_drop_file);
+    let shutdown_file = Arc::new(tempfile::tempfile().expect("create shutdown descriptor"));
+    let shutdown_weak = Arc::downgrade(&shutdown_file);
+    let tracker_weak = Arc::downgrade(&cache.tracker);
+    let cloned_moka = cache.cache.clone();
+    cache.insert(key("v", "live-on-drop/part.1", false), live_drop_file).await;
+    cache.insert(key("v", "shutdown/part.1", false), shutdown_file).await;
+    cache.invalidate_under("v", "shutdown");
+    if let Some(handle) = cache.maintenance_task.take() {
+        handle.abort();
+        let _ = handle.await;
+    }
+    drop(cache);
+    assert!(
+        live_drop_weak.upgrade().is_none(),
+        "Drop for FdCache must retire live descriptors even if a cloned Moka handle survives"
+    );
+    assert!(
+        shutdown_weak.upgrade().is_none(),
+        "dropping cache when maintenance is aborted must still release descriptors"
+    );
+    assert!(
+        tracker_weak.upgrade().is_none(),
+        "dropping cache must release FdCacheTracker without reference cycles"
+    );
+    drop(cloned_moka);
+
+    // Short-TTL expired entries (including mixed expired and fresh entries) must
+    // be retired by invalidate_under and clear() even though Moka's iterator skips
+    // expired entries.
+    let ttl_cache = FdCache::with_ttl(std::time::Duration::from_millis(15));
+    let expired_file = Arc::new(tempfile::tempfile().expect("create short-ttl descriptor"));
+    let expired_weak = Arc::downgrade(&expired_file);
+    ttl_cache.insert(key("v", "ttl-obj/part.1", false), expired_file).await;
+    tokio::time::sleep(std::time::Duration::from_millis(30)).await;
+    let fresh_file = Arc::new(tempfile::tempfile().expect("create fresh descriptor"));
+    let fresh_weak = Arc::downgrade(&fresh_file);
+    ttl_cache.insert(key("v", "ttl-obj/part.2", false), fresh_file).await;
+    ttl_cache.invalidate_under("v", "ttl-obj");
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while expired_weak.upgrade().is_some() || fresh_weak.upgrade().is_some() {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("invalidate_under must retire both expired and fresh matching descriptors");
+    assert_eq!(ttl_cache.tracker.live_entries(), 0);
+
+    let clear_expired_file = Arc::new(tempfile::tempfile().expect("create clear short-ttl descriptor"));
+    let clear_expired_weak = Arc::downgrade(&clear_expired_file);
+    ttl_cache
+        .insert(key("v", "ttl-clear/part.1", false), clear_expired_file)
+        .await;
+    tokio::time::sleep(std::time::Duration::from_millis(30)).await;
+    let clear_live_file = Arc::new(tempfile::tempfile().expect("create clear live descriptor"));
+    let clear_live_weak = Arc::downgrade(&clear_live_file);
+    ttl_cache.insert(key("v", "ttl-clear/part.2", false), clear_live_file).await;
+    ttl_cache.clear();
+    assert!(
+        clear_expired_weak.upgrade().is_none() && clear_live_weak.upgrade().is_none(),
+        "clear() must synchronously retire both TTL-expired and live descriptors"
+    );
+    assert_eq!(ttl_cache.tracker.live_entries(), 0);
+}
+
+#[tokio::test]
+async fn fd_cache_lockstep_oracle_equivalence_and_counter_conservation() {
+    use std::collections::HashMap;
+    use std::sync::atomic::Ordering;
+
+    let (_directory, file) = fixture();
+    let cache = FdCache::with_ttl(std::time::Duration::from_secs(3600));
+    let mut oracle: HashMap<FdKey, Arc<FdCacheEntry>> = HashMap::new();
+
+    let volumes = ["vol-a", "vol-b", "vol-c", ".rustfs.sys/multipart"];
+    let paths = [
+        "a/b",
+        "a/b/part.1",
+        "a/b/dir/part.2",
+        "a/bc",
+        "a/bc/part.1",
+        "prefix/obj-0001.bin/0198a3f0/part.1",
+        "prefix/obj-0002.bin/0198a3f0/part.1",
+        "d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13/part.1",
+    ];
+    let prefixes = [
+        "a",
+        "a/",
+        "a/b",
+        "a/b/",
+        "a/b/dir",
+        "a/bc",
+        "prefix",
+        "prefix/obj-0001.bin",
+        "prefix/obj-0001.bin/0198a3f0",
+        "d1/d2/d3/d4/d5/c_missing",
+        "d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13",
+        "nonexistent/prefix",
+    ];
+
+    let mut candidate_keys = Vec::new();
+    for v in volumes {
+        for p in paths {
+            for direct in [false, true] {
+                candidate_keys.push(key(v, p, direct));
+            }
+        }
+    }
+
+    let mut rng = 0x9e37_79b9_7f4a_7c15u64;
+    let mut next_u64 = || {
+        rng ^= rng << 13;
+        rng ^= rng >> 7;
+        rng ^= rng << 17;
+        rng
+    };
+
+    for step in 0..1000 {
+        let op = next_u64() % 100;
+        if op < 45 {
+            // Insert or replace (either fresh or stale generation)
+            let k = candidate_keys[(next_u64() as usize) % candidate_keys.len()].clone();
+            let entry = Arc::new(FdCacheEntry {
+                file: Arc::clone(&file),
+                len: step,
+                invalidated: AtomicBool::new(false),
+            });
+            let cur_gen = cache.generation();
+            let stale = (next_u64() % 5) == 0 && cur_gen > 0;
+            let gen_to_use = if stale { cur_gen - 1 } else { cur_gen };
+            cache.insert_if_fresh(k.clone(), Arc::clone(&entry), gen_to_use).await;
+            if !stale {
+                oracle.insert(k, entry);
+            }
+        } else if op < 65 {
+            // Exact invalidation
+            let v = volumes[(next_u64() as usize) % volumes.len()];
+            let p = paths[(next_u64() as usize) % paths.len()];
+            cache.invalidate_exact(v, p).await;
+            oracle.remove(&key(v, p, false));
+            oracle.remove(&key(v, p, true));
+        } else if op < 88 {
+            // Prefix invalidation
+            let v = volumes[(next_u64() as usize) % volumes.len()];
+            let raw_prefix = prefixes[(next_u64() as usize) % prefixes.len()];
+            cache.invalidate_under(v, raw_prefix);
+            let trimmed = raw_prefix.trim_end_matches('/');
+            oracle.retain(|k, _| {
+                !(k.volume == v && (k.path == trimmed || k.path.strip_prefix(trimmed).is_some_and(|r| r.starts_with('/'))))
+            });
+        } else if op < 97 {
+            // Volume invalidation
+            let v = volumes[(next_u64() as usize) % volumes.len()];
+            cache.invalidate_volume(v);
+            oracle.retain(|k, _| k.volume != v);
+        } else {
+            // Full clear
+            cache.clear();
+            oracle.clear();
+        }
+
+        // Verify lockstep equivalence against the reference oracle after every operation.
+        for k in &candidate_keys {
+            let got = cache.get(k).await;
+            let expected = oracle.get(k);
+            match (got, expected) {
+                (Some(g), Some(e)) => assert!(
+                    Arc::ptr_eq(&g, e),
+                    "step {step}: pointer mismatch for key ({}, {}, {})",
+                    k.volume,
+                    k.path,
+                    k.direct
+                ),
+                (None, None) => {}
+                (g, e) => panic!(
+                    "step {step}: oracle mismatch for key ({}, {}, {}): got={}, expected={}",
+                    k.volume,
+                    k.path,
+                    k.direct,
+                    g.is_some(),
+                    e.is_some()
+                ),
+            }
+        }
+    }
+
+    // Clear and wait for maintenance to drain all entries, then verify exact counter conservation.
+    cache.clear();
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("clear must drain all live entries and pending invalidations");
+
+    assert_eq!(cache.tracker.live_entries(), 0);
+    assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
+    assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn fd_cache_concurrent_insert_and_invalidation_stress() {
+    use std::sync::atomic::{AtomicU64, Ordering};
+
+    let (_directory, file) = fixture();
+    let cache = Arc::new(FdCache::new());
+
+    // Deterministic Dekker-partner interleaving check: pause `CachedFdEntry::new`
+    // at `entry_shards[shard].lock()` after it has passed the initial generation
+    // check and incremented `volume_counts`, bump generation via an unmatched
+    // invalidation, and verify the post-`new()` recheck refuses publication and
+    // preserves the existing entry.
+    let seam_key = key("v-seam", "obj-1/part.1", false);
+    cache.insert(seam_key.clone(), Arc::clone(&file)).await;
+    let initial_entry = cache.get(&seam_key).await.expect("initial entry must exist");
+    let (vol_hash, vol_slot) = super::fd_cache_volume_hash_and_slot("v-seam");
+    let entry_shard = (vol_hash as usize) & (super::FD_CACHE_PREFIX_SHARDS - 1);
+    let stale_gen = cache.generation();
+    let stale_entry = Arc::new(FdCacheEntry {
+        file: Arc::clone(&file),
+        len: 999_999,
+        invalidated: AtomicBool::new(false),
+    });
+    let blocked_insert = {
+        let hold = cache.tracker.entry_shards[entry_shard].lock();
+        let cache_clone = Arc::clone(&cache);
+        let seam_key_clone = seam_key.clone();
+        let task = tokio::spawn(async move {
+            cache_clone.insert_if_fresh(seam_key_clone, stale_entry, stale_gen).await;
+        });
+        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+        while cache.tracker.volume_counts[usize::from(vol_slot)].load(Ordering::SeqCst) < 2 {
+            assert!(
+                std::time::Instant::now() < deadline,
+                "insert never reached CachedFdEntry::new with the previous entry still live"
+            );
+            std::thread::yield_now();
+        }
+        cache.invalidate_under("v-seam", "unmatched-prefix");
+        drop(hold);
+        task
+    };
+    blocked_insert.await.expect("blocked insert task should complete");
+    let preserved = cache
+        .get(&seam_key)
+        .await
+        .expect("refused stale insert must not retire a valid previous entry");
+    assert!(Arc::ptr_eq(&preserved, &initial_entry));
+    cache.invalidate_volume("v-seam");
+
+    let floor_v1 = Arc::new(AtomicU64::new(0));
+    let floor_v2 = Arc::new(AtomicU64::new(0));
+    let mut handles = Vec::new();
+
+    for t in 0..8usize {
+        let cache = Arc::clone(&cache);
+        let file = Arc::clone(&file);
+        let floor_v1 = Arc::clone(&floor_v1);
+        let floor_v2 = Arc::clone(&floor_v2);
+        handles.push(tokio::spawn(async move {
+            for i in 0..250u64 {
+                let is_v1 = (t + i as usize).is_multiple_of(2);
+                let vol = if is_v1 { "v1" } else { "v2" };
+                let path = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
+                let k = key(vol, &path, i.is_multiple_of(2));
+                let g = cache.generation();
+                let entry = Arc::new(FdCacheEntry {
+                    file: Arc::clone(&file),
+                    len: g,
+                    invalidated: AtomicBool::new(false),
+                });
+                cache.insert_if_fresh(k.clone(), entry, g).await;
+                let min_gen = if is_v1 {
+                    floor_v1.load(Ordering::SeqCst)
+                } else {
+                    floor_v2.load(Ordering::SeqCst)
+                };
+                if let Some(hit) = cache.get(&k).await {
+                    assert!(
+                        hit.len >= min_gen,
+                        "stale descriptor served after volume invalidation: opened_gen={}, floor_gen={min_gen}",
+                        hit.len
+                    );
+                }
+            }
+        }));
+    }
+
+    for t in 0..4usize {
+        let cache = Arc::clone(&cache);
+        let floor_v1 = Arc::clone(&floor_v1);
+        let floor_v2 = Arc::clone(&floor_v2);
+        handles.push(tokio::spawn(async move {
+            for i in 0..250u64 {
+                let is_v1 = (t + i as usize).is_multiple_of(2);
+                let vol = if is_v1 { "v1" } else { "v2" };
+                match i % 4 {
+                    0 => cache.invalidate_under(vol, &format!("dir-{}/obj-{}", t % 4, i % 16)),
+                    1 => {
+                        let p = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
+                        cache.invalidate_exact(vol, &p).await;
+                    }
+                    2 => {
+                        let before = cache.generation();
+                        cache.invalidate_volume(vol);
+                        let floor = before + 1;
+                        if is_v1 {
+                            floor_v1.fetch_max(floor, Ordering::SeqCst);
+                        } else {
+                            floor_v2.fetch_max(floor, Ordering::SeqCst);
+                        }
+                    }
+                    _ => cache.invalidate_under("uncached-vol", "uncached/prefix"),
+                }
+                tokio::task::yield_now().await;
+            }
+        }));
+    }
+
+    for h in handles {
+        h.await.expect("worker task should succeed");
+    }
+
+    cache.clear();
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("clear after concurrent stress must drain all live entries");
+
+    assert_eq!(cache.tracker.live_entries(), 0);
+    assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
+    assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
+}

crates/ecstore/src/disk/fs.rs

@@ -148,47 +148,58 @@ pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
     native_io(move || create_dir_all_std(&path)).await
 }
 
-fn is_dir_error(e: &io::Error) -> bool {
+fn is_dir_error(path: &Path, e: &io::Error) -> bool {
     e.raw_os_error() == Some(libc::EISDIR)
         || e.kind() == io::ErrorKind::IsADirectory
         // macOS: remove_file on a directory returns EPERM
         || (cfg!(target_os = "macos") && e.raw_os_error() == Some(libc::EPERM))
+        || is_windows_dir_access_denied(path, e)
+}
+
+/// Windows reports `remove_file` on a directory (or directory symlink) as
+/// ERROR_ACCESS_DENIED, which is indistinguishable from a real permission
+/// failure without inspecting the entry itself.
+#[cfg(windows)]
+fn is_windows_dir_access_denied(path: &Path, e: &io::Error) -> bool {
+    use std::os::windows::fs::FileTypeExt;
+
+    e.kind() == io::ErrorKind::PermissionDenied
+        && std::fs::symlink_metadata(path).is_ok_and(|metadata| {
+            let file_type = metadata.file_type();
+            file_type.is_dir() || file_type.is_symlink_dir()
+        })
+}
+
+#[cfg(not(windows))]
+fn is_windows_dir_access_denied(_path: &Path, _e: &io::Error) -> bool {
+    false
 }
 
 #[tracing::instrument(level = "debug", skip_all)]
 pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
-    // Try remove_file first; fall back to remove_dir if it's a directory
-    let file_path = path.as_ref().to_path_buf();
-    match native_io(move || std::fs::remove_file(file_path)).await {
-        Ok(()) => Ok(()),
-        Err(e) if is_dir_error(&e) => {
-            let path = path.as_ref().to_path_buf();
-            native_io(move || std::fs::remove_dir(path)).await
-        }
-        Err(e) => Err(e),
-    }
+    let path = path.as_ref().to_path_buf();
+    native_io(move || remove_std(path)).await
 }
 
 #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
 pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
     // Try remove_file first; fall back to remove_dir_all if it's a directory
-    let file_path = path.as_ref().to_path_buf();
-    match native_io(move || std::fs::remove_file(file_path)).await {
+    let path = path.as_ref().to_path_buf();
+    native_io(move || match std::fs::remove_file(&path) {
         Ok(()) => Ok(()),
-        Err(e) if is_dir_error(&e) => {
-            let path = path.as_ref().to_path_buf();
-            native_io(move || std::fs::remove_dir_all(path)).await
-        }
+        Err(e) if is_dir_error(&path, &e) => std::fs::remove_dir_all(&path),
         Err(e) => Err(e),
-    }
+    })
+    .await
 }
 
 #[tracing::instrument(level = "debug", skip_all)]
 pub fn remove_std(path: impl AsRef<Path>) -> io::Result<()> {
     // Try remove_file first; fall back to remove_dir if it's a directory
-    match std::fs::remove_file(path.as_ref()) {
+    let path = path.as_ref();
+    match std::fs::remove_file(path) {
         Ok(()) => Ok(()),
-        Err(e) if is_dir_error(&e) => std::fs::remove_dir(path.as_ref()),
+        Err(e) if is_dir_error(path, &e) => std::fs::remove_dir(path),
         Err(e) => Err(e),
     }
 }