test(gateway): assert the store-fault answer names nothing

The 500 answer to a credential-store fault is checked to carry neither the looked-up key, nor a secret, nor a reason for the failed lookup. The test comment records that the earlier pin answered 403.

.config/make/lint-fmt.mak

@@ -70,11 +70,21 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
 	@echo "📦 Checking s3s footprint ratchet..."
 	./scripts/check_s3s_footprint.sh
 
+.PHONY: gateway-stack-release-boundary-check
+gateway-stack-release-boundary-check: ## Check no release build can enable the gateway-stack feature
+	@echo "🚧 Checking gateway-stack release boundary..."
+	./scripts/check_gateway_stack_release_boundary.sh
+
 .PHONY: test-count-ratchet-check
 test-count-ratchet-check: ## Check no workspace crate fell below its test floor or rose above its #[ignore] ceiling in the committed baseline
 	@echo "🧪 Checking per-crate test-count ratchet..."
 	./scripts/check_test_count_ratchet.sh
 
+.PHONY: test-seams-check
+test-seams-check: ## Check tests outside an operation's owner call its use-case only through rustfs/src/app/test_seams
+	@echo "🧪 Checking cross-owner test seams..."
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_seams.py
+
 .PHONY: fips-wording-check
 fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities
 	@echo "📣 Checking cryptographic capability wording guard..."

.config/make/pre-commit.mak

@@ -20,11 +20,11 @@ planning-docs-check: ## Check that no planning-type documents are committed
 	./scripts/check_no_planning_docs.sh
 
 .PHONY: pre-commit
-pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
+pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check gateway-stack-release-boundary-check test-count-ratchet-check test-seams-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
 	@echo "✅ All pre-commit checks passed!"
 
 .PHONY: pre-pr
-pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
+pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check gateway-stack-release-boundary-check test-count-ratchet-check test-seams-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
 	@echo "✅ All pre-PR checks passed!"
 
 .PHONY: dev-check

.config/make/tests.mak

@@ -61,11 +61,15 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_check_doc_paths.py
 	./scripts/check_embedded_secrets.sh --self-test
 	./scripts/check_s3s_footprint.sh --self-test
+	./scripts/check_gateway_stack_release_boundary.sh --self-test
 	./scripts/test_check_test_count_ratchet.sh
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/codemods/split_lifecycle_transition_api_test.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_seams.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_download_release_asset.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_reference_binary.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
 	$(RUSTFS_PYTHON_BIN) -m unittest discover -s scripts/release -p "test_*.py"
@@ -92,6 +96,7 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_junit_diff.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_reference_parity.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
 	./scripts/validate_object_data_cache_cold_stampede.sh --self-test

.config/reference-binary.toml

@@ -0,0 +1,20 @@
+# Pinned RustFS release used as the "reference" target of parity lanes
+# (.github/workflows/reference-parity.yml). This is the single place that names
+# the reference release; .github/actions/reference-binary resolves, downloads and
+# verifies it, and fails on a missing pin, a missing asset or a digest mismatch.
+#
+# Bumping the reference is a reviewed event: every per-test difference between
+# the candidate and the reference is measured against this exact build.
+#
+# Each sha256 is the value published in the release's SHA256SUMS asset, which
+# matches the digest GitHub reports for the same asset. The action checks both.
+
+schema = 1
+repository = "rustfs/rustfs"
+version = "1.0.5"
+tag = "1.0.5"
+
+[platforms.linux-x86_64-gnu]
+asset = "rustfs-linux-x86_64-gnu-v1.0.5.zip"
+sha256 = "5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0"
+executable = "rustfs"

.config/reference-parity-targets.toml

@@ -0,0 +1,24 @@
+# Targets of the s3-tests reference parity lane (.github/workflows/reference-parity.yml).
+#
+# Exactly one target uses `binary = "reference"` (the release pinned in
+# .config/reference-binary.toml). Every other target uses `binary = "candidate"`
+# (the binary built from the branch under test) and is compared per node id
+# against the reference. All targets run the same harness, test list and server
+# configuration; `env` adds only the target's own RUSTFS_* settings.
+#
+# Adding a target is one more [[targets]] table, for example:
+#
+#   [[targets]]
+#   name = "candidate-gateway"
+#   binary = "candidate"
+#   env = { RUSTFS_S3_STACK = "gateway" }
+
+schema = 1
+
+[[targets]]
+name = "reference"
+binary = "reference"
+
+[[targets]]
+name = "candidate-legacy"
+binary = "candidate"

.config/reference-parity-tolerated.txt

@@ -0,0 +1,12 @@
+# Tolerated per-test differences of the s3-tests reference parity lane.
+#
+# One entry per line, four fields separated by " | ":
+#
+#   <target> | <pytest node id> | <tracking issue URL> | <reason>
+#
+# <target> is a candidate name from .config/reference-parity-targets.toml, the
+# node id is exactly as the lane prints it (s3tests/functional/test_s3.py::...),
+# and the issue URL is https://github.com/rustfs/<repo>/issues/<number>.
+#
+# Same semantics as scripts/junit_diff.py --allow: an entry whose difference no
+# longer occurs is stale and fails the lane, so remove it when it is fixed.

.config/s3s-footprint-baseline.txt

@@ -13,5 +13,5 @@
 #                 header's rule clears (rustfs_s3_types's macro); fails closed
 # ecstore_files   Rust files referencing s3s paths under crates/ecstore/src
 files=65
-s3_error_lines=219
+s3_error_lines=217
 ecstore_files=0

.config/test-count-baseline.txt

@@ -12,7 +12,7 @@
 # Regenerate with:
 #   scripts/check_test_count_ratchet.sh --update
 e2e_test	1026	20
-rustfs	6440	64
+rustfs	6449	64
 rustfs-audit	70	0
 rustfs-checksums	28	1
 rustfs-common	36	0
@@ -25,6 +25,7 @@ rustfs-data-usage	58	0
 rustfs-ecstore	7858	2
 rustfs-extension-schema	23	0
 rustfs-filemeta	361	0
+rustfs-gateway-adapter	74	0
 rustfs-heal	1128	3
 rustfs-heal-contracts	8	0
 rustfs-iam	415	0

.github/actions/quick-checks/action.yml

@@ -97,10 +97,18 @@ runs:
       shell: bash
       run: ./scripts/check_s3s_footprint.sh
 
+    - name: Check gateway-stack release boundary
+      shell: bash
+      run: ./scripts/check_gateway_stack_release_boundary.sh
+
     - name: Check per-crate test-count ratchet
       shell: bash
       run: ./scripts/check_test_count_ratchet.sh
 
+    - name: Check cross-owner tests use test seams
+      shell: bash
+      run: python3 ./scripts/check_test_seams.py
+
     - name: Check cryptographic capability wording
       shell: bash
       run: ./scripts/check_fips_wording.sh

.github/actions/reference-binary/action.yml

@@ -0,0 +1,96 @@
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "Reference RustFS binary"
+description: "Download the pinned reference RustFS release, verify its sha256 and expose the executable path. A missing pin, missing asset or digest mismatch fails the step."
+
+inputs:
+  pin-file:
+    description: "Pin file naming the reference release (repository-relative)"
+    required: false
+    default: ".config/reference-binary.toml"
+  platform:
+    description: "Platform table in the pin file"
+    required: false
+    default: "linux-x86_64-gnu"
+  destination:
+    description: "Directory that receives the executable; defaults to $RUNNER_TEMP/reference-binary"
+    required: false
+    default: ""
+
+outputs:
+  path:
+    description: "Absolute path of the verified executable"
+    value: ${{ steps.unpack.outputs.path }}
+  version:
+    description: "Pinned reference version"
+    value: ${{ steps.pin.outputs.version }}
+  sha256:
+    description: "Pinned and verified archive sha256"
+    value: ${{ steps.pin.outputs.sha256 }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Resolve reference pin
+      id: pin
+      shell: bash
+      env:
+        PIN_TOOL: ${{ github.action_path }}/reference_binary.py
+        PIN_FILE: ${{ inputs.pin-file }}
+        PIN_PLATFORM: ${{ inputs.platform }}
+      run: |
+        set -euo pipefail
+        resolved="$(python3 "$PIN_TOOL" resolve --pin "$PIN_FILE" --platform "$PIN_PLATFORM")"
+        printf '%s\n' "$resolved"
+        printf '%s\n' "$resolved" >> "$GITHUB_OUTPUT"
+
+    - name: Download and verify reference archive
+      id: download
+      shell: bash
+      env:
+        GH_TOKEN: ${{ github.token }}
+        ASSET_DOWNLOADER: ${{ github.action_path }}/../setup/download_release_asset.py
+        REF_REPOSITORY: ${{ steps.pin.outputs.repository }}
+        REF_TAG: ${{ steps.pin.outputs.tag }}
+        REF_ASSET: ${{ steps.pin.outputs.asset }}
+        REF_SHA256: ${{ steps.pin.outputs.sha256 }}
+      run: |
+        set -euo pipefail
+        archive="$RUNNER_TEMP/reference-archive/$REF_ASSET"
+        # Fails unless the release lists exactly one asset of this name whose
+        # published digest equals the pin and whose downloaded bytes match it.
+        python3 "$ASSET_DOWNLOADER" --repository "$REF_REPOSITORY" --tag "$REF_TAG" \
+          --asset "$REF_ASSET" --output "$archive" --sha256 "$REF_SHA256"
+        echo "archive=$archive" >> "$GITHUB_OUTPUT"
+
+    - name: Unpack reference executable
+      id: unpack
+      shell: bash
+      env:
+        PIN_TOOL: ${{ github.action_path }}/reference_binary.py
+        REF_ARCHIVE: ${{ steps.download.outputs.archive }}
+        REF_SHA256: ${{ steps.pin.outputs.sha256 }}
+        REF_EXECUTABLE: ${{ steps.pin.outputs.executable }}
+        REF_DESTINATION: ${{ inputs.destination }}
+      run: |
+        set -euo pipefail
+        destination="${REF_DESTINATION:-$RUNNER_TEMP/reference-binary}"
+        unpacked="$(python3 "$PIN_TOOL" unpack --archive "$REF_ARCHIVE" --sha256 "$REF_SHA256" \
+          --executable "$REF_EXECUTABLE" --destination "$destination")"
+        path="${unpacked#path=}"
+        test -x "$path"
+        echo "Reference executable: $path"
+        "$path" --version
+        echo "path=$path" >> "$GITHUB_OUTPUT"

.github/actions/reference-binary/reference_binary.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Resolve and unpack the pinned reference RustFS release binary.
+
+`resolve` reads the pin file and prints the selected platform's identity as
+`key=value` lines (GitHub step-output format). `unpack` re-verifies the
+downloaded archive digest against the pin and extracts exactly one executable.
+Every missing or malformed input is a failure; nothing is skipped.
+"""
+
+import argparse
+import hashlib
+from pathlib import Path
+import re
+import sys
+import tomllib
+import zipfile
+
+REPOSITORY = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*")
+TAG = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+-]*")
+PLATFORM = re.compile(r"[a-z0-9][a-z0-9_-]*")
+ASSET = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+-]*\.zip")
+SHA256 = re.compile(r"[0-9a-f]{64}")
+EXECUTABLE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*")
+MAX_EXECUTABLE_BYTES = 1024 * 1024 * 1024
+
+
+class PinError(Exception):
+    """A pin, archive or argument that cannot be trusted."""
+
+
+def field(table, key, pattern, where):
+    value = table.get(key)
+    if not isinstance(value, str) or not pattern.fullmatch(value):
+        raise PinError(f"{where}: missing or invalid '{key}'")
+    return value
+
+
+def resolve(pin_path, platform):
+    if not PLATFORM.fullmatch(platform or ""):
+        raise PinError("invalid platform name")
+    try:
+        data = tomllib.loads(Path(pin_path).read_text(encoding="utf-8"))
+    except FileNotFoundError as exc:
+        raise PinError(f"{pin_path}: reference pin file is missing") from exc
+    except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError) as exc:
+        raise PinError(f"{pin_path}: cannot read reference pin: {exc}") from exc
+    if data.get("schema") != 1 or type(data.get("schema")) is not int:
+        raise PinError(f"{pin_path}: unsupported pin schema")
+    identity = {
+        "repository": field(data, "repository", REPOSITORY, pin_path),
+        "version": field(data, "version", TAG, pin_path),
+        "tag": field(data, "tag", TAG, pin_path),
+    }
+    platforms = data.get("platforms")
+    if not isinstance(platforms, dict) or not platforms:
+        raise PinError(f"{pin_path}: no pinned platforms")
+    entry = platforms.get(platform)
+    if not isinstance(entry, dict):
+        raise PinError(f"{pin_path}: no pin for platform '{platform}'")
+    where = f"{pin_path} [platforms.{platform}]"
+    identity["asset"] = field(entry, "asset", ASSET, where)
+    identity["sha256"] = field(entry, "sha256", SHA256, where)
+    identity["executable"] = field(entry, "executable", EXECUTABLE, where)
+    if identity["version"] not in identity["asset"]:
+        raise PinError(f"{where}: asset name does not carry version {identity['version']}")
+    unknown = set(entry) - {"asset", "sha256", "executable"}
+    if unknown:
+        raise PinError(f"{where}: unknown keys {sorted(unknown)}")
+    return identity
+
+
+def file_sha256(path):
+    with open(path, "rb") as stream:
+        return hashlib.file_digest(stream, "sha256").hexdigest()
+
+
+def unpack(archive, sha256, executable, destination):
+    if not SHA256.fullmatch(sha256 or "") or not EXECUTABLE.fullmatch(executable or ""):
+        raise PinError("invalid expected digest or executable name")
+    try:
+        actual = file_sha256(archive)
+    except OSError as exc:
+        raise PinError(f"{archive}: reference archive is missing or unreadable") from exc
+    if actual != sha256:
+        raise PinError(f"{archive}: sha256 {actual} does not match the pinned digest")
+    try:
+        with zipfile.ZipFile(archive) as bundle:
+            matches = [info for info in bundle.infolist() if info.filename == executable]
+            if len(matches) != 1:
+                raise PinError(f"{archive}: must contain exactly one top-level '{executable}'")
+            info = matches[0]
+            if info.is_dir() or not 0 < info.file_size <= MAX_EXECUTABLE_BYTES:
+                raise PinError(f"{archive}: '{executable}' is not a plausible executable")
+            destination = Path(destination)
+            destination.mkdir(parents=True, exist_ok=True)
+            target = destination / executable
+            with bundle.open(info) as source, open(target, "wb") as sink:
+                while chunk := source.read(1024 * 1024):
+                    sink.write(chunk)
+    except zipfile.BadZipFile as exc:
+        raise PinError(f"{archive}: not a valid zip archive") from exc
+    target.chmod(0o755)
+    return target
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    sub = parser.add_subparsers(dest="mode", required=True)
+    res = sub.add_parser("resolve")
+    res.add_argument("--pin", required=True)
+    res.add_argument("--platform", required=True)
+    unp = sub.add_parser("unpack")
+    unp.add_argument("--archive", required=True)
+    unp.add_argument("--sha256", required=True)
+    unp.add_argument("--executable", required=True)
+    unp.add_argument("--destination", required=True)
+    args = parser.parse_args(argv)
+    try:
+        if args.mode == "resolve":
+            for key, value in resolve(args.pin, args.platform).items():
+                print(f"{key}={value}")
+        else:
+            print(f"path={unpack(args.archive, args.sha256, args.executable, args.destination)}")
+    except PinError as exc:
+        print(f"reference binary: {exc}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scheduled-validations.json

@@ -869,6 +869,33 @@
       "failure_owner": "watchdog",
       "origin": "baseline"
     },
+    {
+      "workflow": ".github/workflows/reference-parity.yml",
+      "name": "Reference Parity",
+      "roles": [
+        "pr",
+        "periodic",
+        "manual"
+      ],
+      "events": [
+        "pull_request",
+        "schedule",
+        "workflow_dispatch"
+      ],
+      "cadence": "nightly",
+      "timezone": "UTC",
+      "schedule": [
+        "37 1 * * *"
+      ],
+      "owner": "@overtrue",
+      "policy": "active",
+      "monitor": "workflow",
+      "max_age_hours": 36,
+      "result_owner": "reference-parity.yml",
+      "disposition": "s3-tests per-node-id parity of branch candidates against the pinned reference release",
+      "failure_owner": "watchdog",
+      "origin": "added"
+    },
     {
       "workflow": ".github/workflows/functional-chain-health.yml",
       "name": "Functional chain health",
@@ -994,6 +1021,26 @@
       "policy_reason": "Preserve operator-disabled state; enabling requires a separate policy decision.",
       "origin": "baseline"
     },
+    {
+      "workflow": ".github/workflows/manual-main-packages.yml",
+      "name": "Manual main packages",
+      "roles": [
+        "manual"
+      ],
+      "events": [
+        "workflow_dispatch"
+      ],
+      "cadence": "event",
+      "timezone": "UTC",
+      "schedule": [],
+      "owner": "@overtrue",
+      "policy": "active",
+      "monitor": "none",
+      "result_owner": "manual-main-packages.yml",
+      "disposition": "Packages the origin/main tip as workflow artifacts only. No GitHub Release or download-channel writes.",
+      "monitor_reason": "Operator-requested packaging has no periodic freshness obligation.",
+      "origin": "manual-main-packages"
+    },
     {
       "workflow": ".github/workflows/minio-interop.yml",
       "name": "minio-interop",

.github/workflows/ci.yml

@@ -115,7 +115,7 @@ jobs:
             printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
             echo "Base CI policy unavailable; running the full matrix."
           fi
-          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
+          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=4$' "$GITHUB_OUTPUT"; then
             printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
             echo "Base policy does not cover this workflow contract; retaining the full matrix."
           fi
@@ -705,6 +705,53 @@ jobs:
           retention-days: 3
           if-no-files-found: warn
 
+  test-and-lint-gateway-stack:
+    name: Test and Lint (gateway-stack)
+    # The gateway S3 stack (rustfs/backlog#2734) sits behind the rustfs
+    # `gateway-stack` feature, which no other pull-request lane lints or tests.
+    # Unlike the dormant feature lanes above, this one runs for every code
+    # change: each operation slice puts its handler tests, its authorization
+    # census and its end-to-end refusal test behind the feature, and a lane that
+    # ran only on demand would let them rot behind a green pull request.
+    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    needs: [ quick-checks, classify-changes ]
+    # Self-hosted for build scale and cache ownership: a workspace-scale Cargo
+    # build that reads the ci-dev cache written on this image.
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-dev
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
+
+      # A feature that does not compile fails here; nothing below is skipped
+      # into a pass.
+      - name: Run clippy with gateway-stack
+        run: cargo clippy --locked -p rustfs --all-targets --features gateway-stack -- -D warnings
+
+      # The gateway stack's own tests: the assembly, the authorizer table and
+      # its census, the handler template, the migrated use cases, and the
+      # embedded server driven through the whole middleware stack.
+      - name: Run tests with gateway-stack
+        env:
+          # The same bounded link concurrency as the other feature lanes.
+          CARGO_BUILD_JOBS: "2"
+        run: |
+          cargo nextest run --profile ci --locked -p rustfs --features gateway-stack --no-tests fail \
+            --status-level all --final-status-level all \
+            -E 'test(/gateway_stack|gateway_handlers|server::stack::|list_buckets|legacy_stack_is_always_selectable/) | binary(embedded_gateway_stack_test)'
+
   build-rustfs-debug-binary:
     name: Build RustFS Debug Binary
     if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
@@ -1579,6 +1626,7 @@ jobs:
       - fips-provider
       - test-and-lint-rio-v2
       - test-and-lint-protocols
+      - test-and-lint-gateway-stack
       - build-rustfs-debug-binary
       - uring-integration
       - e2e-tests
@@ -1613,6 +1661,7 @@ jobs:
       - core-contract
       - test-and-lint-rio-v2
       - test-and-lint-protocols
+      - test-and-lint-gateway-stack
       - build-rustfs-debug-binary
       - build-rustfs-debug-binary-rio-v2
       - uring-integration

.github/workflows/manual-main-packages.yml

@@ -0,0 +1,415 @@
+# Copyright 2026 RustFS Team
+# SPDX-License-Identifier: Apache-2.0
+
+# Manual packaging of the current origin/main tip. Workflow artifacts only.
+# Dispatch uses this file from the selected branch. The build fetches
+# refs/heads/main and checks out that commit, not the dispatch ref or a tag.
+
+name: Manual main packages
+
+on:
+  workflow_dispatch:
+    inputs:
+      architecture:
+        description: "Linux targets: x86 (x86_64 musl+gnu), arm (aarch64 musl+gnu), or all"
+        required: true
+        type: choice
+        options:
+          - x86
+          - arm
+          - all
+        default: all
+      version:
+        description: "Binary and package version: MAJOR.MINOR.PATCH, or MAJOR.MINOR.PATCH-(alpha|beta|rc).N. No leading v."
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+  CARGO_INCREMENTAL: 0
+
+jobs:
+  prepare:
+    name: Resolve main tip and version
+    if: github.repository == 'rustfs/rustfs'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    outputs:
+      source_sha: ${{ steps.pin.outputs.source_sha }}
+      version: ${{ steps.version.outputs.version }}
+      toolchain: ${{ steps.inputs.outputs.toolchain }}
+      console: ${{ steps.inputs.outputs.console }}
+      matrix: ${{ steps.matrix.outputs.matrix }}
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          ref: main
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Pin the tip of origin/main
+        id: pin
+        shell: bash
+        run: |
+          set -euo pipefail
+          GIT_TERMINAL_PROMPT=0 git fetch --no-tags origin refs/heads/main
+          sha=$(git rev-parse FETCH_HEAD)
+          git checkout --detach "$sha"
+          test "$(git rev-parse HEAD)" = "$sha"
+          printf 'source_sha=%s\n' "$sha" >> "$GITHUB_OUTPUT"
+          printf 'Pinned origin/main %s\n' "$sha"
+
+      - name: Validate package version
+        id: version
+        shell: bash
+        env:
+          PACKAGE_VERSION: ${{ inputs.version }}
+        run: |
+          set -euo pipefail
+          bash scripts/release/package_manual_artifacts.sh validate \
+            --version "$PACKAGE_VERSION" \
+            --arch x86_64 \
+            --libc gnu | tee "$RUNNER_TEMP/package-version.txt"
+          printf 'version=%s\n' "$PACKAGE_VERSION" >> "$GITHUB_OUTPUT"
+
+      - name: Install stable Rust
+        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+        with:
+          toolchain: stable
+
+      - name: Resolve console assets and toolchain
+        id: inputs
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          toolchain=$(rustc --version | awk '{print $2}')
+          [[ "$toolchain" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
+          gh api repos/rustfs/console/releases/latest > console-release.json
+          TOOLCHAIN="$toolchain" python3 - <<'PY'
+          import json, os, sys
+          sys.path.insert(0, "scripts/release")
+          import candidate
+          console = candidate.console_asset(candidate.read_json("console-release.json"))
+          candidate.emit_output("console", json.dumps(console, separators=(",", ":")))
+          candidate.emit_output("toolchain", os.environ["TOOLCHAIN"])
+          print(f"toolchain={os.environ['TOOLCHAIN']}")
+          print(f"console={console['tag']} {console['name']}")
+          PY
+
+      - name: Select Linux targets
+        id: matrix
+        shell: bash
+        env:
+          ARCHITECTURE: ${{ inputs.architecture }}
+        run: |
+          set -euo pipefail
+          matrix=$(bash scripts/release/package_manual_artifacts.sh matrix "$ARCHITECTURE")
+          count=$(python3 -c 'import json,sys; print(len(json.loads(sys.argv[1])["include"]))' "$matrix")
+          [[ "$count" -gt 0 ]]
+          printf 'matrix=%s\n' "$matrix" >> "$GITHUB_OUTPUT"
+          printf 'Selected %s target(s) for %s\n' "$count" "$ARCHITECTURE"
+
+  package:
+    name: Package RustFS (${{ matrix.target_id }})
+    needs: prepare
+    if: github.repository == 'rustfs/rustfs'
+    # x86_64 uses sm-standard-4, the official release compile pool: the workspace
+    # release build needs that disk (placement rule: disk) and reads build.yml's
+    # cache without writing it. aarch64 uses ubicloud-standard-16-arm so the
+    # packaged binary and console smoke execute natively (placement rule: hardware).
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 210
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      RUSTUP_TOOLCHAIN: ${{ needs.prepare.outputs.toolchain }}
+      RUSTFLAGS: "${{ matrix.rustflags }}"
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.prepare.outputs.matrix) }}
+    steps:
+      - name: Checkout the pinned origin/main commit
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          ref: ${{ needs.prepare.outputs.source_sha }}
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Confirm the checkout is that main commit
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
+        run: |
+          set -euo pipefail
+          test "$(git rev-parse HEAD)" = "$SOURCE_SHA"
+          GIT_TERMINAL_PROMPT=0 git fetch --no-tags origin refs/heads/main
+          git merge-base --is-ancestor "$SOURCE_SHA" FETCH_HEAD
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: ${{ needs.prepare.outputs.toolchain }}
+          target: ${{ matrix.target }}
+          cache-shared-key: build-${{ matrix.target }}${{ matrix.glibc && format('-glibc-{0}', matrix.glibc) || '' }}${{ matrix.os == 'ubicloud-standard-16-arm' && '-ubicloud-arm' || '' }}
+          # build.yml owns this key. This job only restores it.
+          cache-save-if: 'false'
+          install-cross-tools: ${{ matrix.cross }}
+          install-build-packaging-tools: 'true'
+          install-test-tools: 'false'
+
+      - name: Download static console assets
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          CONSOLE_IDENTITY: ${{ needs.prepare.outputs.console }}
+        run: |
+          set -euo pipefail
+          python3 - <<'PYCONSOLE'
+          import json, os, pathlib, subprocess, zipfile
+          value = json.loads(os.environ["CONSOLE_IDENTITY"])
+          subprocess.run(["python3", ".github/actions/setup/download_release_asset.py",
+              "--repository", "rustfs/console", "--tag", value["tag"], "--asset", value["name"],
+              "--sha256", value["sha256"], "--output", "console.zip"], check=True)
+          destination = pathlib.Path("rustfs/static")
+          destination.mkdir(parents=True, exist_ok=True)
+          with zipfile.ZipFile("console.zip") as archive:
+              for member in archive.infolist():
+                  path = pathlib.PurePosixPath(member.filename)
+                  if path.is_absolute() or ".." in path.parts or "\\" in member.filename:
+                      raise SystemExit("unsafe Console archive member")
+              archive.extractall(destination)
+          if not (destination / "index.html").is_file():
+              raise SystemExit("Console archive has no index.html")
+          PYCONSOLE
+          rm -f console.zip
+
+      - name: Build RustFS
+        shell: bash
+        env:
+          RUSTFS_BUILD_VERSION: ${{ needs.prepare.outputs.version }}
+          BUILD_TARGET: ${{ matrix.target }}
+          BUILD_OS: ${{ matrix.build_os }}
+          BUILD_FEATURES: ${{ matrix.features }}
+          BUILD_GLIBC: ${{ matrix.glibc }}
+          BUILD_CROSS: ${{ matrix.cross }}
+        run: |
+          set -euo pipefail
+          touch rustfs/build.rs
+
+          feature_args=()
+          if [[ -n "$BUILD_FEATURES" ]]; then
+            feature_args=(--features "$BUILD_FEATURES")
+          fi
+
+          # gnu targets pin glibc 2.31 through cargo-zigbuild (<target>.<glibc>).
+          target_arg="$BUILD_TARGET"
+          if [[ -n "$BUILD_GLIBC" ]]; then
+            target_arg="${target_arg}.${BUILD_GLIBC}"
+          fi
+
+          profile_args=()
+          if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
+            profile_args=(--config 'profile.release.strip="none"')
+          fi
+
+          case "$BUILD_CROSS" in
+            true)
+              cargo zigbuild --release --locked --target "$target_arg" "${feature_args[@]}" "${profile_args[@]}" -p rustfs --bin rustfs
+              ;;
+            false)
+              cargo build --release --locked --target "$target_arg" "${feature_args[@]}" "${profile_args[@]}" -p rustfs --bin rustfs
+              ;;
+            *)
+              printf 'unexpected cross flag: %q\n' "$BUILD_CROSS" >&2
+              exit 1
+              ;;
+          esac
+
+          bin="target/${BUILD_TARGET}/release/rustfs"
+          version_output=$("$bin" --version)
+          first=${version_output%%$'\n'*}
+          if [[ "$first" != "rustfs ${RUSTFS_BUILD_VERSION}" ]]; then
+            printf 'binary version line is %q, expected %q\n' "$first" "rustfs ${RUSTFS_BUILD_VERSION}" >&2
+            printf '%s\n' "$version_output" >&2
+            exit 1
+          fi
+          if ! grep -Fxq "build target : ${BUILD_TARGET}" <<< "$version_output" || \
+             ! grep -Fxq "build os     : ${BUILD_OS}" <<< "$version_output" || \
+             ! grep -Fxq "rust channel : stable-${BUILD_TARGET}" <<< "$version_output"; then
+            printf 'release binary reports an unexpected compile target\n' >&2
+            printf '%s\n' "$version_output" >&2
+            exit 1
+          fi
+          if [[ -n "$BUILD_GLIBC" ]]; then
+            python3 scripts/release/check_glibc.py "$BUILD_GLIBC" "$bin"
+          fi
+
+      - name: Generate the CPU symbol catalog
+        if: endsWith(matrix.target, '-linux-gnu') && matrix.features == 'pyroscope'
+        shell: bash
+        env:
+          BUILD_TARGET: ${{ matrix.target }}
+        run: |
+          set -euo pipefail
+          cargo metadata --locked --format-version 1 --filter-platform "$BUILD_TARGET" \
+            --features rustfs/pyroscope > "$RUNNER_TEMP/cpu-catalog-metadata.json"
+          features=$(jq -ce '
+            (.packages[] | select(.name == "rustfs") | .id) as $id |
+            .resolve.nodes[] | select(.id == $id) | .features | map(select(. != "default")) | sort
+          ' "$RUNNER_TEMP/cpu-catalog-metadata.json")
+          release_dir="target/${BUILD_TARGET}/release"
+          rm -f "$release_dir/rustfs.cpu-symbol-catalog.json" "$release_dir/rustfs.cpu-symbol-catalog.sha256"
+          cargo run --locked -p rustfs-connect-symbol-catalog -- \
+            "$release_dir/rustfs" "$(git rev-parse HEAD)" "$BUILD_TARGET" "$features" \
+            "$release_dir/rustfs.cpu-symbol-catalog.json" > "$release_dir/rustfs.cpu-symbol-catalog.sha256"
+
+      - name: Install DEB and RPM tools
+        shell: bash
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y ruby ruby-dev build-essential rpm fakeroot cpio
+          sudo gem install fpm --version 1.17.0 --no-document
+
+      - name: Package zip, deb, and rpm
+        shell: bash
+        env:
+          PACKAGE_VERSION: ${{ needs.prepare.outputs.version }}
+          ARCH: ${{ matrix.arch }}
+          LIBC: ${{ matrix.libc }}
+          BUILD_TARGET: ${{ matrix.target }}
+          BUILD_FEATURES: ${{ matrix.features }}
+          SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
+        run: |
+          set -euo pipefail
+          binary="target/${BUILD_TARGET}/release/rustfs"
+          extra=()
+          if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
+            extra=(
+              --zip-extra "target/${BUILD_TARGET}/release/rustfs.cpu-symbol-catalog.json"
+              --zip-extra "target/${BUILD_TARGET}/release/rustfs.cpu-symbol-catalog.sha256"
+            )
+          fi
+          bash scripts/release/package_manual_artifacts.sh package \
+            --version "$PACKAGE_VERSION" \
+            --arch "$ARCH" \
+            --libc "$LIBC" \
+            --binary "$binary" \
+            --output dist \
+            "${extra[@]}"
+          printf 'Packaged %s from origin/main %s\n' "$PACKAGE_VERSION" "$SOURCE_SHA"
+
+      - name: Verify the packaged binary
+        shell: bash
+        env:
+          PACKAGE_VERSION: ${{ needs.prepare.outputs.version }}
+          ARCH: ${{ matrix.arch }}
+          LIBC: ${{ matrix.libc }}
+          BUILD_TARGET: ${{ matrix.target }}
+          BUILD_OS: ${{ matrix.build_os }}
+          BUILD_FEATURES: ${{ matrix.features }}
+        run: |
+          set -euo pipefail
+          stem="rustfs-linux-${ARCH}-${LIBC}-v${PACKAGE_VERSION}"
+          package_file="dist/${stem}.zip"
+          [[ -f "$package_file" && -f "dist/${stem}.deb" && -f "dist/${stem}.rpm" ]]
+          [[ "$(find dist -maxdepth 1 -type f | wc -l)" -eq 3 ]]
+          archive_members=$(unzip -Z1 "$package_file" | sort)
+          if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
+            expected_members=$(printf '%s\n' rustfs rustfs.cpu-symbol-catalog.json rustfs.cpu-symbol-catalog.sha256 | sort)
+          else
+            expected_members=$(printf '%s\n' rustfs)
+          fi
+          if [[ "$archive_members" != "$expected_members" ]]; then
+            printf 'Linux package does not contain the expected files\n' >&2
+            printf '%s\n' "$archive_members" >&2
+            exit 1
+          fi
+          binary_dir=$(mktemp -d)
+          trap 'rm -rf "$binary_dir"' EXIT
+          unzip -q "$package_file" rustfs -d "$binary_dir"
+          version_output=$("$binary_dir/rustfs" --version)
+          first=${version_output%%$'\n'*}
+          if [[ "$first" != "rustfs ${PACKAGE_VERSION}" ]]; then
+            printf 'packaged binary version line is %q, expected %q\n' "$first" "rustfs ${PACKAGE_VERSION}" >&2
+            exit 1
+          fi
+          if ! grep -Fxq "build target : ${BUILD_TARGET}" <<< "$version_output" || \
+             ! grep -Fxq "build os     : ${BUILD_OS}" <<< "$version_output"; then
+            printf 'packaged binary reports an unexpected compile target\n' >&2
+            exit 1
+          fi
+
+      - name: Verify packaged console
+        if: matrix.target == 'x86_64-unknown-linux-gnu' || matrix.target == 'aarch64-unknown-linux-gnu'
+        shell: bash
+        env:
+          PACKAGE_VERSION: ${{ needs.prepare.outputs.version }}
+          ARCH: ${{ matrix.arch }}
+          LIBC: ${{ matrix.libc }}
+        run: |
+          set -euo pipefail
+          package_file="dist/rustfs-linux-${ARCH}-${LIBC}-v${PACKAGE_VERSION}.zip"
+
+          find_free_port() {
+            python3 - <<'PY'
+          import socket
+
+          with socket.socket() as sock:
+              sock.bind(("127.0.0.1", 0))
+              print(sock.getsockname()[1])
+          PY
+          }
+
+          package_dir=$(mktemp -d)
+          data_dir=$(mktemp -d)
+          log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
+          server_pid=""
+          trap '
+            if [[ -n "${server_pid:-}" ]]; then
+              kill "$server_pid" 2>/dev/null || true
+              wait "$server_pid" 2>/dev/null || true
+            fi
+            rm -rf "$package_dir" "$data_dir"
+          ' EXIT
+
+          unzip -q "$package_file" -d "$package_dir"
+          api_port=$(find_free_port)
+          console_port=$(find_free_port)
+          console_url="http://127.0.0.1:${console_port}/rustfs/console/"
+
+          "$package_dir/rustfs" server \
+            --address "127.0.0.1:${api_port}" \
+            --console-enable \
+            --console-address "127.0.0.1:${console_port}" \
+            --access-key console-smoke \
+            --secret-key console-smoke-secret \
+            "$data_dir" >"$log_file" 2>&1 &
+          server_pid=$!
+
+          for _ in {1..40}; do
+            response=$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)
+            if [[ "$response" == 200\ text/html* ]]; then
+              exit 0
+            fi
+            sleep 0.25
+          done
+
+          printf 'Console endpoint did not return 200 text/html: %s\n' "$response" >&2
+          cat "$log_file"
+          exit 1
+
+      - name: Upload packages
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: packages-${{ matrix.target_id }}-v${{ needs.prepare.outputs.version }}
+          path: dist/*
+          if-no-files-found: error
+          compression-level: 0
+          retention-days: 30

.github/workflows/mint.yml

@@ -112,9 +112,11 @@ jobs:
     # GitHub-hosted: reliably provides a running Docker daemon + buildx +
     # python3. See the header note (ci-2) for why the self-hosted sm-standard-4
     # label was abandoned (no docker.sock on the pod its only run landed on).
+    # sm-standard-2 has no docker.sock either (weekly runs failed at "Enable
+    # buildx" from 2026-09-27 to 2026-11-08), so keep this job on ubuntu-latest.
     # TODO(ci-8): scheduled-failure alerting (auto-open issue on a red weekly
     # run) is added by the ci-8 composite action; do not implement it here.
-    runs-on: sm-standard-2
+    runs-on: ubuntu-latest
     timeout-minutes: 120
     steps:
       - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7

.github/workflows/reference-parity.yml

@@ -0,0 +1,308 @@
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# s3-tests reference parity lane (rustfs/backlog#2773).
+#
+# Runs the ceph/s3-tests implemented list (scripts/s3-tests/implemented_tests.txt)
+# against every target in .config/reference-parity-targets.toml and fails on
+# any per-node-id difference between a candidate and the reference:
+#
+#   - reference: the release pinned in .config/reference-binary.toml, downloaded
+#     and sha256-verified by .github/actions/reference-binary;
+#   - candidate targets: the debug binary built from this checkout exactly as
+#     ci.yml's build-rustfs-debug-binary job builds it for S3 Implemented Tests.
+#
+# Every target runs the same harness (scripts/s3-tests/run.sh from this
+# checkout), test list and server configuration on its own hosted runner. A
+# target's failures get one rerun against a fresh server; a case that passes it
+# is flaky, not a difference. scripts/s3-tests/reference_parity.py judges the
+# result and honours .config/reference-parity-tolerated.txt, whose entries must
+# each match a real difference.
+#
+# Report-only: not part of the Test and Lint aggregate. Scheduled failures are
+# owned by the scheduled-validation watchdog (.github/scheduled-validations.json).
+
+name: Reference Parity
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/reference-parity.yml"
+      - ".github/actions/reference-binary/**"
+      - ".config/reference-binary.toml"
+      - ".config/reference-parity-targets.toml"
+      - ".config/reference-parity-tolerated.txt"
+      - "scripts/s3-tests/reference_parity.py"
+      - "scripts/junit_diff.py"
+  schedule:
+    - cron: "37 1 * * *"
+  workflow_dispatch:
+
+# Matches ci.yml: these variables are hashed into the rust-cache key, so the
+# candidate build reads the same ci-dev cache as build-rustfs-debug-binary.
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  plan:
+    name: Plan parity targets
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      targets: ${{ steps.targets.outputs.targets }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Validate target list
+        id: targets
+        run: |
+          set -euo pipefail
+          targets="$(python3 scripts/s3-tests/reference_parity.py targets --config .config/reference-parity-targets.toml)"
+          echo "targets=${targets}" >> "$GITHUB_OUTPUT"
+          echo "Parity targets: ${targets}"
+
+  build-candidate:
+    name: Build candidate binary
+    # Critical-path CPU and cache ownership: the same debug build as ci.yml's
+    # build-rustfs-debug-binary, reading the ci-dev cache written on sm-standard-4.
+    runs-on: sm-standard-4
+    timeout-minutes: 60
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-dev
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
+
+      - name: Build debug binary
+        run: python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
+
+      - name: Upload candidate binary
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: reference-parity-candidate-binary
+          overwrite: true
+          path: |
+            target/debug/rustfs
+            target/debug/rustfs.e2e.json
+          if-no-files-found: error
+          retention-days: 1
+
+  s3tests:
+    name: s3-tests (${{ matrix.target.name }})
+    needs: [ plan, build-candidate ]
+    runs-on: ubuntu-latest
+    timeout-minutes: 75
+    strategy:
+      fail-fast: false
+      matrix:
+        target: ${{ fromJSON(needs.plan.outputs.targets) }}
+    env:
+      TARGET_NAME: ${{ matrix.target.name }}
+      TARGET_KIND: ${{ matrix.target.binary }}
+      # Above this many first-run failures the target is systemically broken;
+      # a rerun would only repeat it, so the first run is judged as is.
+      RERUN_CAP: "50"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Fetch pinned reference binary
+        id: reference
+        if: matrix.target.binary == 'reference'
+        uses: ./.github/actions/reference-binary
+
+      - name: Download candidate binary
+        if: matrix.target.binary == 'candidate'
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: reference-parity-candidate-binary
+          path: target/debug
+
+      - name: Select target binary
+        env:
+          REFERENCE_PATH: ${{ steps.reference.outputs.path }}
+        run: |
+          set -euo pipefail
+          case "$TARGET_KIND" in
+            reference) binary="$REFERENCE_PATH" ;;
+            candidate) chmod +x target/debug/rustfs; binary="$PWD/target/debug/rustfs" ;;
+            *) echo "unknown target binary kind: $TARGET_KIND" >&2; exit 1 ;;
+          esac
+          test -x "$binary"
+          echo "TARGET_BINARY=$binary" >> "$GITHUB_ENV"
+          mkdir -p "parity/$TARGET_NAME"
+          {
+            echo "target=$TARGET_NAME"
+            echo "binary=$TARGET_KIND"
+            echo "sha256=$(sha256sum "$binary" | cut -d' ' -f1)"
+            echo "version=$("$binary" --version | tr '\n' ' ')"
+          } | tee "parity/$TARGET_NAME/target.txt"
+
+      - name: Apply target environment
+        env:
+          TARGET_JSON: ${{ toJSON(matrix.target) }}
+        run: |
+          set -euo pipefail
+          python3 scripts/s3-tests/reference_parity.py export-env --target-json "$TARGET_JSON" \
+            | tee -a "$GITHUB_ENV" "parity/$TARGET_NAME/target-env.txt"
+
+      - name: Run implemented s3-tests
+        run: |
+          set -uo pipefail
+          report="parity/$TARGET_NAME"
+          harness="artifacts/s3tests-single"
+
+          # Same invocation as ci.yml's S3 Implemented Tests, with a fresh data
+          # root and port per run. $1 optionally replaces the test list.
+          run_suite() {
+            local list="$1" label="$2" root port
+            root="${RUNNER_TEMP}/rustfs-s3tests-${label}"
+            rm -rf "$harness" "$root"
+            mkdir -p "$root"
+            port="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
+            env DEPLOY_MODE=binary \
+              RUSTFS_BINARY="$TARGET_BINARY" \
+              TEST_MODE=single \
+              MAXFAIL=0 \
+              XDIST=4 \
+              S3_PORT="$port" \
+              DATA_ROOT="$root" \
+              S3TESTS_CONF="$harness/s3tests.conf" \
+              ${list:+IMPLEMENTED_TESTS_FILE="$list"} \
+              ./scripts/s3-tests/run.sh
+          }
+
+          # A run is usable only if it produced a JUnit report; test failures
+          # are data for the comparison, a missing report is a lane failure.
+          keep() {
+            local prefix="$1"
+            for file in pytest.log compat-report.md; do
+              if [[ -f "$harness/$file" ]]; then cp "$harness/$file" "$report/${prefix}${file}"; fi
+            done
+            if [[ -f "$harness/rustfs-single/rustfs.log" ]]; then cp "$harness/rustfs-single/rustfs.log" "$report/${prefix}rustfs.log"; fi
+            test -s "$harness/junit.xml" || { echo "::error::$TARGET_NAME produced no JUnit report (${prefix:-first} run)"; return 1; }
+            cp "$harness/junit.xml" "$report/${prefix}junit.xml"
+          }
+
+          run_suite "" first
+          echo "first_run_exit=$?" >> "$report/target.txt"
+          keep "" || exit 1
+          cp "$harness/selected-nodeids.txt" "$report/selected-nodeids.txt" || exit 1
+
+          failures="$(python3 scripts/s3-tests/reference_parity.py failed --report "$report")" || exit 1
+          count="$(printf '%s' "$failures" | grep -c . || true)"
+          echo "first_run_failures=$count" >> "$report/target.txt"
+          if (( count == 0 )); then
+            exit 0
+          fi
+          if (( count > RERUN_CAP )); then
+            echo "::warning::$TARGET_NAME has $count failures (cap $RERUN_CAP); judging the first run without a rerun"
+            echo "rerun=skipped-over-cap" >> "$report/target.txt"
+            exit 0
+          fi
+          printf '%s\n' "$failures" > "$RUNNER_TEMP/rerun-tests.txt"
+          run_suite "$RUNNER_TEMP/rerun-tests.txt" rerun
+          echo "rerun_exit=$?" >> "$report/target.txt"
+          keep rerun- || exit 1
+
+      - name: Upload target report
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: reference-parity-report-${{ matrix.target.name }}
+          overwrite: true
+          path: parity/${{ matrix.target.name }}/
+          if-no-files-found: error
+          retention-days: 14
+
+  compare:
+    name: Compare targets
+    needs: [ plan, s3tests ]
+    if: always() && needs.plan.result == 'success'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download target reports
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: reference-parity-report-*
+          path: parity-download
+
+      - name: Compare per node id
+        env:
+          TARGETS: ${{ needs.plan.outputs.targets }}
+          S3TESTS_RESULT: ${{ needs.s3tests.result }}
+        run: |
+          set -uo pipefail
+          mkdir -p parity
+          for dir in parity-download/reference-parity-report-*; do
+            if [[ -d "$dir" ]]; then mv "$dir" "parity/${dir#parity-download/reference-parity-report-}"; fi
+          done
+          python3 scripts/s3-tests/reference_parity.py compare \
+            --targets-json "$TARGETS" \
+            --reports parity \
+            --tolerated .config/reference-parity-tolerated.txt \
+            --markdown "$GITHUB_STEP_SUMMARY" \
+            --json parity/diff.json > parity/diff.md
+          status=$?
+          cat parity/diff.md
+          if [[ "$S3TESTS_RESULT" != "success" ]]; then
+            echo "::error::s3-tests target jobs finished with '$S3TESTS_RESULT'"
+            status=1
+          fi
+          exit "$status"
+
+      - name: Upload parity diff
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: reference-parity-diff
+          overwrite: true
+          path: |
+            parity/diff.md
+            parity/diff.json
+          if-no-files-found: warn
+          retention-days: 14

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Notifications and audit
+
+- Notify and audit targets that keep the default `queue_dir` now work on native Windows: the defaults `/opt/rustfs/events` and `/opt/rustfs/audit` resolve per node, to `%ProgramData%\RustFS\events` and `%ProgramData%\RustFS\audit` on Windows, instead of failing validation as non-absolute paths. The new `RUSTFS_TARGET_STORE_DIR` (Helm `config.rustfs.target_store_dir`) relocates both default queues on any platform. Linux and macOS behavior is unchanged when it is unset, persisted configuration is unchanged, and pending entries are not migrated when the root changes. See `docs/operations/target-queue-storage.md`.
+
 ### Deployment
 
 - Existing volumes created by older root-running container images must be writable by the non-root runtime identity before upgrading. The default container UID/GID is `10001:10001`; stop the affected instances, verify host mount paths, and for ordinary bind mounts run `sudo chown -R 10001:10001 /actual/data /actual/logs` on each RustFS data/log mount. Account for ACLs, NFS, Kubernetes, and rootless UID mappings. Image-build ownership changes do not migrate mounted data. See the Docker Quick Start in [README.md](README.md).

Cargo.lock

@@ -4698,6 +4698,12 @@ version = "0.3.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
 
+[[package]]
+name = "futures-timer"
+version = "3.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968"
+
 [[package]]
 name = "futures-util"
 version = "0.3.34"
@@ -7914,10 +7920,34 @@ version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6"
 dependencies = [
+ "phf_macros",
  "phf_shared 0.14.0",
  "serde",
 ]
 
+[[package]]
+name = "phf_generator"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100"
+dependencies = [
+ "fastrand",
+ "phf_shared 0.14.0",
+]
+
+[[package]]
+name = "phf_macros"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085"
+dependencies = [
+ "phf_generator",
+ "phf_shared 0.14.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
 [[package]]
 name = "phf_shared"
 version = "0.12.1"
@@ -9609,6 +9639,8 @@ dependencies = [
  "rustfs-ecstore",
  "rustfs-extension-schema",
  "rustfs-filemeta",
+ "rustfs-gateway",
+ "rustfs-gateway-adapter",
  "rustfs-gateway-types",
  "rustfs-heal",
  "rustfs-heal-contracts",
@@ -9999,10 +10031,119 @@ dependencies = [
  "xxhash-rust",
 ]
 
+[[package]]
+name = "rustfs-gateway"
+version = "0.62.0"
+source = "git+https://github.com/rustfs/gateway.git?rev=7915d06950f88611d883f6ab3d95efc90c65dec0#7915d06950f88611d883f6ab3d95efc90c65dec0"
+dependencies = [
+ "arc-swap",
+ "bytes",
+ "form_urlencoded",
+ "futures-timer",
+ "futures-util",
+ "http 1.5.0",
+ "http-body",
+ "http-body-util",
+ "httparse",
+ "hyper",
+ "pin-project-lite",
+ "rustfs-gateway-core",
+ "rustfs-gateway-http",
+ "rustfs-gateway-macros",
+ "rustfs-gateway-sig",
+ "rustfs-gateway-stream",
+ "rustfs-gateway-types",
+ "rustfs-gateway-xml",
+ "sha2 0.11.1",
+ "tokio",
+ "tower",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "rustfs-gateway-adapter"
+version = "1.0.5"
+dependencies = [
+ "http 1.5.0",
+ "rustfs-credentials",
+ "rustfs-gateway",
+ "rustfs-gateway-http",
+ "rustfs-iam",
+ "rustfs-policy",
+ "serde_json",
+ "time",
+ "tokio",
+]
+
+[[package]]
+name = "rustfs-gateway-core"
+version = "0.60.0"
+source = "git+https://github.com/rustfs/gateway.git?rev=7915d06950f88611d883f6ab3d95efc90c65dec0#7915d06950f88611d883f6ab3d95efc90c65dec0"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "http 1.5.0",
+ "md-5 0.11.0",
+ "percent-encoding",
+ "rustfs-gateway-http",
+ "rustfs-gateway-sig",
+ "rustfs-gateway-stream",
+ "rustfs-gateway-types",
+ "rustfs-gateway-xml",
+ "serde",
+ "serde_json",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "rustfs-gateway-http"
+version = "0.1.4"
+source = "git+https://github.com/rustfs/gateway.git?rev=7915d06950f88611d883f6ab3d95efc90c65dec0#7915d06950f88611d883f6ab3d95efc90c65dec0"
+dependencies = [
+ "hmac 0.13.0",
+ "http 1.5.0",
+ "httparse",
+ "memchr",
+ "phf 0.14.0",
+ "rustfs-gateway-stream",
+ "rustfs-gateway-types",
+ "sha2 0.11.1",
+ "smallvec",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "rustfs-gateway-macros"
+version = "0.1.4"
+source = "git+https://github.com/rustfs/gateway.git?rev=7915d06950f88611d883f6ab3d95efc90c65dec0#7915d06950f88611d883f6ab3d95efc90c65dec0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.9",
+]
+
+[[package]]
+name = "rustfs-gateway-sig"
+version = "0.15.0"
+source = "git+https://github.com/rustfs/gateway.git?rev=7915d06950f88611d883f6ab3d95efc90c65dec0#7915d06950f88611d883f6ab3d95efc90c65dec0"
+dependencies = [
+ "hmac 0.13.0",
+ "http 1.5.0",
+ "rustfs-gateway-http",
+ "sha1 0.11.0",
+ "sha2 0.11.1",
+ "smallvec",
+ "subtle",
+ "zeroize",
+]
+
 [[package]]
 name = "rustfs-gateway-stream"
 version = "0.2.0"
-source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
+source = "git+https://github.com/rustfs/gateway.git?rev=7915d06950f88611d883f6ab3d95efc90c65dec0#7915d06950f88611d883f6ab3d95efc90c65dec0"
 dependencies = [
  "bitflags 2.13.2",
  "bytes",
@@ -10012,8 +10153,8 @@ dependencies = [
 
 [[package]]
 name = "rustfs-gateway-types"
-version = "0.32.0+aws.2026-10-02"
-source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
+version = "0.40.0+aws.2026-10-02"
+source = "git+https://github.com/rustfs/gateway.git?rev=7915d06950f88611d883f6ab3d95efc90c65dec0#7915d06950f88611d883f6ab3d95efc90c65dec0"
 dependencies = [
  "bytes",
  "crc-fast",
@@ -10036,7 +10177,7 @@ dependencies = [
 [[package]]
 name = "rustfs-gateway-xml"
 version = "0.4.0"
-source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
+source = "git+https://github.com/rustfs/gateway.git?rev=7915d06950f88611d883f6ab3d95efc90c65dec0#7915d06950f88611d883f6ab3d95efc90c65dec0"
 dependencies = [
  "quick-xml 0.41.0",
 ]
@@ -12603,10 +12744,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.4.3",
+ "getrandom 0.3.4",
  "once_cell",
  "rustix",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]

Cargo.toml

@@ -26,6 +26,7 @@ members = [
     "crates/ecstore", # Erasure coding storage implementation
     "crates/e2e_test", # End-to-end test suite
     "crates/filemeta", # File metadata management
+    "crates/gateway-adapter", # RustFS implementations of the gateway extension traits
     "crates/heal", # Erasure set and object healing
     "crates/heal-contracts", # Heal request/response channel contracts
     "crates/iam", # Identity and Access Management
@@ -93,6 +94,7 @@ redundant_clone = "warn"
 [workspace.dependencies]
 # RustFS Internal Crates
 rustfs = { path = "./rustfs", version = "1.0.5" }
+rustfs-gateway-adapter = { path = "crates/gateway-adapter", version = "1.0.5" }
 rustfs-heal = { path = "crates/heal", version = "1.0.5" }
 rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.5" }
 rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.5" }
@@ -147,8 +149,14 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
 # engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
 # gateway in this workspace; no crate may enable a `compat-s3s*` feature except
 # the s3s-facing edge of the rustfs binary.
-rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.32.0", default-features = false }
-rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.2.0", default-features = false }
+rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", version = "0.40.0", default-features = false }
+rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", version = "0.2.0", default-features = false }
+# The gateway framework itself, consumed for its extension traits by the ring-2
+# adapter. `default-features = false` keeps the `server` feature, and with it the
+# server crate, out of the build.
+rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", version = "0.62.0", default-features = false }
+# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
+rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", default-features = false }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"

Dockerfile.source

@@ -24,9 +24,10 @@
 #
 # Supports cross-compilation for amd64 and arm64 via TARGETPLATFORM.
 
-ARG TARGETPLATFORM
-ARG BUILDPLATFORM
-ARG TARGETARCH
+# Do not declare TARGETPLATFORM, BUILDPLATFORM or TARGETARCH here: a global
+# declaration without a value shadows BuildKit's automatic values, so the
+# stage-level re-declarations below see empty strings. They are only
+# declared inside the stage that uses them.
 ARG RUSTFS_BUILD_FEATURES=""
 ARG RC_RELEASE=latest
 
@@ -169,7 +170,24 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
     ;; \
     esac
 
-RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
+# TARGETARCH is only provided by BuildKit. Fall back to TARGETPLATFORM (for
+# example set by docker-compose) and then to the build host, like the cargo
+# step above, so every path that builds this file passes a real architecture.
+RUN set -eu; \
+    rc_arch="${TARGETARCH:-}"; \
+    if [ -z "${rc_arch}" ]; then \
+    case "${TARGETPLATFORM:-}" in \
+    linux/amd64) rc_arch=amd64 ;; \
+    linux/arm64) rc_arch=arm64 ;; \
+    *) \
+    case "$(uname -m)" in \
+    x86_64) rc_arch=amd64 ;; \
+    aarch64|arm64) rc_arch=arm64 ;; \
+    *) echo "TARGETARCH is empty and TARGETPLATFORM='${TARGETPLATFORM:-}' is not linux/amd64 or linux/arm64; cannot choose an rc architecture on $(uname -m)" >&2; exit 1 ;; \
+    esac ;; \
+    esac; \
+    fi; \
+    sh /usr/src/rustfs/scripts/download_rc.sh "${rc_arch}" musl "${RC_RELEASE}" /usr/local/bin/rc
 
 # -----------------------------
 # Development stage (keeps toolchain)

README.md

@@ -199,7 +199,7 @@ docker run -d --name rustfs -p 9000:9000 \
 Notes:
 - `RUSTFS_NOTIFY_ENABLE=true` enables the global notify module switch.
 - For ARN `arn:rustfs:sqs::primary:webhook`, use instance-scoped env vars with `_PRIMARY`.
-- If queue dir is omitted, the official image maps `/opt/rustfs/events` into the persistent `/data` volume so it is writable by the runtime user and pending events survive container recreation. Override `queue_dir` when using another deployment layout.
+- If queue dir is omitted, the official image maps `/opt/rustfs/events` into the persistent `/data` volume so it is writable by the runtime user and pending events survive container recreation. Override `queue_dir` when using another deployment layout, or set `RUSTFS_TARGET_STORE_DIR` to move every default queue; native Windows defaults to `%ProgramData%\RustFS`. See [Target queue storage](docs/operations/target-queue-storage.md).
 - `RUSTFS_NOTIFY_WEBHOOK_SKIP_TLS_VERIFY_PRIMARY` defaults to `false`; enabling it skips webhook TLS certificate verification, allows MITM attacks, and emits a startup warning. Prefer `RUSTFS_NOTIFY_WEBHOOK_CLIENT_CA_PRIMARY` for private CAs.
 - Since `1.0.0-beta.11`, webhook endpoints on private or container networks
   (`Docker Compose service names`, `host.docker.internal`, RFC 1918 addresses) are

crates/config/README.md

@@ -276,6 +276,13 @@ Drive timeout profile preset:
   - values must be positive; `0` or an invalid value falls back to the default, and values above `60` are clamped to `60`.
   - the setting is read by the aggregating node only; it does not change the internode RPC wire contract. Any retry shares one round deadline rather than receiving a fresh timeout.
 
+## Target queue directory
+
+- `RUSTFS_TARGET_STORE_DIR`
+  - absolute root for notify and audit target queues whose `queue_dir` is the default `/opt/rustfs/events` or `/opt/rustfs/audit`; they open `<root>/events` and `<root>/audit` on this node.
+  - unset keeps those paths on Linux and macOS; native Windows uses `%ProgramData%\RustFS\events` and `%ProgramData%\RustFS\audit`.
+  - explicit `queue_dir` values are never relocated, and the persisted target configuration is unchanged. Changing the root does not migrate pending entries. See `docs/operations/target-queue-storage.md`.
+
 ## Startup filesystem boundary policy
 
 - `RUSTFS_UNSUPPORTED_FS_POLICY` controls startup behavior when RustFS detects local endpoint filesystems that are outside the supported production boundary.

crates/config/src/constants/targets.rs

@@ -144,6 +144,12 @@ pub const POSTGRES_QUEUE_LIMIT: &str = "queue_limit";
 /// Applies to both notify and audit target queue stores.
 pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
 
+/// Environment variable that relocates the root of the default notify and audit queue directories.
+///
+/// It applies only to targets whose `queue_dir` is the logical default (`EVENT_DEFAULT_DIR` or
+/// `AUDIT_DEFAULT_DIR`); their queues move to `<root>/events` and `<root>/audit` on this node.
+pub const ENV_TARGET_STORE_DIR: &str = "RUSTFS_TARGET_STORE_DIR";
+
 /// Queue-store compression is enabled by default to reduce disk footprint.
 pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
 

crates/config/src/observability/mod.rs

@@ -91,7 +91,12 @@ pub const ENV_OBS_LOG_MATCH_MODE: &str = "RUSTFS_OBS_LOG_MATCH_MODE";
 
 /// Default values for log cleanup
 pub const DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES: u64 = 2 * 1024 * 1024 * 1024; // 2 GiB
-pub const DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES: u64 = 0; // No single file limit
+pub const DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES: u64 = 0; // No per-file trim by the cleaner
+/// Size at which the active log file is rolled when
+/// `RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES` is unset. Without a size roll
+/// the active file is bounded only by the rotation period, and the cleaner
+/// never touches the active file. An explicit `0` keeps size rolling off.
+pub const DEFAULT_OBS_LOG_ACTIVE_FILE_ROLL_SIZE_BYTES: u64 = 256 * 1024 * 1024; // 256 MiB
 pub const DEFAULT_OBS_LOG_COMPRESS_OLD_FILES: bool = true;
 pub const DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL: u32 = 6;
 pub const DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM_GZIP: &str = "gzip";

crates/e2e_test/src/distributed/multipart_fence_acceptance_test.rs

@@ -1285,7 +1285,9 @@ async fn list_parts_disk_loss_after_metadata_read(
     let error = result.expect_err("one missing directory and three unavailable participants cannot prove an empty upload");
     let response = error.raw_response().expect("ListParts must return an S3 service response");
     assert_eq!(response.status().as_u16(), 503);
-    assert_eq!(error.as_service_error().and_then(|err| err.meta().code()), Some("SlowDownRead"));
+    // Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
+    // ListParts read-quorum failure is reported as ServiceUnavailable.
+    assert_eq!(error.as_service_error().and_then(|err| err.meta().code()), Some("ServiceUnavailable"));
     assert!(response.headers().get("x-rustfs-error-cause").is_none());
     assert!(response.headers().get("retry-after").is_none());
     assert!(

crates/e2e_test/src/distributed/upgrade_test.rs

@@ -493,7 +493,12 @@ async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &Upgra
     assert!(legacy.parts().is_empty());
     let current = current.expect_err("current coordinator must not infer empty from missing directories and offline disks");
     assert_eq!(current.raw_response().expect("current S3 response").status().as_u16(), 503);
-    assert_eq!(current.as_service_error().and_then(|error| error.meta().code()), Some("SlowDownRead"));
+    // Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
+    // ListParts read-quorum failure is reported as ServiceUnavailable.
+    assert_eq!(
+        current.as_service_error().and_then(|error| error.meta().code()),
+        Some("ServiceUnavailable")
+    );
     dist.cluster.start_node_from_binary(3, previous).await?;
     wait_for_upgrade_write_readiness(&clients, &seed.history_bucket, "mixed-list-restored", CREDENTIAL_TIMEOUT).await?;
     for (path, before) in original_files {

crates/e2e_test/src/dual_stack_differential/reference.rs

@@ -45,9 +45,6 @@ impl SideA {
             return Ok(Self::SameBinary);
         };
         let path = PathBuf::from(value);
-        if !path.exists() {
-            return Ok(Self::SameBinary);
-        }
         validate_executable(&path)?;
         Ok(Self::Reference(path))
     }
@@ -82,8 +79,7 @@ fn is_executable(_metadata: &std::fs::Metadata) -> bool {
 /// What `<binary> --version` printed, trimmed. Fails when the binary does not run, exits
 /// non-zero, or prints nothing, because an empty version would report nothing as observed.
 pub(super) async fn binary_version(binary: &Path) -> Result<String, BoxError> {
-    let _ = binary;
-    let output = tokio::process::Command::new("echo").arg("mutated").output().await?;
+    let output = tokio::process::Command::new(binary).arg("--version").output().await?;
     if !output.status.success() {
         return Err(format!("{} --version failed with {}", binary.display(), output.status).into());
     }

crates/ecstore/src/bucket/migration.rs

@@ -17,7 +17,7 @@
 use crate::bucket::metadata::BUCKET_METADATA_FILE;
 use crate::bucket::replication::ReplicationMigrationBridge;
 use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
-use crate::error::{Error, Result, is_err_strict_not_found, is_err_strict_volume_not_found};
+use crate::error::{Error, Result, is_err_strict_not_found, is_err_strict_volume_not_found, to_object_err};
 use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
 use crate::storage_api_contracts::{
     bucket::{BucketInfo, BucketOperations, BucketOptions},
@@ -334,6 +334,14 @@ async fn migration_target_exists<S: EcstoreObjectOperations>(store: &S, path: &s
     }
 }
 
+/// Names the target record in a failed migration write, as the existence
+/// probe's error already does. The commit path returns bare disk errors, and on
+/// Windows a blocked record is first seen here: reading through a regular-file
+/// ancestor reports the record missing instead of refusing access to it.
+fn migration_write_error(err: Error, path: &str) -> Error {
+    to_object_err(err, vec![RUSTFS_META_BUCKET, path])
+}
+
 async fn migrate_one_if_missing<S>(
     store: Arc<S>,
     opts: &ObjectOptions,
@@ -369,7 +377,10 @@ where
         .unwrap_or(data);
 
     let mut put_data = PutObjReader::from_vec(data);
-    store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts).await?;
+    store
+        .put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts)
+        .await
+        .map_err(|err| migration_write_error(err, path))?;
     info!("Migrated {label}");
     Ok(())
 }
@@ -481,7 +492,10 @@ where
                 Err(_) => return Err(MigrationMetadataError::Incompatible(path.to_owned()).into()),
             };
             let mut put_data = PutObjReader::from_vec(data);
-            store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts).await?;
+            store
+                .put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts)
+                .await
+                .map_err(|err| migration_write_error(err, path))?;
             info!("Migrated IAM config: {path}");
             total_migrated += 1;
         }
@@ -531,6 +545,27 @@ mod tests {
         ));
     }
 
+    #[test]
+    fn migration_write_errors_name_the_target_record_without_fragmenting_groups() {
+        use super::{Error, migration_write_error};
+
+        let first = migration_write_error(Error::FileAccessDenied, "buckets/first/.metadata.bin");
+        let second = migration_write_error(Error::FileAccessDenied, "config/iam/format.json");
+        assert!(matches!(first, Error::PrefixAccessDenied(..)), "unexpected error: {first:?}");
+        assert!(
+            first.to_string().contains(".rustfs.sys/buckets/first/.metadata.bin"),
+            "the startup error must name the blocked record: {first}"
+        );
+        assert_eq!(first, second, "record paths must not fragment error grouping");
+
+        let unrelated = migration_write_error(Error::other("disk offline"), "buckets/first/.metadata.bin");
+        assert_eq!(
+            unrelated,
+            Error::other("disk offline"),
+            "errors without a record form pass through unchanged"
+        );
+    }
+
     #[test]
     fn migration_errors_group_by_cause_and_retain_typed_record_context() {
         use super::{Error, MigrationMetadataError};

crates/ecstore/src/cluster/rpc/client.rs

@@ -41,8 +41,12 @@ use std::{
     future::Future,
     io::ErrorKind,
     pin::Pin,
-    sync::{LazyLock, Mutex},
+    sync::{
+        Arc, LazyLock, Mutex,
+        atomic::{AtomicU8, Ordering},
+    },
     task::{Context, Poll},
+    time::Duration,
 };
 use tonic::{service::interceptor::InterceptedService, transport::Channel};
 use tower::Service;
@@ -70,7 +74,7 @@ pub(crate) async fn scanner_control_time_out_client(
     let channel = get_channel_for_class(addr, ChannelClass::Control)
         .await
         .map_err(|err| crate::error::Error::other(err.to_string()))?;
-    let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
+    let channel = ReplayScopeChannel::new(BatchLockDeadlineChannel::new(channel), interceptor.replay_scope_audience());
     let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize;
     Ok(ScannerControlServiceClient::with_interceptor(channel, interceptor)
         .max_decoding_message_size(limit)
@@ -84,7 +88,7 @@ pub async fn heal_control_time_out_client(
     let interceptor = interceptor.with_rpc_audience(addr)?;
     let channel = get_channel_for_class(addr, ChannelClass::Control).await?;
     let max_message_size = rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE;
-    let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
+    let channel = ReplayScopeChannel::new(BatchLockDeadlineChannel::new(channel), interceptor.replay_scope_audience());
     Ok(HealControlServiceClient::with_interceptor(channel, interceptor)
         .max_decoding_message_size(max_message_size)
         .max_encoding_message_size(max_message_size))
@@ -97,7 +101,7 @@ pub async fn tier_mutation_control_time_out_client(
     let interceptor = interceptor.with_rpc_audience(addr)?;
     let channel = get_channel_for_class(addr, ChannelClass::Control).await?;
     let max_message_size = rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE;
-    let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
+    let channel = ReplayScopeChannel::new(BatchLockDeadlineChannel::new(channel), interceptor.replay_scope_audience());
     Ok(TierMutationControlServiceClient::with_interceptor(channel, interceptor)
         .max_decoding_message_size(max_message_size)
         .max_encoding_message_size(max_message_size))
@@ -118,7 +122,7 @@ pub async fn node_service_time_out_client_for_class(
     let channel = get_channel_for_class(addr, class).await?;
 
     let max_message_size = rustfs_protos::internode_rpc_max_message_size();
-    let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
+    let channel = ReplayScopeChannel::new(BatchLockDeadlineChannel::new(channel), interceptor.replay_scope_audience());
     Ok(NodeServiceClient::with_interceptor(channel, interceptor)
         .max_decoding_message_size(max_message_size)
         .max_encoding_message_size(max_message_size))
@@ -133,7 +137,7 @@ pub(crate) async fn node_service_delete_recovery_client(
     let channel = rustfs_protos::create_channel_with_timeout(addr, timeout)
         .await
         .map_err(|error| DiskError::RemoteClientUnavailable(error.to_string()))?;
-    let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
+    let channel = ReplayScopeChannel::new(BatchLockDeadlineChannel::new(channel), interceptor.replay_scope_audience());
     let max_message_size = rustfs_protos::internode_rpc_max_message_size();
     Ok(NodeServiceClient::with_interceptor(channel, interceptor)
         .max_decoding_message_size(max_message_size)
@@ -249,7 +253,95 @@ pub struct ReplayScopeChannel<S> {
 }
 
 /// The channel type used by internode clients after v2 authentication and replay-scope handling.
-pub type AuthenticatedChannel = ReplayScopeChannel<Channel>;
+pub type AuthenticatedChannel = ReplayScopeChannel<BatchLockDeadlineChannel<Channel>>;
+
+/// Gate lock-batch payloads after the transport's internal readiness queues.
+/// Other RPC bodies pass through without an additional wrapper.
+#[derive(Clone, Debug)]
+pub struct BatchLockDeadlineChannel<S> {
+    inner: S,
+}
+
+impl<S> BatchLockDeadlineChannel<S> {
+    fn new(inner: S) -> Self {
+        Self { inner }
+    }
+}
+
+impl<S> Service<HttpRequest<tonic::body::Body>> for BatchLockDeadlineChannel<S>
+where
+    S: Service<HttpRequest<tonic::body::Body>>,
+{
+    type Response = S::Response;
+    type Error = S::Error;
+    type Future = S::Future;
+
+    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        self.inner.poll_ready(cx)
+    }
+
+    fn call(&mut self, mut request: HttpRequest<tonic::body::Body>) -> Self::Future {
+        if let Some(deadline) = request.extensions().get::<BatchLockRpcDeadline>().cloned() {
+            let body = std::mem::replace(request.body_mut(), tonic::body::Body::empty());
+            *request.body_mut() = tonic::body::Body::new(BatchLockDeadlineBody {
+                inner: body,
+                deadline,
+                dispatched: false,
+                failed: false,
+            });
+        }
+        self.inner.call(request)
+    }
+}
+
+struct BatchLockDeadlineBody {
+    inner: tonic::body::Body,
+    deadline: BatchLockRpcDeadline,
+    dispatched: bool,
+    failed: bool,
+}
+
+impl http_body::Body for BatchLockDeadlineBody {
+    type Data = bytes::Bytes;
+    type Error = tonic::Status;
+
+    fn poll_frame(
+        mut self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
+        if self.failed {
+            return Poll::Ready(None);
+        }
+        if !self.dispatched
+            && let Err(error) = self.deadline.remaining_before_dispatch()
+        {
+            self.failed = true;
+            return Poll::Ready(Some(Err(error)));
+        }
+        let frame = std::task::ready!(Pin::new(&mut self.inner).poll_frame(cx));
+        if let Some(Ok(frame)) = &frame
+            && frame.is_data()
+            && !self.dispatched
+        {
+            // Codec work can consume the deadline in this poll. Arbitrate only
+            // after encoding, immediately before any payload leaves the body.
+            if let Err(error) = self.deadline.mark_dispatched() {
+                self.failed = true;
+                return Poll::Ready(Some(Err(error)));
+            }
+            self.dispatched = true;
+        }
+        Poll::Ready(frame)
+    }
+
+    fn is_end_stream(&self) -> bool {
+        self.failed || self.inner.is_end_stream()
+    }
+
+    fn size_hint(&self) -> http_body::SizeHint {
+        self.inner.size_hint()
+    }
+}
 
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 enum PeerReplayCapability {
@@ -426,7 +518,9 @@ where
                     )
                 }) {
                     Ok(headers) => request.headers_mut().extend(headers),
-                    Err(error) => debug!(error = %error, "could not attach replay-scoped RPC signature"),
+                    Err(error) => {
+                        debug!(error = %error, "could not attach replay-scoped RPC signature")
+                    }
                 }
             }
         }
@@ -488,6 +582,82 @@ where
     }
 }
 
+/// A batch may leave local readiness queues only while its original budget is live.
+/// Cancellation and submission arbitrate once so a cancelled caller cannot send later.
+#[derive(Clone, Debug)]
+pub(crate) struct BatchLockRpcDeadline {
+    deadline: tokio::time::Instant,
+    max_acquire_wait: Duration,
+    dispatch: Arc<AtomicU8>,
+}
+
+impl BatchLockRpcDeadline {
+    const PENDING: u8 = 0;
+    const DISPATCHED: u8 = 1;
+    const CANCELLED: u8 = 2;
+
+    pub(crate) fn new(deadline: tokio::time::Instant, max_acquire_wait: Duration) -> Self {
+        Self {
+            deadline,
+            max_acquire_wait,
+            dispatch: Arc::new(AtomicU8::new(Self::PENDING)),
+        }
+    }
+
+    /// True proves that this batch cannot send payload or enter the server lock handler.
+    /// Transport headers may already have left an internal channel queue.
+    pub(crate) fn cancel_before_dispatch(&self) -> bool {
+        match self
+            .dispatch
+            .compare_exchange(Self::PENDING, Self::CANCELLED, Ordering::AcqRel, Ordering::Acquire)
+        {
+            Ok(_) => true,
+            Err(state) => state == Self::CANCELLED,
+        }
+    }
+
+    pub(crate) fn was_dispatched(&self) -> bool {
+        self.dispatch.load(Ordering::Acquire) == Self::DISPATCHED
+    }
+
+    fn remaining_before_dispatch(&self) -> Result<Duration, tonic::Status> {
+        let remaining = self.deadline.saturating_duration_since(tokio::time::Instant::now());
+        if remaining <= self.max_acquire_wait || self.dispatch.load(Ordering::Acquire) == Self::CANCELLED {
+            return Err(tonic::Status::deadline_exceeded("Lock batch budget exhausted before payload dispatch"));
+        }
+        Ok(remaining)
+    }
+
+    fn update_timeout(&self, request: &mut tonic::Request<()>) -> Result<(), tonic::Status> {
+        request.set_timeout(self.remaining_before_dispatch()?);
+        Ok(())
+    }
+
+    fn mark_dispatched(&self) -> Result<(), tonic::Status> {
+        self.remaining_before_dispatch()?;
+        self.dispatch
+            .compare_exchange(Self::PENDING, Self::DISPATCHED, Ordering::AcqRel, Ordering::Acquire)
+            .map(|_| ())
+            .map_err(|_| tonic::Status::deadline_exceeded("Lock batch cancelled before dispatch"))
+    }
+
+    #[cfg(test)]
+    pub(super) async fn dispatch_payload_for_test(&self) -> Result<(), tonic::Status> {
+        use http_body_util::BodyExt as _;
+        let mut body = BatchLockDeadlineBody {
+            inner: tonic::body::Body::new(http_body_util::Full::new(bytes::Bytes::from_static(b"payload"))),
+            deadline: self.clone(),
+            dispatched: false,
+            failed: false,
+        };
+        match body.frame().await {
+            Some(Ok(frame)) if frame.is_data() => Ok(()),
+            Some(Err(error)) => Err(error),
+            _ => Err(tonic::Status::internal("test lock payload did not produce a data frame")),
+        }
+    }
+}
+
 pub struct TonicSignatureInterceptor {
     audience: Option<String>,
     body_digest_strict: bool,
@@ -582,11 +752,21 @@ impl TonicInterceptor {
 }
 
 impl tonic::service::Interceptor for TonicInterceptor {
-    fn call(&mut self, req: tonic::Request<()>) -> Result<tonic::Request<()>, tonic::Status> {
-        match self {
+    fn call(&mut self, mut req: tonic::Request<()>) -> Result<tonic::Request<()>, tonic::Status> {
+        let batch_deadline = req.extensions().get::<BatchLockRpcDeadline>().cloned();
+        if let Some(deadline) = &batch_deadline {
+            deadline.update_timeout(&mut req)?;
+        }
+        let mut req = match self {
             TonicInterceptor::Signature(interceptor) => interceptor.call(req),
             TonicInterceptor::NoOp(interceptor) => interceptor.call(req),
+        }?;
+        if let Some(deadline) = batch_deadline {
+            // Signing consumes budget, but Channel readiness only reserves a
+            // queue slot. The payload body arbitrates actual dispatch later.
+            deadline.update_timeout(&mut req)?;
         }
+        Ok(req)
     }
 }
 
@@ -782,6 +962,136 @@ mod tests {
         test_request_for("Ping")
     }
 
+    #[test]
+    fn batch_dispatch_and_cancellation_have_one_winner() {
+        for _ in 0..100 {
+            let deadline = BatchLockRpcDeadline::new(tokio::time::Instant::now() + Duration::from_secs(1), Duration::ZERO);
+            let other = deadline.clone();
+            let barrier = Arc::new(std::sync::Barrier::new(2));
+            let other_barrier = barrier.clone();
+            let submit = std::thread::spawn(move || {
+                other_barrier.wait();
+                other.mark_dispatched().is_ok()
+            });
+            barrier.wait();
+            let cancelled = deadline.cancel_before_dispatch();
+            let submitted = submit.join().expect("submission thread");
+            assert_ne!(cancelled, submitted);
+            assert_eq!(deadline.was_dispatched(), submitted);
+            assert!(deadline.mark_dispatched().is_err());
+        }
+    }
+
+    #[test]
+    fn batch_interceptor_rejects_exhausted_wait_reserve_before_submission() {
+        let deadline =
+            BatchLockRpcDeadline::new(tokio::time::Instant::now() + Duration::from_millis(20), Duration::from_millis(30));
+        let mut request = tonic::Request::new(());
+        request.extensions_mut().insert(deadline.clone());
+        let status = TonicInterceptor::NoOp(NoOpInterceptor)
+            .call(request)
+            .expect_err("insufficient wait reserve");
+        assert_eq!(status.code(), tonic::Code::DeadlineExceeded);
+        assert!(!deadline.was_dispatched());
+        assert!(deadline.cancel_before_dispatch());
+    }
+
+    #[test]
+    fn batch_interceptor_rejects_cancelled_ready_request_and_accepts_live_budget() {
+        let deadline = BatchLockRpcDeadline::new(tokio::time::Instant::now() + Duration::from_secs(2), Duration::from_millis(1));
+        let mut request = tonic::Request::new(());
+        request.extensions_mut().insert(deadline.clone());
+        assert!(deadline.cancel_before_dispatch());
+        assert!(TonicInterceptor::NoOp(NoOpInterceptor).call(request).is_err());
+        assert!(!deadline.was_dispatched());
+
+        let deadline = BatchLockRpcDeadline::new(tokio::time::Instant::now() + Duration::from_secs(2), Duration::from_millis(1));
+        let mut request = tonic::Request::new(());
+        request.extensions_mut().insert(deadline.clone());
+        let request = TonicInterceptor::NoOp(NoOpInterceptor).call(request).expect("live budget");
+        assert!(!deadline.was_dispatched(), "interceptor completion does not bypass the transport queue");
+        assert!(request.metadata().contains_key("grpc-timeout"));
+        assert!(deadline.cancel_before_dispatch());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn batch_body_rejects_payload_after_channel_queue_deadline() {
+        let deadline = BatchLockRpcDeadline::new(tokio::time::Instant::now() + Duration::from_millis(20), Duration::ZERO);
+        tokio::time::advance(Duration::from_millis(21)).await;
+        let error = deadline
+            .dispatch_payload_for_test()
+            .await
+            .expect_err("queued payload expired");
+        assert_eq!(error.code(), tonic::Code::DeadlineExceeded);
+        assert!(!deadline.was_dispatched());
+        assert!(deadline.cancel_before_dispatch());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn batch_body_rejects_payload_when_encoding_crosses_deadline_in_one_poll() {
+        use http_body_util::BodyExt as _;
+        struct SlowEncodingBody {
+            deadline: tokio::time::Instant,
+        }
+        impl http_body::Body for SlowEncodingBody {
+            type Data = bytes::Bytes;
+            type Error = tonic::Status;
+
+            fn poll_frame(
+                self: Pin<&mut Self>,
+                _cx: &mut Context<'_>,
+            ) -> Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
+                // Model codec work that begins before the deadline but blocks
+                // this single poll until the original budget has expired.
+                tokio::time::resume();
+                std::thread::sleep(
+                    self.deadline.saturating_duration_since(tokio::time::Instant::now()) + Duration::from_millis(1),
+                );
+                Poll::Ready(Some(Ok(http_body::Frame::data(bytes::Bytes::from_static(b"encoded payload")))))
+            }
+        }
+        let expires = tokio::time::Instant::now() + Duration::from_millis(20);
+        let deadline = BatchLockRpcDeadline::new(expires, Duration::ZERO);
+        let mut body = BatchLockDeadlineBody {
+            inner: tonic::body::Body::new(SlowEncodingBody { deadline: expires }),
+            deadline: deadline.clone(),
+            dispatched: false,
+            failed: false,
+        };
+        let error = body
+            .frame()
+            .await
+            .expect("first frame result")
+            .expect_err("late encoded payload");
+        assert_eq!(error.code(), tonic::Code::DeadlineExceeded);
+        assert!(!deadline.was_dispatched());
+        assert!(deadline.cancel_before_dispatch());
+        assert!(body.frame().await.is_none(), "failed body must not expose a later frame");
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn batch_body_keeps_cleanup_responsibility_after_first_payload() {
+        use http_body_util::BodyExt as _;
+        let deadline = BatchLockRpcDeadline::new(tokio::time::Instant::now() + Duration::from_millis(20), Duration::ZERO);
+        let frames = futures::stream::iter([
+            Ok::<_, tonic::Status>(http_body::Frame::data(bytes::Bytes::from_static(b"first"))),
+            Ok(http_body::Frame::data(bytes::Bytes::from_static(b"last"))),
+        ]);
+        let mut body = BatchLockDeadlineBody {
+            inner: tonic::body::Body::new(http_body_util::StreamBody::new(frames)),
+            deadline: deadline.clone(),
+            dispatched: false,
+            failed: false,
+        };
+        assert!(body.frame().await.expect("first frame").expect("live payload").is_data());
+        assert!(deadline.was_dispatched());
+        assert!(!deadline.cancel_before_dispatch());
+        tokio::time::advance(Duration::from_millis(21)).await;
+        assert!(body.frame().await.expect("last frame").expect("late drain").is_data());
+        assert!(body.frame().await.is_none());
+        assert!(!deadline.cancel_before_dispatch(), "a sent payload requires late cleanup ownership");
+    }
+
     fn test_request_for(method: &'static str) -> tonic::Request<()> {
         let mut request = tonic::Request::new(());
         request

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -169,6 +169,72 @@ fn decode_delete_recovery_batch_errors(codes: Vec<Option<u32>>, expected_len: us
         .collect()
 }
 
+fn log_delete_recovery_batch_result(
+    endpoint: &Endpoint,
+    expected_len: usize,
+    started: Instant,
+    result: &Result<Vec<Option<DiskError>>>,
+) {
+    let elapsed_ms = started.elapsed().as_millis();
+    let mut disk_error_counts = std::collections::BTreeMap::<u32, usize>::new();
+    let (failure_scope, rpc_status) = match result {
+        Ok(errors) => {
+            for error in errors.iter().flatten() {
+                *disk_error_counts.entry(error.to_u32()).or_default() += 1;
+            }
+            ("object_group", None)
+        }
+        Err(error) => {
+            disk_error_counts.insert(error.to_u32(), expected_len);
+            let rpc_status = match error {
+                DiskError::Io(source) => crate::cluster::rpc::client::embedded_tonic_status(source).map(tonic::Status::code),
+                _ => None,
+            };
+            ("rpc_or_reply", rpc_status)
+        }
+    };
+    let failed_group_count = disk_error_counts.values().sum::<usize>();
+    let result = if result.is_err() || failed_group_count > 0 {
+        "failed"
+    } else {
+        "ok"
+    };
+    if result == "failed" || elapsed_ms >= 1_000 {
+        warn!(
+            event = EVENT_REMOTE_DISK_RPC,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
+            result,
+            operation = "apply_versions",
+            failure_scope,
+            pool_index = endpoint.pool_idx,
+            set_index = endpoint.set_idx,
+            disk_index = endpoint.disk_idx,
+            attempted_group_count = expected_len,
+            failed_group_count,
+            disk_error_counts = ?disk_error_counts,
+            rpc_status = ?rpc_status,
+            elapsed_ms,
+            "Recoverable batch delete RPC completed"
+        );
+    } else {
+        debug!(
+            event = EVENT_REMOTE_DISK_RPC,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
+            result,
+            operation = "apply_versions",
+            pool_index = endpoint.pool_idx,
+            set_index = endpoint.set_idx,
+            disk_index = endpoint.disk_idx,
+            attempted_group_count = expected_len,
+            failed_group_count,
+            elapsed_ms,
+            "Recoverable batch delete RPC completed"
+        );
+    }
+}
+
 fn validate_delete_recovery_inspection(
     volume: &str,
     object: &str,
@@ -931,8 +997,16 @@ pub struct RemoteDisk {
     #[cfg(test)]
     recovery_monitor_teardown_hook: Arc<tokio::sync::Mutex<Option<Arc<RecoveryMonitorTeardownHook>>>>,
     data_transport: Arc<dyn InternodeDataTransport>,
+    /// Bound the per-operation failure warnings of this peer. An unreachable
+    /// peer fails every in-flight shard operation; the counters keep the exact
+    /// numbers per operation. Per handle, so one peer cannot hide another's.
+    timeout_logs: rustfs_utils::LogThrottle,
+    network_error_logs: rustfs_utils::LogThrottle,
 }
 
+/// Shortest gap between two failure warnings of one kind from one remote disk.
+const REMOTE_DISK_RPC_FAILURE_LOG_INTERVAL_MS: u64 = 5_000;
+
 struct RecoveryMonitorLease {
     active: Arc<AtomicBool>,
 }
@@ -1364,6 +1438,8 @@ impl RemoteDisk {
             #[cfg(test)]
             recovery_monitor_teardown_hook: Arc::new(tokio::sync::Mutex::new(None)),
             data_transport,
+            timeout_logs: rustfs_utils::LogThrottle::new(REMOTE_DISK_RPC_FAILURE_LOG_INTERVAL_MS),
+            network_error_logs: rustfs_utils::LogThrottle::new(REMOTE_DISK_RPC_FAILURE_LOG_INTERVAL_MS),
         };
         record_drive_runtime_state(ep, RuntimeDriveHealthState::Online);
 
@@ -2090,17 +2166,20 @@ impl RemoteDisk {
             "op" => op.to_string()
         )
         .increment(1);
-        warn!(
-            event = EVENT_REMOTE_DISK_RPC,
-            component = LOG_COMPONENT_ECSTORE,
-            subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
-            endpoint = %self.endpoint,
-            addr = %self.addr,
-            op,
-            timeout_ms = timeout_duration.as_millis(),
-            state = "timeout",
-            "Remote disk operation timed out"
-        );
+        if let Some(suppressed) = self.timeout_logs.claim() {
+            warn!(
+                event = EVENT_REMOTE_DISK_RPC,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
+                endpoint = %self.endpoint,
+                addr = %self.addr,
+                op,
+                timeout_ms = timeout_duration.as_millis(),
+                suppressed,
+                state = "timeout",
+                "Remote disk operation timed out"
+            );
+        }
     }
 
     async fn handle_network_like_error<T>(
@@ -2120,17 +2199,20 @@ impl RemoteDisk {
                 "op" => op.to_string()
             )
             .increment(1);
-            warn!(
-                event = EVENT_REMOTE_DISK_RPC,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
-                endpoint = %self.endpoint,
-                addr = %self.addr,
-                op,
-                timeout_ms = timeout_duration.as_millis(),
-                state = "network_like_error",
-                "Remote disk operation returned network-like error"
-            );
+            if let Some(suppressed) = self.network_error_logs.claim() {
+                warn!(
+                    event = EVENT_REMOTE_DISK_RPC,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
+                    endpoint = %self.endpoint,
+                    addr = %self.addr,
+                    op,
+                    timeout_ms = timeout_duration.as_millis(),
+                    suppressed,
+                    state = "network_like_error",
+                    "Remote disk operation returned network-like error"
+                );
+            }
             if failure_health_action == FailureHealthAction::MarkFailure {
                 self.mark_faulty_and_evict("operation_network_error").await;
             }
@@ -4373,6 +4455,7 @@ impl DiskAPI for RemoteDisk {
         opts: DeleteOptions,
     ) -> Vec<Option<DiskError>> {
         let expected_len = versions.len();
+        let started = Instant::now();
         let result = match self
             .recover_delete_transaction(volume, DeleteRecoveryRpcOperation::ApplyVersions { versions, opts })
             .await
@@ -4381,6 +4464,7 @@ impl DiskAPI for RemoteDisk {
             Ok(_) => Err(Error::other("unexpected recoverable batch delete reply")),
             Err(error) => Err(error),
         };
+        log_delete_recovery_batch_result(&self.endpoint, expected_len, started, &result);
         match result {
             Ok(errors) => errors,
             Err(error) => vec![Some(error); expected_len],
@@ -5673,6 +5757,62 @@ mod tests {
         assert!(decode_delete_recovery_batch_errors(vec![Some(u32::MAX)], 1).is_err());
     }
 
+    #[test]
+    fn recoverable_batch_delete_diagnostics_preserve_context_and_redact_error_details() {
+        let logs = CapturedLogs::default();
+        let subscriber = Registry::default().with(
+            tracing_subscriber::fmt::layer()
+                .with_writer(logs.clone())
+                .with_ansi(false)
+                .without_time()
+                .json()
+                .flatten_event(true)
+                .with_span_list(true),
+        );
+        let _guard = tracing::subscriber::set_default(subscriber);
+        let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
+        let endpoint = Endpoint {
+            url: url::Url::parse("http://127.0.0.1:9000/private-disk-path").expect("valid diagnostic endpoint"),
+            is_local: false,
+            pool_idx: 1,
+            set_idx: 2,
+            disk_idx: 3,
+        };
+        let rpc_failure = Err(DiskError::from(Status::deadline_exceeded("private-error-detail")));
+        let group_failures = Ok(vec![None, Some(DiskError::FileNotFound), Some(DiskError::FileNotFound)]);
+        tracing::info_span!("request-span", request_id = "req-batch-delete-diagnostics").in_scope(|| {
+            log_delete_recovery_batch_result(&endpoint, 1000, Instant::now(), &rpc_failure);
+            log_delete_recovery_batch_result(&endpoint, 3, Instant::now(), &group_failures);
+        });
+        let entries = logs.lines();
+        assert_eq!(entries.len(), 2, "one summary per disk batch, never per object");
+        assert_eq!(entries[0]["failure_scope"], "rpc_or_reply");
+        assert_eq!(entries[0]["failed_group_count"], 1000);
+        assert_eq!(entries[0]["rpc_status"], "Some(DeadlineExceeded)");
+        assert_eq!(entries[1]["failure_scope"], "object_group");
+        assert_eq!(entries[1]["failed_group_count"], 2);
+        for entry in entries {
+            assert_eq!(entry["pool_index"], 1);
+            assert_eq!(entry["set_index"], 2);
+            assert_eq!(entry["disk_index"], 3);
+            assert!(
+                entry["spans"]
+                    .as_array()
+                    .expect("request spans recorded")
+                    .iter()
+                    .any(|span| { span["request_id"] == "req-batch-delete-diagnostics" })
+            );
+            let text = entry.to_string();
+            assert!(!text.contains("private-error-detail"));
+            assert!(!text.contains("private-disk-path"));
+        }
+        assert!(rpc_failure.is_err(), "diagnostics borrow and preserve the original failure");
+        assert_eq!(
+            group_failures,
+            Ok(vec![None, Some(DiskError::FileNotFound), Some(DiskError::FileNotFound)])
+        );
+    }
+
     #[test]
     fn delete_recovery_inspection_rejects_wrong_transaction_object_and_protocol() {
         let transaction = Uuid::new_v4();
@@ -11688,6 +11828,66 @@ mod tests {
         );
     }
 
+    /// A peer that fails every operation logs one warning per interval, each
+    /// operation still gets its own error, and another peer's first warning is
+    /// not held back by the noisy one.
+    #[tokio::test]
+    async fn rpc_failure_warnings_are_bounded_per_remote_disk() {
+        async fn remote_disk(port: u16) -> RemoteDisk {
+            let url = url::Url::parse(&format!("http://127.0.0.1:{port}/data")).expect("endpoint url");
+            let endpoint = Endpoint {
+                url,
+                is_local: false,
+                pool_idx: 0,
+                set_idx: 0,
+                disk_idx: 0,
+            };
+            RemoteDisk::new(
+                &endpoint,
+                &DiskOption {
+                    cleanup: false,
+                    health_check: false,
+                },
+                Arc::new(TcpHttpInternodeDataTransport),
+            )
+            .await
+            .expect("remote disk handle")
+        }
+        let noisy = remote_disk(59981).await;
+        let quiet = remote_disk(59982).await;
+
+        for _ in 0..50 {
+            let error = noisy
+                .execute_with_timeout_for_op_and_health_action(
+                    "read_all",
+                    || async {
+                        Err::<(), Error>(DiskError::Io(std::io::Error::new(
+                            std::io::ErrorKind::ConnectionRefused,
+                            "connection refused",
+                        )))
+                    },
+                    Duration::from_secs(1),
+                    FailureHealthAction::IgnoreFailure,
+                )
+                .await
+                .expect_err("every failing operation still fails");
+            assert!(matches!(error, DiskError::Io(io) if io.kind() == std::io::ErrorKind::ConnectionRefused));
+            noisy.record_timeout("read_all", Duration::from_secs(1));
+        }
+
+        assert!(
+            noisy.network_error_logs.claim().is_none(),
+            "the burst used this peer's network-error window"
+        );
+        assert!(noisy.timeout_logs.claim().is_none(), "the burst used this peer's timeout window");
+        assert_eq!(
+            quiet.network_error_logs.claim(),
+            Some(0),
+            "another peer's first warning is not suppressed"
+        );
+        assert_eq!(quiet.timeout_logs.claim(), Some(0));
+    }
+
     #[tokio::test]
     async fn test_execute_with_timeout_keeps_remote_disk_online_for_business_error() {
         let addr = "http://127.0.0.1:59994".to_string();

crates/ecstore/src/disk/local.rs

@@ -11541,11 +11541,14 @@ impl LocalDisk {
     ) -> Result<()> {
         multipart_staging::reject_reserved_anchor(path)?;
         let fuse_with_admission = admission.is_some() && !crash_inject::is_armed(CrashPoint::MetaWriteAfterTmpBeforeRename, path);
+        // Held until the staging root is published, by the filesystem work
+        // itself, so a concurrent listing cannot prune the marker in between.
+        let mut index_pin = None;
         let prepared_index = if volume == RUSTFS_META_MULTIPART_BUCKET {
             if fuse_with_admission {
                 self.multipart_index.prepare_record_head(path, &buf).await
             } else {
-                self.multipart_index.record_head(path, &buf).await?;
+                index_pin = self.multipart_index.record_head(path, &buf).await?;
                 None
             }
         } else {
@@ -11571,8 +11574,10 @@ impl LocalDisk {
             let root = self.publication_root.clone();
             let lease = os::acquire_namespace_mutation_lease_with_owner(&file_path, namespace_owner).await;
             os::run_blocking_namespace_operation_with_sync(lease, do_sync, move || {
-                if let Some(record) = prepared_index {
-                    record.run_std()?;
+                let _index_pin = prepared_index.map(|record| record.run_std()).transpose()?;
+                #[cfg(test)]
+                if let Some(pin) = &_index_pin {
+                    pin.pause_for_test();
                 }
                 super::fs::create_dir_all_std(&tmp_volume_dir)?;
                 #[cfg(test)]
@@ -11616,12 +11621,16 @@ impl LocalDisk {
             return Err(DiskError::Unexpected);
         }
 
+        let publication_owner: Option<Arc<dyn Send + Sync>> = match index_pin {
+            Some(pin) => Some(Arc::new((namespace_owner.clone(), pin))),
+            None => namespace_owner.clone(),
+        };
         os::rename_all_with_precondition_and_parent_sync(
             &tmp_file_path,
             &file_path,
             &volume_dir,
             &self.publication_root,
-            namespace_owner.clone(),
+            publication_owner,
             admission.as_ref().map(|admission| admission.precondition()),
             admission.is_some() && sync && durability.syncs_commit_metadata(),
         )

crates/ecstore/src/disk/local/commit.rs

@@ -521,8 +521,12 @@ impl LocalDisk {
             state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
         })?;
         crate::hp_guard!("LocalDisk::rename_data");
-        if dst_volume == super::super::RUSTFS_META_MULTIPART_BUCKET {
-            self.multipart_index.record_info(dst_path, &fi).await?;
+        if dst_volume == super::super::RUSTFS_META_MULTIPART_BUCKET
+            && let Some(pin) = self.multipart_index.record_info(dst_path, &fi).await?
+        {
+            // The owner chain reaches every lease this rename publishes under,
+            // so the index marker stays pinned until that work has returned.
+            state.namespace_owner = Some(Arc::new((state.namespace_owner.take(), pin)));
         }
         #[cfg(feature = "e2e-test-hooks")]
         if fi.is_healing() {

crates/ecstore/src/disk/local/delete_recovery.rs

@@ -41,19 +41,53 @@ struct DiscoverySession {
     touched: Instant,
 }
 
-/// Plain (non-transaction) children of an object-scoped discovery root cost one
-/// budget entry per this many names: enough to bound a hostile prefix without
-/// making a real folder prefix undeletable.
+/// Names an object-scoped scan skips without a stat (plain children of the
+/// key's directory, and entries of a UUID child that cannot be delete evidence)
+/// cost one budget entry per this many names: enough to bound a hostile prefix
+/// without making a real folder prefix undeletable.
 const PLAIN_CHILDREN_PER_BUDGET_ENTRY: u32 = 64;
 
+/// Entries one step of the resumable (background) discovery may visit. The
+/// foreground bounded protocol has its own, larger per-call allowance.
+const DISCOVERY_STEP_ENTRIES: u32 = 4096;
+
+/// Wall-clock allowance of one bounded discovery call: a third of the drive
+/// timeout that wraps the call (including its wait for a lane worker), capped at
+/// ten seconds, so an incomplete walk returns before the call can time out.
+fn bounded_discovery_time_limit() -> std::time::Duration {
+    (crate::disk::disk_store::get_max_timeout_duration() / 3).min(std::time::Duration::from_secs(10))
+}
+
+/// Whether an entry inside a UUID child of an object-scoped root can name one
+/// of the key's delete transactions: a journal file of a rollback directory or
+/// a data-directory marker. Every other name is inert to discovery.
+fn may_be_delete_evidence(name: &str) -> bool {
+    [PREPARED, STORAGE_FORMAT_FILE_BACKUP, DELETE_QUORUM_COMMITTED_FILE].contains(&name)
+        || [
+            DELETE_DATA_DIR_MARKER_PREFIX,
+            RESERVED_DELETE_DATA_DIR_MARKER_PREFIX,
+            RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX,
+        ]
+        .iter()
+        .any(|prefix| delete_transaction_marker_id(name, prefix).is_some())
+}
+
 impl DiscoverySession {
     /// Consume one bounded step, returning ownership only when traversal remains.
     fn advance(self, limit: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>)> {
-        self.advance_bounded(limit, super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)
+        self.advance_bounded(limit, DISCOVERY_STEP_ENTRIES, &|| false)
             .map(|(transactions, next, _)| (transactions, next))
     }
 
-    fn advance_bounded(mut self, limit: u32, max_entries: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>, u32)> {
+    /// Walk until `limit` transactions, `max_entries` charged entries, the end
+    /// of the scope, or `should_stop` (checked before every entry); any early
+    /// stop returns the remaining traversal as continuation state.
+    fn advance_bounded(
+        mut self,
+        limit: u32,
+        max_entries: u32,
+        should_stop: &dyn Fn() -> bool,
+    ) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>, u32)> {
         if self.frames.is_empty() {
             match std::fs::read_dir(&self.root) {
                 Ok(entries) => self.frames.push((String::new(), entries)),
@@ -66,10 +100,11 @@ impl DiscoverySession {
         let mut transactions = BTreeSet::new();
         let mut seen_candidates = HashSet::new();
         let mut visited = 0u32;
-        let mut plain_children = 0u32;
+        let mut skipped_by_name = 0u32;
         while !self.frames.is_empty()
             && transactions.len() < usize::try_from(limit).map_err(|_| std::io::Error::from(ErrorKind::InvalidInput))?
             && visited < max_entries
+            && !should_stop()
         {
             let (parent, entries) = self
                 .frames
@@ -90,13 +125,22 @@ impl DiscoverySession {
                 return Err(std::io::Error::from(ErrorKind::InvalidData));
             }
             // An object-scoped scan only ever descends into transaction (UUID)
-            // directories at the root. Plain child keys of a prefix directory are
-            // skipped by name without a stat; they are charged at one entry per
-            // readdir batch so the work stays bounded while deleting a missing key
-            // that names a prefix with thousands of children can still complete.
-            if self.object.is_some() && parent.is_empty() && !canonical_uuid(&name).is_some_and(|id| !id.is_nil()) {
-                plain_children += 1;
-                if plain_children.is_multiple_of(PLAIN_CHILDREN_PER_BUDGET_ENTRY) {
+            // directories at the root, and inside them only journal files and
+            // data-directory markers can be evidence. Every other name (plain
+            // child keys of a prefix directory, and the `xl.meta`, data
+            // directories or children of a UUID-named child key) is skipped by
+            // name without a stat and charged at one entry per readdir batch, so
+            // the work stays bounded while deleting a missing key that names a
+            // folder prefix with many children, UUID-named or not, can complete.
+            if self.object.is_some()
+                && if parent.is_empty() {
+                    !canonical_uuid(&name).is_some_and(|id| !id.is_nil())
+                } else {
+                    !may_be_delete_evidence(&name)
+                }
+            {
+                skipped_by_name += 1;
+                if skipped_by_name.is_multiple_of(PLAIN_CHILDREN_PER_BUDGET_ENTRY) {
                     visited += 1;
                 }
                 continue;
@@ -1391,8 +1435,16 @@ impl LocalDisk {
         self.validate_delete_discovery_owner(&session)?;
         // The blocking owner drops every directory handle even if the RPC waiter disappears.
         // A partial inventory is never evidence of absence and never enters DISCOVERY_SESSIONS.
-        os::run_blocking_namespace_operation(lease, move || {
-            let (transactions, next, visited) = session.advance_bounded(256, max_entries)?;
+        // A large folder prefix may take a while to walk, so the probe rides the
+        // durable-mutation lane like bounded inspection, never the one-worker
+        // foreground lane that every other mutation on the drive queues behind.
+        // It also stops, incomplete, when its waiter goes away or well before the
+        // drive timeout, so a slow walk answers SlowDown instead of marking the
+        // drive failed or holding a lane worker after the request is gone.
+        let deadline = Instant::now() + bounded_discovery_time_limit();
+        os::run_cancellable_blocking_namespace_operation_with_sync(lease, true, move |is_cancelled| {
+            let should_stop = || is_cancelled() || Instant::now() >= deadline;
+            let (transactions, next, visited) = session.advance_bounded(256, max_entries, &should_stop)?;
             Ok(super::super::BoundedDeleteRecoveryDiscovery {
                 transactions: if next.is_none() { transactions } else { Vec::new() },
                 visited,
@@ -6561,6 +6613,93 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn delete_recovery_bounded_discovery_charges_uuid_named_children_once_and_keeps_evidence() {
+        let (_temp, disk, _, _) = fixture().await;
+        let path = disk.delete_discovery_path("recovery", Some("prefix")).expect("prefix path");
+        std::fs::create_dir_all(&path).expect("physical prefix");
+        // UUID-named child keys of the folder: each holds its own `xl.meta` and a
+        // data directory, neither of which can be evidence for the folder's key.
+        const CHILDREN: u32 = 2000;
+        for _ in 0..CHILDREN {
+            let child = path.join(Uuid::new_v4().to_string());
+            std::fs::create_dir_all(child.join(Uuid::new_v4().to_string())).expect("child key data dir");
+            std::fs::write(child.join(STORAGE_FORMAT_FILE), b"meta").expect("child key metadata");
+        }
+        // A UUID-named sub-prefix with children of its own.
+        const SUB_PREFIX_CHILDREN: u32 = 640;
+        let sub_prefix = path.join(Uuid::new_v4().to_string());
+        for index in 0..SUB_PREFIX_CHILDREN {
+            std::fs::create_dir_all(sub_prefix.join(format!("child-{index}"))).expect("sub-prefix child");
+        }
+        // Evidence of two interrupted transactions of the key itself: a journaled
+        // rollback directory, and a data-directory marker whose rollback is gone.
+        let journaled = Uuid::new_v4();
+        std::fs::create_dir_all(path.join(journaled.to_string())).expect("rollback directory");
+        std::fs::write(path.join(journaled.to_string()).join(PREPARED), b"{}").expect("journal");
+        let marker_only = Uuid::new_v4();
+        let data_dir = path.join(Uuid::new_v4().to_string());
+        std::fs::create_dir_all(&data_dir).expect("residual data directory");
+        std::fs::write(data_dir.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{marker_only}")), [])
+            .expect("reservation marker");
+
+        let page = disk
+            .discover_delete_transactions_bounded(
+                "recovery",
+                "prefix",
+                super::super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES,
+            )
+            .await
+            .expect("bounded probe");
+        assert!(page.complete, "a folder of UUID-named keys is a complete inventory");
+        // One entry per descent (children, sub-prefix, rollback and data
+        // directory) and per evidence file; inert names at 1/64.
+        let inert = 2 * CHILDREN + SUB_PREFIX_CHILDREN;
+        assert_eq!(page.visited, CHILDREN + 3 + 2 + inert / PLAIN_CHILDREN_PER_BUDGET_ENTRY);
+        let found = page.transactions.iter().map(|id| id.transaction).collect::<BTreeSet<_>>();
+        assert!(found.contains(&journaled), "a journaled rollback directory is evidence");
+        assert!(found.contains(&marker_only), "a marker-only residue is evidence");
+        assert_eq!(found.len(), 2, "no child key is mistaken for a transaction");
+    }
+
+    #[tokio::test]
+    async fn delete_recovery_bounded_discovery_stops_on_request_and_skips_only_inert_irregular_entries() {
+        let (_temp, disk, _, _) = fixture().await;
+        let path = disk.delete_discovery_path("recovery", Some("prefix")).expect("prefix path");
+        let child = path.join(Uuid::new_v4().to_string());
+        std::fs::create_dir_all(&child).expect("UUID-named child key");
+        // A stop request (cancellation or the wall-clock limit) ends the walk
+        // before any entry, leaving continuation state: never evidence of absence.
+        let session = disk
+            .new_delete_discovery_session("recovery", Some("prefix"), &path)
+            .expect("private session");
+        let (transactions, next, visited) = session
+            .advance_bounded(256, super::super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, &|| true)
+            .expect("stopped walk");
+        assert!(transactions.is_empty());
+        assert_eq!(visited, 0);
+        assert!(next.is_some(), "a stopped walk is incomplete");
+        drop(next);
+        #[cfg(unix)]
+        {
+            // An irregular entry whose name cannot be evidence is skipped unread.
+            std::os::unix::fs::symlink(&path, child.join("inert-link")).expect("inert symlink");
+            let page = disk
+                .discover_delete_transactions_bounded("recovery", "prefix", 4096)
+                .await
+                .expect("inert irregular entries are not evidence");
+            assert!(page.complete && page.transactions.is_empty());
+            // One with an evidence name still fails closed.
+            std::os::unix::fs::symlink(&path, child.join(PREPARED)).expect("evidence-named symlink");
+            assert!(
+                disk.discover_delete_transactions_bounded("recovery", "prefix", 4096)
+                    .await
+                    .is_err(),
+                "an evidence-named symlink is corruption, not a skipped name"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn delete_recovery_bounded_discovery_refuses_partial_inventory_without_public_cursor() {
         let (_temp, disk, _, _) = fixture().await;
@@ -6663,8 +6802,11 @@ mod tests {
         let filler_id = Uuid::new_v4().to_string();
         let filler = path.join(&filler_id);
         fs::create_dir_all(&filler).await.expect("non-evidence directory");
-        for index in 0..5000 {
-            fs::write(filler.join(format!("part-{index:04}")), [])
+        // Inert names are skipped at 1/64, so fill the step with entries that
+        // pass the evidence-name filter but are directories: each costs a full
+        // budget entry and names no transaction.
+        for _ in 0..5000 {
+            fs::create_dir(filler.join(format!("{DELETE_DATA_DIR_MARKER_PREFIX}{}", Uuid::new_v4())))
                 .await
                 .expect("non-evidence entry");
         }

crates/ecstore/src/disk/local/multipart_index.rs

@@ -4,6 +4,11 @@
 //! Bucket-to-staging-root candidates: one marker per staging root, recorded
 //! before the root is published and pruned once the root is gone. Source
 //! metadata stays authoritative.
+//!
+//! A marker without a staging root is either a leftover or an upload whose
+//! root is still being published. The publisher proves the second case by
+//! holding a [`PublicationPin`] from before it records the marker until its
+//! filesystem work has returned; no elapsed time is taken as that proof.
 
 use super::{LocalDisk, read_metadata_only_std};
 use crate::disk::{
@@ -29,9 +34,6 @@ use uuid::Uuid;
 
 const INDEX: &str = ".rustfs.sys/multipart-index-v1";
 const REBUILD_DIRENTS_PER_JOB: usize = 64;
-/// A marker is recorded before its staging root is published. A marker younger
-/// than this without a root is an upload in flight, never a leftover to prune.
-const MULTIPART_INDEX_PRUNE_GRACE: std::time::Duration = std::time::Duration::from_secs(300);
 type DirectoryEntries = Box<dyn Iterator<Item = io::Result<String>> + Send>;
 type PhysicalRoot = (u64, [u8; 16]);
 static OWNERS: LazyLock<parking_lot::Mutex<HashMap<PhysicalRoot, Weak<IndexOwner>>>> =
@@ -56,9 +58,15 @@ pub(super) struct IndexOwner {
     // drop so a cancelled scan can be resumed by the next caller.
     driving: AtomicBool,
     disabled: AtomicBool,
+    // Markers whose staging root some publisher is creating right now, with
+    // the number of publishers. Boot-local like the index itself: every writer
+    // of this physical drive in this process shares this owner.
+    publishing: parking_lot::Mutex<HashMap<PathBuf, usize>>,
     #[cfg(test)]
     rebuild_head_pause: parking_lot::Mutex<Option<RebuildHeadPause>>,
     #[cfg(test)]
+    publication_pause: parking_lot::Mutex<Option<RebuildHeadPause>>,
+    #[cfg(test)]
     observed_rebuild_queue: parking_lot::Mutex<Option<Arc<os::DiskBlockingQueue>>>,
     #[cfg(test)]
     rebuild_single_job: AtomicBool,
@@ -120,23 +128,6 @@ impl IndexDirectory {
             std::fs::File::open(self.path.join(name))
         }
     }
-    fn modified(&self, name: &str) -> io::Result<std::time::SystemTime> {
-        #[cfg(unix)]
-        {
-            self.open_file(name)?.metadata()?.modified()
-        }
-        #[cfg(not(unix))]
-        {
-            std::fs::metadata(self.path.join(name))?.modified()
-        }
-    }
-    /// Whether the marker is still inside the publication grace period. A
-    /// marker dated in the future (clock stepped back) is never a leftover.
-    fn marker_is_young(&self, name: &str) -> bool {
-        self.modified(name)
-            .ok()
-            .is_some_and(|modified| modified.elapsed().map_or(true, |age| age < MULTIPART_INDEX_PRUNE_GRACE))
-    }
     fn remove_file(&self, name: &str) -> io::Result<()> {
         #[cfg(unix)]
         {
@@ -192,6 +183,10 @@ impl RebuildCursor {
                     Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
                     Err(error) => return Err(error),
                 };
+                // The head is decoded; release the source pin before indexing. On
+                // Windows the pin denies delete sharing, so holding it would make a
+                // concurrent abort of this upload fail with a sharing violation.
+                drop(source);
                 let source_root = relative
                     .file_name()
                     .and_then(|name| name.to_str())
@@ -280,9 +275,12 @@ impl IndexOwner {
             initialized: OnceLock::new(),
             driving: AtomicBool::new(false),
             disabled: AtomicBool::new(false),
+            publishing: parking_lot::Mutex::new(HashMap::new()),
             #[cfg(test)]
             rebuild_head_pause: parking_lot::Mutex::new(None),
             #[cfg(test)]
+            publication_pause: parking_lot::Mutex::new(None),
+            #[cfg(test)]
             observed_rebuild_queue: parking_lot::Mutex::new(None),
             #[cfg(test)]
             rebuild_single_job: AtomicBool::new(false),
@@ -394,8 +392,8 @@ impl IndexOwner {
             // The index is boot-local: every start rescans the staging area
             // before serving it, so a marker lost to a crash is recreated before
             // it can be consulted. Creation is enough; no fsync of the marker or
-            // its ancestors. An existing marker is touched so pruning sees the
-            // new upload's age, not the previous one's.
+            // its ancestors. An existing marker needs nothing: the caller's
+            // publication pin, not the marker's age, keeps it from being pruned.
             match openat(
                 &parent,
                 source_root,
@@ -407,16 +405,7 @@ impl IndexOwner {
                         return Err(io::Error::other("index marker is not a file"));
                     }
                 }
-                Err(rustix::io::Errno::EXIST) => {
-                    // Best effort: a filesystem that refuses to set times keeps
-                    // the previous upload's age, which only shortens the prune
-                    // grace for this key; index completeness does not depend on it.
-                    if let Ok(marker) =
-                        openat(&parent, source_root, OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, Mode::empty())
-                    {
-                        let _ = std::fs::File::from(marker).set_modified(std::time::SystemTime::now());
-                    }
-                }
+                Err(rustix::io::Errno::EXIST) => {}
                 Err(error) => return Err(error.into()),
             }
         }
@@ -432,6 +421,22 @@ impl IndexOwner {
         Ok(())
     }
 
+    /// Declare that the caller is about to publish the staging root of
+    /// `marker`. Must precede the marker lease the caller records under, so a
+    /// prune that found no pin has released that lease before the marker is
+    /// (re)created.
+    fn pin_publication(self: &Arc<Self>, marker: PathBuf) -> PublicationPin {
+        *self.publishing.lock().entry(marker.clone()).or_default() += 1;
+        PublicationPin {
+            owner: Arc::clone(self),
+            marker,
+        }
+    }
+
+    fn is_publishing(&self, marker: &Path) -> bool {
+        self.publishing.lock().contains_key(marker)
+    }
+
     fn ensure_current_root(&self) -> io::Result<()> {
         #[cfg(unix)]
         let current = {
@@ -513,9 +518,12 @@ impl IndexOwner {
         }
     }
 
-    pub(super) async fn record_info(self: &Arc<Self>, path: &str, info: &FileInfo) -> Result<()> {
+    /// Record the marker of `path`'s staging root. The returned pin must stay
+    /// alive until the filesystem work that publishes the root has returned,
+    /// including work that outlives a cancelled caller.
+    pub(super) async fn record_info(self: &Arc<Self>, path: &str, info: &FileInfo) -> Result<Option<PublicationPin>> {
         if canonical_upload(path).is_none() {
-            return Ok(());
+            return Ok(None);
         }
         let owner = self.clone();
         let path = path.to_owned();
@@ -531,6 +539,7 @@ impl IndexOwner {
                     .join(path.split('/').next().unwrap_or_default())
             })
             .unwrap_or_else(|| owner.base.join(INDEX));
+        let pin = self.pin_publication(marker.clone());
         let lease = os::acquire_namespace_mutation_lease_with_owner(&marker, None).await;
         os::run_blocking_namespace_operation_with_sync(lease, true, move || {
             match owner.record(&path, &info) {
@@ -544,7 +553,7 @@ impl IndexOwner {
             }
         })
         .await?;
-        Ok(())
+        Ok(Some(pin))
     }
 
     pub(super) async fn prepare_record_head(self: &Arc<Self>, path: &str, bytes: &[u8]) -> Option<PreparedHeadRecord> {
@@ -578,20 +587,23 @@ impl IndexOwner {
                     .join(upload.split('/').next().unwrap_or_default())
             })
             .unwrap_or_else(|| self.base.join(INDEX));
+        let pin = self.pin_publication(marker.clone());
         let lease = os::acquire_namespace_mutation_lease_with_owner(&marker, None).await;
         Some(PreparedHeadRecord {
-            owner: Arc::clone(self),
             upload: upload.to_owned(),
             info,
+            pin,
             _lease: lease,
         })
     }
 
-    pub(super) async fn record_head(self: &Arc<Self>, path: &str, bytes: &[u8]) -> Result<()> {
+    /// Record the marker of the staging root that `path` belongs to. See
+    /// [`Self::record_info`] for how long the returned pin must live.
+    pub(super) async fn record_head(self: &Arc<Self>, path: &str, bytes: &[u8]) -> Result<Option<PublicationPin>> {
         let Some(prepared) = self.prepare_record_head(path, bytes).await else {
-            return Ok(());
+            return Ok(None);
         };
-        prepared.run().await
+        prepared.run().await.map(Some)
     }
 
     async fn page(self: &Arc<Self>, bucket: &str, marker: Option<&str>, disk_id: Uuid) -> Result<MultipartIndexPage> {
@@ -642,17 +654,20 @@ impl IndexOwner {
             // to every key ever multipart-initiated on this drive.
             let mut live = Vec::with_capacity(roots.len());
             let mut stale = Vec::new();
+            let markers = owner.base.join(INDEX).join(sha(&bucket));
             for root in roots {
+                // The marker precedes its staging root: a pinned one is an
+                // upload in flight and stays a candidate. The pin is read
+                // before the root, so a publisher that finished in between has
+                // already made its root visible.
+                if owner.is_publishing(&markers.join(&root)) {
+                    live.push(root);
+                    continue;
+                }
                 match owner.directory(&Path::new(RUSTFS_META_MULTIPART_BUCKET).join(&root)) {
                     Ok(_) => live.push(root),
                     Err(error) if matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::NotADirectory) => {
-                        // The marker precedes its staging root; a young one is an
-                        // upload in flight and stays a candidate.
-                        if directory.as_ref().is_some_and(|index| index.marker_is_young(&root)) {
-                            live.push(root);
-                        } else {
-                            stale.push(root);
-                        }
+                        stale.push(root);
                     }
                     Err(error) => return Err(DiskError::from(error)),
                 }
@@ -677,9 +692,11 @@ impl IndexOwner {
     }
 
     /// Remove one marker whose staging root was gone when a page examined it.
-    /// The marker's own namespace lease serializes this with `record_info`; the
-    /// staging root and the marker's age are re-checked under that lease, so an
-    /// upload recorded in between keeps its marker.
+    /// The marker's own namespace lease serializes this with recording. Under
+    /// that lease the publication pin is read first and the staging root
+    /// second: a publisher pinned before the read keeps the marker, one that
+    /// already finished has made its root visible, and one that pins later
+    /// records its marker only after this lease is released.
     async fn prune_marker(self: &Arc<Self>, bucket: &str, root: &str) -> Result<()> {
         let owner = self.clone();
         let bucket_directory = Path::new(INDEX).join(sha(bucket));
@@ -690,6 +707,9 @@ impl IndexOwner {
         // not queue behind every other mutation on the one-worker foreground lane.
         os::run_blocking_namespace_operation_with_sync(lease, true, move || {
             owner.ensure_current_root()?;
+            if owner.is_publishing(&marker) {
+                return Ok(());
+            }
             match owner.directory(&Path::new(RUSTFS_META_MULTIPART_BUCKET).join(&root)) {
                 Ok(_) => return Ok(()),
                 Err(error) if matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::NotADirectory) => {}
@@ -700,9 +720,6 @@ impl IndexOwner {
                 Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
                 Err(error) => return Err(error),
             };
-            if directory.marker_is_young(&root) {
-                return Ok(());
-            }
             match directory.remove_file(&root) {
                 Err(error) if error.kind() != io::ErrorKind::NotFound => Err(error),
                 _ => Ok(()),
@@ -713,34 +730,69 @@ impl IndexOwner {
     }
 }
 
-pub(super) struct PreparedHeadRecord {
+/// Proof that the staging root of one marker is being published. While any pin
+/// for a marker is alive, pruning leaves that marker alone.
+#[must_use = "dropping the pin lets a concurrent listing prune the marker before its staging root exists"]
+pub(super) struct PublicationPin {
     owner: Arc<IndexOwner>,
+    marker: PathBuf,
+}
+
+impl PublicationPin {
+    /// Test seam between recording the marker and publishing the staging root.
+    #[cfg(test)]
+    pub(super) fn pause_for_test(&self) {
+        let pause = self.owner.publication_pause.lock().take();
+        if let Some(pause) = pause {
+            let _pause = os::enter_blocking_test_hook();
+            let _ = pause.started.send(());
+            let _ = pause.release.recv_timeout(std::time::Duration::from_secs(10));
+        }
+    }
+}
+
+impl Drop for PublicationPin {
+    fn drop(&mut self) {
+        let mut publishing = self.owner.publishing.lock();
+        if let Some(count) = publishing.get_mut(&self.marker) {
+            *count -= 1;
+            if *count == 0 {
+                publishing.remove(&self.marker);
+            }
+        }
+    }
+}
+
+pub(super) struct PreparedHeadRecord {
     upload: String,
     info: FileInfo,
+    pin: PublicationPin,
     _lease: Arc<os::NamespaceMutationLease>,
 }
 
 impl PreparedHeadRecord {
-    pub(super) fn run_std(self) -> io::Result<()> {
+    /// Record the marker and release its lease. The caller publishes the
+    /// staging root while it holds the returned pin.
+    pub(super) fn run_std(self) -> io::Result<PublicationPin> {
         let Self {
-            owner,
             upload,
             info,
+            pin,
             _lease,
         } = self;
-        match owner.record(&upload, &info) {
+        match pin.owner.record(&upload, &info) {
             Err(error) if error.kind() == io::ErrorKind::InvalidData => {
-                owner.disabled.store(true, Ordering::Release);
-                Ok(())
+                pin.owner.disabled.store(true, Ordering::Release);
+                Ok(pin)
             }
-            result => result,
+            Err(error) => Err(error),
+            Ok(()) => Ok(pin),
         }
     }
 
-    async fn run(self) -> Result<()> {
+    async fn run(self) -> Result<PublicationPin> {
         let lease = Arc::clone(&self._lease);
-        os::run_blocking_namespace_operation_with_sync(lease, true, move || self.run_std()).await?;
-        Ok(())
+        Ok(os::run_blocking_namespace_operation_with_sync(lease, true, move || self.run_std()).await?)
     }
 }
 
@@ -958,47 +1010,274 @@ mod tests {
         let live = format!("{:064x}", 1);
         let finished = format!("{:064x}", 2);
         let in_flight = format!("{:064x}", 3);
-        let clock_stepped = format!("{:064x}", 4);
-        for name in [&live, &finished, &in_flight, &clock_stepped] {
+        for name in [&live, &finished, &in_flight] {
             std::fs::write(directory.join(name), b"1").expect("marker");
         }
         // Completion and abort remove the staging root and leave the marker.
         std::fs::create_dir_all(root.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&live)).expect("live staging root");
-        set_marker_modified(&directory.join(&finished), std::time::SystemTime::now() - MULTIPART_INDEX_PRUNE_GRACE * 2);
-        // A marker dated after a backwards clock step is never a leftover.
-        set_marker_modified(
-            &directory.join(&clock_stepped),
-            std::time::SystemTime::now() + MULTIPART_INDEX_PRUNE_GRACE * 2,
-        );
-        let page = disk.multipart_index.page("bucket", None, Uuid::new_v4()).await.expect("page");
-        // The in-flight marker has no staging root yet but is young: a candidate, not a leftover.
-        assert_eq!(page.roots, vec![live.clone(), in_flight.clone(), clock_stepped.clone()]);
+        let owner = &disk.multipart_index;
+        let publishing = owner.pin_publication(owner.base.join(INDEX).join(sha("bucket")).join(&in_flight));
+        let second_publisher = owner.pin_publication(owner.base.join(INDEX).join(sha("bucket")).join(&in_flight));
+        let page = owner.page("bucket", None, Uuid::new_v4()).await.expect("page");
+        // The in-flight marker has no staging root yet but is pinned: a candidate, not a leftover.
+        assert_eq!(page.roots, vec![live.clone(), in_flight.clone()]);
         assert!(page.next_marker.is_none());
         assert!(directory.join(&live).exists(), "a live root keeps its marker");
-        assert!(directory.join(&in_flight).exists(), "a marker inside the grace period is kept");
-        assert!(directory.join(&clock_stepped).exists(), "a future-dated marker is kept");
-        assert!(!directory.join(&finished).exists(), "a finished root's marker is pruned");
-        // The next page no longer pays for the pruned marker.
-        let page = disk
-            .multipart_index
+        assert!(directory.join(&in_flight).exists(), "a pinned marker is kept");
+        assert!(!directory.join(&finished).exists(), "a finished root's marker is pruned at once");
+        // One of two publishers of the same root finishing does not unpin it.
+        drop(publishing);
+        let page = owner
             .page("bucket", None, Uuid::new_v4())
             .await
-            .expect("page after pruning");
-        assert_eq!(page.roots, vec![live, in_flight, clock_stepped]);
+            .expect("page with one publisher left");
+        assert_eq!(page.roots, vec![live.clone(), in_flight.clone()]);
+        assert!(directory.join(&in_flight).exists(), "the remaining publisher keeps the marker");
+        // A publication that ended without a staging root leaves a leftover.
+        drop(second_publisher);
+        let page = owner
+            .page("bucket", None, Uuid::new_v4())
+            .await
+            .expect("page after the failed publication");
+        assert_eq!(page.roots, vec![live]);
+        assert!(!directory.join(&in_flight).exists(), "an unpinned marker without a root is pruned");
+        assert!(owner.publishing.lock().is_empty(), "finished publishers leave no pin behind");
     }
 
-    /// Move a marker's modification time outside the publication grace period.
-    fn age_marker(marker: &Path) {
-        set_marker_modified(marker, std::time::SystemTime::now() - MULTIPART_INDEX_PRUNE_GRACE * 2);
+    fn pause_next_publication(owner: &IndexOwner) -> (tokio::sync::oneshot::Receiver<()>, std::sync::mpsc::Sender<()>) {
+        let (started, waiting) = tokio::sync::oneshot::channel();
+        let (release, released) = std::sync::mpsc::channel();
+        *owner.publication_pause.lock() = Some(RebuildHeadPause {
+            started,
+            release: released,
+        });
+        (waiting, release)
     }
 
-    fn set_marker_modified(marker: &Path, modified: std::time::SystemTime) {
+    /// The window the pin closes: the marker is recorded and its lease is
+    /// released, the staging root is not published yet, and a listing examines
+    /// the bucket. The marker is dated a day back so that no grace period could
+    /// be what keeps it.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn multipart_index_listing_during_source_publication_keeps_the_new_upload_listed() {
+        let root = tempfile::tempdir().expect("disk root");
+        let disk = Arc::new(open(root.path()).await);
+        let (path, info) = upload("bucket", "published-while-listed");
+        let source_root = path.split('/').next().expect("source sha").to_owned();
+        let marker = root.path().join(INDEX).join(sha("bucket")).join(&source_root);
+        let source_head = root
+            .path()
+            .join(RUSTFS_META_MULTIPART_BUCKET)
+            .join(&path)
+            .join(STORAGE_FORMAT_FILE);
+        let (paused, release) = pause_next_publication(&disk.multipart_index);
+        let writer = tokio::spawn({
+            let disk = Arc::clone(&disk);
+            let path = path.clone();
+            async move { disk.write_metadata("bucket", RUSTFS_META_MULTIPART_BUCKET, &path, info).await }
+        });
+        tokio::time::timeout(std::time::Duration::from_secs(5), paused)
+            .await
+            .expect("the publication reaches the pause")
+            .expect("pause sender");
+        assert!(marker.exists(), "the marker precedes the staging root");
+        assert!(!source_head.exists(), "the staging root is not published yet");
         std::fs::File::options()
             .write(true)
-            .open(marker)
+            .open(&marker)
             .expect("marker")
-            .set_modified(modified)
-            .expect("set marker time");
+            .set_modified(std::time::SystemTime::now() - std::time::Duration::from_secs(86_400))
+            .expect("age the marker");
+
+        let identity = Uuid::new_v4();
+        let page = disk
+            .multipart_index
+            .page("bucket", None, identity)
+            .await
+            .expect("page during publication");
+        assert_eq!(page.roots, vec![source_root.clone()], "an upload in flight stays a candidate");
+        assert!(marker.exists(), "a listing must not prune the marker of an upload in flight");
+
+        release.send(()).expect("resume the publication");
+        writer.await.expect("writer task").expect("the upload metadata is published");
+        assert!(source_head.exists());
+        let page = disk
+            .multipart_index
+            .page("bucket", None, identity)
+            .await
+            .expect("page after publication");
+        assert_eq!(page.roots, vec![source_root], "the published upload is listed");
+        assert!(marker.exists());
+        assert!(disk.multipart_index.publishing.lock().is_empty());
+    }
+
+    /// A timed-out caller drops its future while the filesystem work goes on.
+    /// The pin belongs to that work, so the marker survives until the staging
+    /// root the detached work publishes is visible.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn multipart_index_cancelled_publisher_keeps_its_marker_until_the_detached_work_returns() {
+        let root = tempfile::tempdir().expect("disk root");
+        let disk = Arc::new(open(root.path()).await);
+        let (path, info) = upload("bucket", "caller-timed-out");
+        let source_root = path.split('/').next().expect("source sha").to_owned();
+        let marker = root.path().join(INDEX).join(sha("bucket")).join(&source_root);
+        let source_head = root
+            .path()
+            .join(RUSTFS_META_MULTIPART_BUCKET)
+            .join(&path)
+            .join(STORAGE_FORMAT_FILE);
+        let (paused, release) = pause_next_publication(&disk.multipart_index);
+        let writer = tokio::spawn({
+            let disk = Arc::clone(&disk);
+            let path = path.clone();
+            async move { disk.write_metadata("bucket", RUSTFS_META_MULTIPART_BUCKET, &path, info).await }
+        });
+        tokio::time::timeout(std::time::Duration::from_secs(5), paused)
+            .await
+            .expect("the publication reaches the pause")
+            .expect("pause sender");
+        writer.abort();
+        assert!(writer.await.expect_err("the caller is gone").is_cancelled());
+
+        let identity = Uuid::new_v4();
+        let page = disk
+            .multipart_index
+            .page("bucket", None, identity)
+            .await
+            .expect("page after the caller left");
+        assert_eq!(page.roots, vec![source_root.clone()]);
+        assert!(marker.exists(), "the detached publication still owns the marker");
+
+        release.send(()).expect("resume the detached publication");
+        tokio::time::timeout(std::time::Duration::from_secs(10), async {
+            while !source_head.exists() || !disk.multipart_index.publishing.lock().is_empty() {
+                tokio::time::sleep(std::time::Duration::from_millis(5)).await;
+            }
+        })
+        .await
+        .expect("the detached publication completes");
+        let page = disk
+            .multipart_index
+            .page("bucket", None, identity)
+            .await
+            .expect("page after the detached publication");
+        assert_eq!(page.roots, vec![source_root], "the upload the detached work published is listed");
+        assert!(marker.exists());
+    }
+
+    /// A page classifies a marker as stale before the publisher pins it; the
+    /// prune that follows must read the pin again under the marker lease.
+    #[tokio::test]
+    async fn multipart_index_prune_rechecks_the_pin_under_the_marker_lease() {
+        let root = tempfile::tempdir().expect("disk root");
+        let disk = open(root.path()).await;
+        let owner = &disk.multipart_index;
+        let directory = root.path().join(INDEX).join(sha("bucket"));
+        std::fs::create_dir_all(&directory).expect("marker directory");
+        let name = format!("{:064x}", 9);
+        std::fs::write(directory.join(&name), b"1").expect("marker");
+
+        let pin = owner.pin_publication(owner.base.join(INDEX).join(sha("bucket")).join(&name));
+        owner.prune_marker("bucket", &name).await.expect("prune of a pinned marker");
+        assert!(directory.join(&name).exists(), "a publisher that pinned after the page keeps its marker");
+
+        drop(pin);
+        owner.prune_marker("bucket", &name).await.expect("prune of a leftover");
+        assert!(!directory.join(&name).exists(), "the same marker is a leftover once nobody publishes it");
+    }
+
+    /// The publisher pins before it queues on the marker lease. A prune that
+    /// holds the lease and found no pin has therefore unlinked the marker
+    /// before the publisher records it, and the record recreates it.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn multipart_index_publisher_pins_before_it_waits_for_the_marker_lease() {
+        let root = tempfile::tempdir().expect("disk root");
+        let disk = Arc::new(open(root.path()).await);
+        let owner = Arc::clone(&disk.multipart_index);
+        let (path, info) = upload("bucket", "queued-behind-a-prune");
+        let source_root = path.split('/').next().expect("source sha").to_owned();
+        let marker = owner.base.join(INDEX).join(sha("bucket")).join(&source_root);
+        let visible_marker = root.path().join(INDEX).join(sha("bucket")).join(&source_root);
+
+        // Stand in for a prune that owns the marker lease and already read "no pin".
+        let prune_lease = os::acquire_namespace_mutation_lease_with_owner(&marker, None).await;
+        assert!(!owner.is_publishing(&marker));
+        let writer = tokio::spawn({
+            let disk = Arc::clone(&disk);
+            let path = path.clone();
+            async move { disk.write_metadata("bucket", RUSTFS_META_MULTIPART_BUCKET, &path, info).await }
+        });
+        tokio::time::timeout(std::time::Duration::from_secs(5), async {
+            while !owner.is_publishing(&marker) {
+                tokio::time::sleep(std::time::Duration::from_millis(2)).await;
+            }
+        })
+        .await
+        .expect("the publisher pins while the marker lease is still held by the prune");
+        assert!(!visible_marker.exists(), "the marker is recorded only under the lease");
+
+        drop(prune_lease);
+        writer.await.expect("writer task").expect("the upload metadata is published");
+        assert!(visible_marker.exists(), "the record after the prune recreates the marker");
+        let page = owner.page("bucket", None, Uuid::new_v4()).await.expect("page");
+        assert_eq!(page.roots, vec![source_root]);
+    }
+
+    /// Stale markers at the start of a page, at its cursor, and filling a whole
+    /// page: every page still satisfies the receiver's contract and advances.
+    #[tokio::test]
+    async fn multipart_index_pages_of_stale_markers_keep_valid_cursors() {
+        let root = tempfile::tempdir().expect("disk root");
+        let disk = open(root.path()).await;
+        let directory = root.path().join(INDEX).join(sha("bucket"));
+        std::fs::create_dir_all(&directory).expect("marker directory");
+        // Page one is entirely stale. Page two is live except its first name
+        // and the name its cursor lands on. Two live names follow.
+        let total = MULTIPART_INDEX_PAGE_SIZE * 2 + 2;
+        let stale = |i: usize| i <= MULTIPART_INDEX_PAGE_SIZE || i == MULTIPART_INDEX_PAGE_SIZE * 2 - 1;
+        for i in 0..total {
+            let name = format!("{i:064x}");
+            std::fs::write(directory.join(&name), b"1").expect("marker");
+            if !stale(i) {
+                std::fs::create_dir_all(root.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&name)).expect("staging root");
+            }
+        }
+        let disk_id = Uuid::new_v4();
+        let first = disk.multipart_index.page("bucket", None, disk_id).await.expect("first page");
+        first
+            .validate("bucket", None, disk_id)
+            .expect("an empty page with a cursor validates");
+        assert!(first.roots.is_empty(), "a fully stale page has no roots");
+        let cursor = first.next_marker.clone().expect("more markers follow a fully stale page");
+        assert_eq!(cursor, format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE - 1));
+
+        let second = disk
+            .multipart_index
+            .page("bucket", Some(&cursor), disk_id)
+            .await
+            .expect("second page");
+        second
+            .validate("bucket", Some(&cursor), disk_id)
+            .expect("stale first and cursor names validate");
+        assert_eq!(second.roots.len(), MULTIPART_INDEX_PAGE_SIZE - 2);
+        assert_eq!(second.roots.first(), Some(&format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE + 1)));
+        let cursor = second
+            .next_marker
+            .clone()
+            .expect("the cursor is the stale last examined name");
+        assert_eq!(cursor, format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE * 2 - 1));
+        assert!(second.roots.last() < Some(&cursor));
+
+        let third = disk
+            .multipart_index
+            .page("bucket", Some(&cursor), disk_id)
+            .await
+            .expect("third page");
+        third.validate("bucket", Some(&cursor), disk_id).expect("last page validates");
+        assert_eq!(third.roots.len(), 2);
+        assert!(third.next_marker.is_none());
+        let remaining = std::fs::read_dir(&directory).expect("marker directory").count();
+        assert_eq!(remaining, total - MULTIPART_INDEX_PAGE_SIZE - 2, "every stale marker is pruned");
     }
 
     #[tokio::test]
@@ -1015,7 +1294,6 @@ mod tests {
                 std::fs::create_dir_all(root.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&name)).expect("staging root");
             }
         }
-        age_marker(&directory.join(&stale));
         let disk_id = Uuid::new_v4();
         let first = disk.multipart_index.page("bucket", None, disk_id).await.expect("first page");
         first

crates/ecstore/src/disk/local/multipart_source_state.rs

@@ -813,7 +813,8 @@ impl SourceStore {
         Ok(())
     }
     pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
-        self.multipart_index
+        let index_pin = self
+            .multipart_index
             .record_head(&format!("{}/xl.meta", self.path), bytes)
             .await?;
         invalidate_active_multipart_source_head(&self.source);
@@ -822,6 +823,7 @@ impl SourceStore {
         let root = self.publication_root.clone();
         let bytes = bytes.to_vec();
         os::run_blocking_namespace_operation(self.lease.clone(), move || {
+            let _index_pin = index_pin;
             os::replace_multipart_source_metadata(&path, &bytes, &base, &root)
         })
         .await

crates/ecstore/src/disk/mod.rs

@@ -2585,7 +2585,13 @@ pub struct BoundedDeleteRecoveryDiscovery {
     pub complete: bool,
 }
 
-pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 4096;
+/// Entries one bounded discovery call may charge on one drive. Object-scoped
+/// discovery charges each transaction-shaped (UUID) child of the key's
+/// directory once and every name that cannot be delete evidence at
+/// 1/`PLAIN_CHILDREN_PER_BUDGET_ENTRY`, so this admits a missing-key delete of a
+/// folder prefix with this many UUID-named children per drive. The request
+/// budget (`DeleteRecoveryWorkBudget`) still bounds the total across keys.
+pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 262_144;
 pub const DELETE_RECOVERY_MAX_INSPECTION_ENTRIES: u32 = 65536;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

crates/ecstore/src/erasure/coding/bitrot.rs

@@ -15,7 +15,7 @@
 use crate::disk::error::Error as DiskError;
 use crate::erasure::codec::buffer_pool::{get_ec_buffer, return_ec_buffer};
 use pin_project_lite::pin_project;
-use rustfs_utils::HashAlgorithm;
+use rustfs_utils::{HashAlgorithm, LogThrottle};
 use std::future::poll_fn;
 use std::io::IoSlice;
 use std::pin::Pin;
@@ -28,6 +28,13 @@ const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_ERASURE: &str = "erasure";
 const EVENT_BITROT_SHORT_SHARD_READ: &str = "bitrot_short_shard_read";
 const EVENT_BITROT_HASH_MISMATCH: &str = "bitrot_hash_mismatch";
+/// A failing drive raises these once per shard block of every read it serves.
+/// The events carry no object identity (the caller logs that), so one line per
+/// interval with the suppressed count loses nothing but repetition.
+const BITROT_FAILURE_LOG_INTERVAL_MS: u64 = 5_000;
+static BITROT_SHORT_SHARD_READ_LOGS: LogThrottle = LogThrottle::new(BITROT_FAILURE_LOG_INTERVAL_MS);
+static BITROT_HASH_MISMATCH_LOGS: LogThrottle = LogThrottle::new(BITROT_FAILURE_LOG_INTERVAL_MS);
+static BITROT_READ_ERROR_LOGS: LogThrottle = LogThrottle::new(BITROT_FAILURE_LOG_INTERVAL_MS);
 const MAX_RETAINED_CHUNKS_PER_BLOCK: usize = 64;
 const MAX_CHUNK_POLLS_PER_YIELD: usize = MAX_RETAINED_CHUNKS_PER_BLOCK + 1;
 
@@ -259,18 +266,41 @@ fn reserve_bitrot_scratch(
 
 /// A truncated shard is `UnexpectedEof`, not a short success (backlog#799 B2).
 fn short_shard_read(got: usize, want: usize) -> std::io::Error {
-    error!(
-        event = EVENT_BITROT_SHORT_SHARD_READ,
-        component = LOG_COMPONENT_ECSTORE,
-        subsystem = LOG_SUBSYSTEM_ERASURE,
-        state = "failed",
-        got,
-        want,
-        "short shard read: got {got} of {want} bytes"
-    );
+    if let Some(suppressed) = BITROT_SHORT_SHARD_READ_LOGS.claim() {
+        error!(
+            event = EVENT_BITROT_SHORT_SHARD_READ,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_ERASURE,
+            state = "failed",
+            got,
+            want,
+            suppressed,
+            "short shard read: got {got} of {want} bytes"
+        );
+    }
     std::io::Error::new(std::io::ErrorKind::UnexpectedEof, DiskError::FileCorrupt)
 }
 
+fn log_hash_mismatch(data_len: usize) {
+    if let Some(suppressed) = BITROT_HASH_MISMATCH_LOGS.claim() {
+        error!(
+            event = EVENT_BITROT_HASH_MISMATCH,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_ERASURE,
+            state = "failed",
+            data_len,
+            suppressed,
+            "bitrot hash mismatch"
+        );
+    }
+}
+
+fn log_read_error(error: &std::io::Error) {
+    if let Some(suppressed) = BITROT_READ_ERROR_LOGS.claim() {
+        error!(suppressed, "bitrot reader read error: {error}");
+    }
+}
+
 /// Split a `[hash][data]` block, verify the hash (unless `skip_verify`), and
 /// return the data slice plus the time spent hashing. The caller writes `data`
 /// into its sink **only after** this returns `Ok`, so a shard that fails the
@@ -286,14 +316,7 @@ fn split_and_verify<'a>(hash_algo: &HashAlgorithm, skip_verify: bool, block: &'a
     let actual_hash = hash_algo.hash_encode(data);
     let verify = verify_start.elapsed();
     if actual_hash.as_ref() != hash {
-        error!(
-            event = EVENT_BITROT_HASH_MISMATCH,
-            component = LOG_COMPONENT_ECSTORE,
-            subsystem = LOG_SUBSYSTEM_ERASURE,
-            state = "failed",
-            data_len = data.len(),
-            "bitrot hash mismatch"
-        );
+        log_hash_mismatch(data.len());
         return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, DiskError::FileCorrupt));
     }
     Ok((data, verify))
@@ -345,7 +368,7 @@ where
                     .inner
                     .read_buf(&mut (&mut *out).limit(remaining))
                     .await
-                    .inspect_err(|e| error!("bitrot reader read error: {e}"))?;
+                    .inspect_err(log_read_error)?;
                 if n == 0 {
                     break;
                 }
@@ -459,14 +482,7 @@ where
                 for chunk in block_chunks() {
                     let take = remaining.min(chunk.len());
                     if actual_hash.as_ref()[hash_offset..hash_offset + take] != chunk[..take] {
-                        error!(
-                            event = EVENT_BITROT_HASH_MISMATCH,
-                            component = LOG_COMPONENT_ECSTORE,
-                            subsystem = LOG_SUBSYSTEM_ERASURE,
-                            state = "failed",
-                            data_len = want,
-                            "bitrot hash mismatch"
-                        );
+                        log_hash_mismatch(want);
                         return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, DiskError::FileCorrupt));
                     }
                     hash_offset += take;
@@ -594,10 +610,7 @@ where
 {
     let mut filled = 0;
     while filled < buf.len() {
-        let n = reader.read(&mut buf[filled..]).await.map_err(|e| {
-            error!("bitrot reader read error: {}", e);
-            e
-        })?;
+        let n = reader.read(&mut buf[filled..]).await.inspect_err(log_read_error)?;
         if n == 0 {
             break;
         }
@@ -1258,6 +1271,34 @@ mod tests {
         ));
     }
 
+    /// The log line is bounded; the rejection is not. Every damaged block is
+    /// refused with the same typed error however many came before it.
+    #[test]
+    fn repeated_bitrot_failures_are_all_rejected_while_their_log_is_bounded() {
+        use crate::disk::error::Error as DiskError;
+        let algo = HashAlgorithm::HighwayHash256;
+        let data = b"one of many damaged blocks";
+        let mut frame = algo.hash_encode(data).as_ref().to_vec();
+        frame.extend_from_slice(data);
+        *frame.last_mut().expect("test frame has payload") ^= 1;
+
+        for _ in 0..1_000 {
+            let error = super::split_and_verify(&algo, false, &frame).expect_err("every mismatch rejects its shard");
+            assert_eq!(error.kind(), io::ErrorKind::InvalidData);
+            assert_eq!(DiskError::from(error), DiskError::FileCorrupt);
+            let error = super::short_shard_read(3, 4);
+            assert_eq!(error.kind(), io::ErrorKind::UnexpectedEof);
+            assert_eq!(DiskError::from(error), DiskError::FileCorrupt);
+            super::log_read_error(&io::Error::other("drive read failed"));
+        }
+
+        // Each burst used its own window, so a further event of the same kind
+        // inside the interval is suppressed.
+        assert!(super::BITROT_HASH_MISMATCH_LOGS.claim().is_none());
+        assert!(super::BITROT_SHORT_SHARD_READ_LOGS.claim().is_none());
+        assert!(super::BITROT_READ_ERROR_LOGS.claim().is_none());
+    }
+
     struct FragmentedSource {
         chunks: VecDeque<Bytes>,
     }

crates/ecstore/src/erasure/coding/decode.rs

@@ -894,6 +894,13 @@ fn record_shard_read_result(
     }
 }
 
+/// A slow or misaligned shard reader is retired once per stripe of every read
+/// it lags on. The lines name only a shard index, so one per interval with the
+/// suppressed count is enough.
+const RETIRED_READER_LOG_INTERVAL_MS: u64 = 5_000;
+static ABANDONED_READER_LOGS: rustfs_utils::LogThrottle = rustfs_utils::LogThrottle::new(RETIRED_READER_LOG_INTERVAL_MS);
+static UNALIGNED_PARITY_READER_LOGS: rustfs_utils::LogThrottle = rustfs_utils::LogThrottle::new(RETIRED_READER_LOG_INTERVAL_MS);
+
 fn retire_abandoned_readers(errs: &mut [Option<Error>], retire_readers: &mut ShardIndexes, active_readers: &[bool]) {
     for (i, active) in active_readers.iter().enumerate() {
         if !*active {
@@ -904,7 +911,9 @@ fn retire_abandoned_readers(errs: &mut [Option<Error>], retire_readers: &mut Sha
             errs[i] = Some(Error::from(io::Error::new(ErrorKind::TimedOut, "shard read abandoned after read quorum")));
         }
         retire_readers.push(i);
-        warn!(shard_index = i, "retiring in-flight shard reader after read quorum");
+        if let Some(suppressed) = ABANDONED_READER_LOGS.claim() {
+            warn!(shard_index = i, suppressed, "retiring in-flight shard reader after read quorum");
+        }
     }
 }
 
@@ -2024,10 +2033,12 @@ where
             self.engaged[idx] = true;
             true
         } else {
-            warn!(
-                shard_index = idx,
-                stripe_index, "retiring parity reader that cannot be aligned to the current stripe"
-            );
+            if let Some(suppressed) = UNALIGNED_PARITY_READER_LOGS.claim() {
+                warn!(
+                    shard_index = idx,
+                    stripe_index, suppressed, "retiring parity reader that cannot be aligned to the current stripe"
+                );
+            }
             self.readers[idx] = None;
             false
         }
@@ -2844,6 +2855,29 @@ impl Erasure {
 #[cfg(test)]
 mod tests {
     use super::*;
+
+    /// The warning is bounded; the retirement is not. Every call retires every
+    /// reader still in flight and records why, however many calls came before.
+    #[test]
+    fn abandoned_readers_are_retired_on_every_call_while_their_log_is_bounded() {
+        let active_readers = [true, false, true, true];
+        for _ in 0..1_000 {
+            let mut errs: Vec<Option<Error>> = vec![None, None, Some(Error::other("already failed")), None];
+            let mut retire_readers = ShardIndexes::new();
+            retire_abandoned_readers(&mut errs, &mut retire_readers, &active_readers);
+
+            assert_eq!(retire_readers.as_slice(), &[0, 2, 3]);
+            assert!(errs[1].is_none(), "an inactive reader is left alone");
+            assert!(errs[0].is_some() && errs[3].is_some(), "an abandoned reader gets its timeout error");
+            assert!(
+                errs[2]
+                    .as_ref()
+                    .is_some_and(|error| error.to_string().contains("already failed")),
+                "an existing error is kept"
+            );
+        }
+        assert!(ABANDONED_READER_LOGS.claim().is_none(), "the burst used the window");
+    }
     use crate::{
         disk::error::DiskError,
         erasure::coding::{BitrotReader, BitrotWriter},

crates/ecstore/src/layout/endpoints.rs

@@ -2149,6 +2149,15 @@ mod test {
         }
     }
 
+    /// Runs endpoint parsing cases whose local paths do not exist. Unix leaves
+    /// missing paths out of the physical-disk independence check, but Windows
+    /// resolves them to their volume, so paths such as `/d1` and `/d2` count as
+    /// one shared device there. These cases cover parsing, not disk
+    /// independence, which has its own tests below.
+    async fn with_disk_independence_check_bypassed<F: std::future::Future<Output = ()>>(cases: F) {
+        async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true"))], cases).await;
+    }
+
     #[test]
     fn endpoint_parse_errors_do_not_echo_url_credentials() {
         let err = Endpoints::try_from(["http://:topsecret@server/path"].as_slice()).unwrap_err();
@@ -2157,8 +2166,13 @@ mod test {
         assert!(!err.to_string().contains("topsecret"));
     }
 
+    #[serial]
     #[tokio::test]
     async fn explicit_local_endpoint_host_selects_only_exact_host_and_server_port() {
+        with_disk_independence_check_bypassed(explicit_local_endpoint_host_selects_only_exact_host_and_server_port_cases()).await;
+    }
+
+    async fn explicit_local_endpoint_host_selects_only_exact_host_and_server_port_cases() {
         let args = vec![
             "http://rustfs-0.rustfs-headless.ns.svc.cluster.local./data0",
             "http://rustfs-0.rustfs-headless.ns.svc.cluster.local.:9443/data1",
@@ -2921,8 +2935,13 @@ mod test {
         }
     }
 
+    #[serial]
     #[tokio::test]
     async fn test_create_pool_endpoints() {
+        with_disk_independence_check_bypassed(test_create_pool_endpoints_cases()).await;
+    }
+
+    async fn test_create_pool_endpoints_cases() {
         #[derive(Default)]
         struct TestCase<'a> {
             num: usize,
@@ -3541,8 +3560,13 @@ mod test {
         (urls, local_flags)
     }
 
+    #[serial]
     #[tokio::test]
     async fn test_create_server_endpoints() {
+        with_disk_independence_check_bypassed(test_create_server_endpoints_cases()).await;
+    }
+
+    async fn test_create_server_endpoints_cases() {
         let test_cases = [
             // Invalid input.
             ("", vec![], false),

crates/ecstore/src/object_api/types.rs

@@ -975,9 +975,11 @@ pub struct DeleteRecoveryWorkBudget {
     remaining: std::sync::atomic::AtomicU32,
 }
 
-/// Entries one disk may visit per recovery step (discovery plus inspection).
-/// Every disk of every set a request fans out to is admitted up to this much
-/// and charged on completion, so a budget must cover that many disks.
+/// Per-drive unit a request budget is sized by (a typical discovery plus one
+/// inspection). Admission charges what a call visited, not this unit, so it only
+/// scales the request total with the drives a request can reach; it is kept
+/// independent of the per-call caps, which may let one key's probe use more of
+/// the same total on a large folder.
 const DELETE_RECOVERY_RESERVATION_PER_DISK: u32 = 4096 + 65_536;
 
 impl Default for DeleteRecoveryWorkBudget {

crates/ecstore/src/set_disk/mod.rs

@@ -11040,7 +11040,12 @@ mod tests {
         let live = Uuid::new_v4();
         let orphan = Uuid::new_v4();
         let object_dir = dir.path().join("bucket/obj");
-        write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &[live]).await;
+        // Part files are inert to discovery and cost 1/64 of an entry, so the
+        // first step is exhausted by what still costs a full entry each: the
+        // data directories of many versions, all referenced by the metadata.
+        let versions = (0..4200).map(|_| Uuid::new_v4()).collect::<Vec<_>>();
+        let referenced = std::iter::once(live).chain(versions.iter().copied()).collect::<Vec<_>>();
+        write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &referenced).await;
         let live_dir = object_dir.join(live.to_string());
         let orphan_dir = object_dir.join(orphan.to_string());
         fs::create_dir_all(&live_dir).await.expect("create live multipart directory");
@@ -11049,10 +11054,14 @@ mod tests {
             .await
             .expect("write orphan payload");
         let parts_dir = live_dir.clone();
+        let versions_root = object_dir.clone();
         tokio::task::spawn_blocking(move || {
             for part in 1..=4200 {
                 std::fs::write(parts_dir.join(format!("part.{part}")), b"live").expect("write live multipart entry");
             }
+            for version in versions {
+                std::fs::create_dir(versions_root.join(version.to_string())).expect("create referenced version directory");
+            }
         })
         .await
         .expect("create multipart fixture");
@@ -11062,7 +11071,10 @@ mod tests {
             .await
             .expect("read first discovery page");
         assert!(first.transactions.is_empty());
-        assert!(first.next.is_some(), "ordinary multipart entries must exhaust the first scan budget");
+        assert!(
+            first.next.is_some(),
+            "the data directories of many versions must exhaust the first scan step"
+        );
         let last = disk
             .discover_delete_transactions("bucket", Some("obj"), first.next, 1)
             .await

crates/ecstore/src/set_disk/ops/delete_recovery.rs

@@ -1709,6 +1709,37 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn delete_recovery_missing_key_naming_a_folder_of_uuid_named_keys_completes() {
+        let bucket = "recover-uuid-named-folder";
+        let (dirs, store, _) = fixture(bucket).await;
+        // More UUID-named child keys per drive than the former 4096-entry probe
+        // could cover once each child's `xl.meta` and data directory were charged
+        // (three entries per child).
+        const CHILDREN: usize = 1500;
+        for dir in &dirs {
+            let prefix = dir.path().join(bucket).join("uploads");
+            for _ in 0..CHILDREN {
+                let child = prefix.join(Uuid::new_v4().to_string());
+                std::fs::create_dir_all(child.join(Uuid::new_v4().to_string())).expect("child key data dir");
+                std::fs::write(child.join(crate::disk::STORAGE_FORMAT_FILE), b"meta").expect("child key metadata");
+            }
+        }
+        let error = store
+            .delete_object(bucket, "uploads", ObjectOptions::default())
+            .await
+            .expect_err("the folder's own key does not exist");
+        assert!(
+            crate::error::is_err_object_not_found(&error),
+            "a complete inventory keeps the missing-key no-op semantics: {error:?}"
+        );
+        assert!(dirs.iter().all(|dir| {
+            std::fs::read_dir(dir.path().join(bucket).join("uploads"))
+                .map(|entries| entries.count() == CHILDREN)
+                .unwrap_or(false)
+        }));
+    }
+
     #[tokio::test]
     async fn delete_recovery_batch_budget_survives_pool_and_set_fanout() {
         use crate::layout::endpoint::Endpoint;

crates/ecstore/src/set_disk/ops/object.rs

@@ -69,6 +69,92 @@ use super::super::{
 use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards};
 use super::delete_recovery::DeleteRecoveryContext;
 
+const EVENT_SET_DISK_DELETE_OBJECTS_STAGE: &str = "set_disk_delete_objects_stage";
+const DELETE_OBJECTS_STAGE_SLOW_MS: u128 = 1_000;
+
+/// Counts use the stage's actual inputs, not failures inherited from an earlier stage.
+struct DeleteObjectsStageSummary {
+    stage: &'static str,
+    started: Instant,
+    attempted_object_count: usize,
+    attempted_group_count: usize,
+    fence_lost: bool,
+    failed_object_count: usize,
+    failed_group_count: usize,
+    disk_error_counts: std::collections::BTreeMap<u32, usize>,
+}
+
+impl DeleteObjectsStageSummary {
+    fn new(stage: &'static str, attempted_object_count: usize, attempted_group_count: usize) -> Self {
+        Self {
+            stage,
+            started: Instant::now(),
+            attempted_object_count,
+            attempted_group_count,
+            fence_lost: false,
+            failed_object_count: 0,
+            failed_group_count: 0,
+            disk_error_counts: Default::default(),
+        }
+    }
+
+    fn record_disk_failure(&mut self, error: &DiskError, object_count: usize) {
+        self.failed_object_count += object_count;
+        self.failed_group_count += 1;
+        // Wire codes preserve typed disk causes without exposing error messages or paths.
+        *self.disk_error_counts.entry(error.to_u32()).or_default() += 1;
+    }
+
+    fn log(self, pool_index: usize, set_index: usize, disk_index: Option<usize>) {
+        let elapsed_ms = self.started.elapsed().as_millis();
+        let state = if self.fence_lost { "fence_lost" } else { "completed" };
+        let result = if self.fence_lost || self.failed_object_count > 0 {
+            "failed"
+        } else {
+            "ok"
+        };
+        if self.fence_lost || self.failed_object_count > 0 || elapsed_ms >= DELETE_OBJECTS_STAGE_SLOW_MS {
+            warn!(
+                event = EVENT_SET_DISK_DELETE_OBJECTS_STAGE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_SET_DISK,
+                state,
+                result,
+                stage = self.stage,
+                pool_index,
+                set_index,
+                disk_index,
+                attempted_object_count = self.attempted_object_count,
+                attempted_group_count = self.attempted_group_count,
+                failed_object_count = self.failed_object_count,
+                failed_group_count = self.failed_group_count,
+                disk_error_counts = ?self.disk_error_counts,
+                elapsed_ms,
+                "Batch delete stage completed"
+            );
+        } else {
+            debug!(
+                event = EVENT_SET_DISK_DELETE_OBJECTS_STAGE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_SET_DISK,
+                state,
+                result,
+                stage = self.stage,
+                pool_index,
+                set_index,
+                disk_index,
+                attempted_object_count = self.attempted_object_count,
+                attempted_group_count = self.attempted_group_count,
+                failed_object_count = self.failed_object_count,
+                failed_group_count = self.failed_group_count,
+                disk_error_counts = ?self.disk_error_counts,
+                elapsed_ms,
+                "Batch delete stage completed"
+            );
+        }
+    }
+}
+
 #[derive(Default)]
 struct DeleteCommitContext<'a> {
     incarnation: Option<Uuid>,
@@ -9975,6 +10061,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         let dist_erasure = self.ctx.is_dist_erasure().await;
         let mut dist_batch_lock_ids = vec![Vec::new(); self.lockers.len()];
 
+        let mut raw_lock_summary = DeleteObjectsStageSummary::new("raw_acquire", objects.len(), unique_lock_count);
         if opts.no_lock {
             locked_objects = unique_objects;
         } else if dist_erasure {
@@ -10014,8 +10101,13 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         for (i, lock_name) in lock_names.iter().enumerate() {
             if let Some(err) = failed_map.get(&(bucket.to_string(), lock_name.to_string())) {
                 del_errs[i] = Some(Error::other(err.to_string()));
+                raw_lock_summary.failed_object_count += 1;
             }
         }
+        if !opts.no_lock {
+            raw_lock_summary.failed_group_count = failed_map.len();
+            raw_lock_summary.log(self.pool_index, self.set_index, None);
+        }
 
         let outer_namespace_lock_fence = opts.namespace_lock_fence.clone();
         let mut transaction_guards = std::collections::BTreeMap::new();
@@ -10023,12 +10115,18 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             // Every raw batch lock is acquired first. Canonical sorting also
             // gives aliases of one disk key exactly one transaction owner.
             let keys: std::collections::BTreeSet<_> = locked_objects.iter().cloned().collect();
-            // Bulk delete is the only multi-key transaction-lock holder; one
-            // batched lock round replaces one quorum round-trip per key and is
-            // deadlock-free by construction (a single `lock_batch` RPC per lock
-            // client, global (shard, key) order at each server). Per-key
-            // failures keep the set layer's per-key error reporting.
+            // Batch unresolved keys per peer under one total acquisition budget.
+            // Confirmed cleanup precedes retries; each key keeps its own quorum
+            // and failure result at the set layer.
             let key_names: Vec<&str> = keys.iter().map(|key| key.as_str()).collect();
+            let mut transaction_summary = DeleteObjectsStageSummary::new(
+                "transaction_acquire",
+                lock_names
+                    .iter()
+                    .filter(|name| locked_objects.contains(name.as_ref()))
+                    .count(),
+                key_names.len(),
+            );
             let acquired = self
                 .acquire_delete_transaction_locks_batch(bucket, &key_names, super::super::get_lock_acquire_timeout())
                 .await;
@@ -10039,14 +10137,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                         transaction_guards.insert(key, guard);
                     }
                     Err(error) => {
+                        transaction_summary.failed_group_count += 1;
                         for (index, lock_name) in lock_names.iter().enumerate() {
                             if lock_name.as_ref() == key {
                                 del_errs[index] = Some(error.clone());
+                                transaction_summary.failed_object_count += 1;
                             }
                         }
                     }
                 }
             }
+            transaction_summary.log(self.pool_index, self.set_index, None);
         }
         let transaction_guards = Arc::new(transaction_guards);
         let current_generations = opts.conditional_delete_generations.as_deref();
@@ -10504,11 +10605,18 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             .as_ref()
             .is_some_and(NamespaceLockFence::is_lock_lost)
         {
+            let mut fence_summary = DeleteObjectsStageSummary::new(
+                "fence_before_prepare",
+                del_errs.iter().filter(|error| error.is_none()).count(),
+                0,
+            );
+            fence_summary.fence_lost = true;
             if dist_erasure {
                 self.release_dist_delete_object_locks_batch(dist_batch_lock_ids).await;
             }
             for (index, object) in objects.iter().enumerate() {
                 if del_errs[index].is_none() {
+                    fence_summary.failed_object_count += 1;
                     del_errs[index] = Some(Error::NamespaceLockQuorumUnavailable {
                         mode: "delete_objects_commit",
                         bucket: bucket.to_string(),
@@ -10518,6 +10626,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     });
                 }
             }
+            fence_summary.log(self.pool_index, self.set_index, None);
             return (del_objects, del_errs, accounting);
         }
 
@@ -10584,8 +10693,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
 
         // let mut errors = Vec::with_capacity(disks.len());
 
-        for disk in disks.iter() {
+        for (disk_index, disk) in disks.iter().enumerate() {
             let disk_namespace_owner = namespace_owner.clone();
+            let group_inputs = &vers;
             let vers = vers.clone();
             let namespace_lock_fence = opts.namespace_lock_fence.clone();
             let lock_lost_during_commit = Arc::clone(&lock_lost_during_commit);
@@ -10594,7 +10704,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     lock_lost_during_commit.store(true, std::sync::atomic::Ordering::Release);
                     return (0..vers.len()).map(|_| Some(DiskError::DiskOngoingReq)).collect();
                 }
-                if let Some(disk) = disk {
+                let mut prepare_summary = DeleteObjectsStageSummary::new(
+                    "prepare",
+                    group_inputs.iter().map(|group| group.versions.len()).sum(),
+                    group_inputs.len(),
+                );
+                let errors = if let Some(disk) = disk {
                     let options = DeleteOptions {
                         old_data_dir: Some(rollback_dir),
                         delete_recovery_bucket_incarnation: opts.expected_bucket_incarnation_id,
@@ -10662,14 +10777,28 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                         errs.push(Some(DiskError::DiskNotFound));
                     }
                     errs
+                };
+                for (error, group) in errors.iter().zip(group_inputs) {
+                    if let Some(error) = error {
+                        prepare_summary.record_disk_failure(error, group.versions.len());
+                    }
                 }
+                prepare_summary.log(self.pool_index, self.set_index, Some(disk_index));
+                errors
             });
         }
 
         let results = join_all(futures).await;
         if transaction_guards.values().any(|guard| guard.is_lock_lost()) {
+            let mut fence_summary = DeleteObjectsStageSummary::new(
+                "transaction_fence_after_prepare",
+                del_errs.iter().filter(|error| error.is_none()).count(),
+                0,
+            );
+            fence_summary.fence_lost = true;
             for (index, object) in objects.iter().enumerate() {
                 if del_errs[index].is_none() {
+                    fence_summary.failed_object_count += 1;
                     accounting[index] = None;
                     del_errs[index] = Some(StorageError::NamespaceLockQuorumUnavailable {
                         mode: "delete_transaction",
@@ -10680,6 +10809,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     });
                 }
             }
+            fence_summary.log(self.pool_index, self.set_index, None);
             if dist_erasure {
                 self.release_dist_delete_object_locks_batch(dist_batch_lock_ids).await;
             }
@@ -10746,8 +10876,15 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         }
 
         if lock_lost_during_commit.load(std::sync::atomic::Ordering::Acquire) {
+            let mut fence_summary = DeleteObjectsStageSummary::new(
+                "fence_during_prepare",
+                del_errs.iter().filter(|error| error.is_none()).count(),
+                0,
+            );
+            fence_summary.fence_lost = true;
             for (index, object) in objects.iter().enumerate() {
                 if del_errs[index].is_none() {
+                    fence_summary.failed_object_count += 1;
                     del_errs[index] = Some(Error::NamespaceLockQuorumUnavailable {
                         mode: "delete_objects_commit",
                         bucket: bucket.to_string(),
@@ -10757,6 +10894,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     });
                 }
             }
+            fence_summary.log(self.pool_index, self.set_index, None);
         }
 
         self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
@@ -10775,6 +10913,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             // Mark all attempted groups before starting publication. A lost
             // response can hide a persisted decision and must never roll back.
             let groups = groups.collect::<Vec<_>>();
+            let mut receipt_summary = DeleteObjectsStageSummary::new(
+                "receipt",
+                groups.iter().map(|index| vers[*index].versions.len()).sum(),
+                groups.len(),
+            );
             published_groups.extend(groups.iter().map(|index| vers[*index].name.clone()));
             // Journal publish fans out per object per drive; keep enough
             // publications in flight to feed the drives' durable-mutation
@@ -10822,11 +10965,13 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             for (index, result) in publications {
                 let group = &vers[index];
                 if let Err(error) = result {
+                    receipt_summary.record_disk_failure(&error, group.versions.len());
                     for version in &group.versions {
                         del_errs[version.idx] = Some(to_object_err(error.clone().into(), vec![bucket, &group.name]));
                     }
                 }
             }
+            receipt_summary.log(self.pool_index, self.set_index, None);
         }
         let mut rollback_futures = Vec::new();
         let committed_receipt_indices =
@@ -28897,3 +29042,24 @@ mod delete_recovery_inventory_tests {
 
 #[cfg(all(test, feature = "test-util"))]
 mod data_movement_prefix_cleanup_tests;
+
+#[cfg(test)]
+mod delete_objects_stage_summary_tests {
+    use super::*;
+
+    #[test]
+    fn batch_delete_diagnostics_count_groups_and_versions_separately() {
+        let mut summary = DeleteObjectsStageSummary::new("receipt", 6, 3);
+        summary.record_disk_failure(&DiskError::ErasureWriteQuorum, 3);
+        summary.record_disk_failure(&DiskError::ErasureWriteQuorum, 2);
+
+        assert_eq!(summary.attempted_object_count, 6);
+        assert_eq!(summary.attempted_group_count, 3);
+        assert_eq!(summary.failed_object_count, 5);
+        assert_eq!(summary.failed_group_count, 2);
+        assert_eq!(
+            summary.disk_error_counts,
+            std::collections::BTreeMap::from([(DiskError::ErasureWriteQuorum.to_u32(), 2)])
+        );
+    }
+}

crates/ecstore/src/store/object.rs

@@ -2533,6 +2533,7 @@ fn return_batch_delete_lock_error_with_accounting(
 }
 
 const EVENT_DETACHED_DELETE_FAILED: &str = "detached_delete_failed";
+const EVENT_BULK_DELETE_LOCK_ACQUIRE: &str = "bulk_delete_lock_acquire";
 const LOG_SUBSYSTEM_STORE_DELETE: &str = "store_delete";
 
 /// A detached delete only fails to join when it panicked or the runtime is
@@ -3567,12 +3568,10 @@ impl ECStore {
     }
 
     /// Acquire the per-object write locks a bulk delete needs before any
-    /// mutation, one batched lock round for the whole key set. Bulk delete is
-    /// the only multi-key write-lock holder, and the batched round is
-    /// deadlock-free by construction (one `lock_batch` RPC per lock client,
-    /// global key order at each server), so no optimistic wave or serial
-    /// fallback is needed. Outcomes are per key: a locked-out key fails alone
-    /// and the rest of the batch proceeds.
+    /// mutation. Each peer receives one batch of unresolved keys per round;
+    /// retries share the original deadline and require confirmed cleanup of
+    /// partial grants. Outcomes remain independent per key so other keys can
+    /// proceed when a contended key exhausts its budget.
     async fn acquire_delete_objects_write_locks(
         &self,
         bucket: &str,
@@ -3584,6 +3583,7 @@ impl ECStore {
         }
 
         let object_names = sorted_unique_delete_object_names(objects);
+        let acquire_started = Instant::now();
         let acquired = self.pools[0].disk_set[0]
             .acquire_object_write_locks_batch(bucket, &object_names, get_lock_acquire_timeout())
             .await;
@@ -3614,6 +3614,35 @@ impl ECStore {
                 }
             }
         }
+        if failures.is_empty() {
+            debug!(
+                event = EVENT_BULK_DELETE_LOCK_ACQUIRE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_STORE_DELETE,
+                result = "success",
+                stage = "raw_acquire",
+                requested_objects = objects.len(),
+                unique_keys = object_names.len(),
+                successful_keys = guards.len(),
+                failed_keys = failures.len(),
+                elapsed_ms = acquire_started.elapsed().as_millis(),
+                "Bulk delete namespace lock acquisition completed"
+            );
+        } else {
+            warn!(
+                event = EVENT_BULK_DELETE_LOCK_ACQUIRE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_STORE_DELETE,
+                result = "partial_failure",
+                stage = "raw_acquire",
+                requested_objects = objects.len(),
+                unique_keys = object_names.len(),
+                successful_keys = guards.len(),
+                failed_keys = failures.len(),
+                elapsed_ms = acquire_started.elapsed().as_millis(),
+                "Bulk delete namespace lock acquisition failed for some keys"
+            );
+        }
         opts.no_lock = true;
         for signal in guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal) {
             opts.add_namespace_lock_lost_signal(signal);
@@ -14405,8 +14434,7 @@ mod batch_delete_lock_tests {
         let second = run(second);
 
         // Whatever mix of per-key lock outcomes the overlap produces, both
-        // batches must finish: the batched round is deadlock-free by
-        // construction (global key order at each lock server).
+        // batches must finish within their acquisition and mutation budgets.
         for batch in [first, second] {
             let (_, errors, _) = tokio::time::timeout(Duration::from_secs(30), batch)
                 .await

crates/gateway-adapter/Cargo.toml

@@ -0,0 +1,48 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[package]
+name = "rustfs-gateway-adapter"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+homepage.workspace = true
+description = "RustFS implementations of the rustfs-gateway extension traits (credentials, policy facts, virtual hosts, bucket owner and request id)."
+keywords = ["rustfs", "gateway", "s3", "iam"]
+categories = ["web-programming", "authentication"]
+authors.workspace = true
+publish = false
+
+[package.metadata.gateway]
+# Documentary only: this crate is a ring-2 consumer of the gateway (it depends on
+# RustFS business crates). The RustFS repository does not run the gateway's ring guard.
+ring = 2
+
+[lints]
+workspace = true
+
+[dependencies]
+http = { workspace = true }
+rustfs-gateway = { workspace = true }
+rustfs-credentials = { workspace = true }
+rustfs-iam = { workspace = true }
+rustfs-policy = { workspace = true }
+serde_json = { workspace = true }
+
+[dev-dependencies]
+rustfs-gateway-http = { workspace = true }
+time = { workspace = true }
+tokio = { workspace = true, features = ["macros", "rt"] }

crates/gateway-adapter/src/authz_audit.rs

@@ -0,0 +1,548 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! The gateway's [`AuthzAuditSink`]: the legacy denial record for a refusal the gateway enforced.
+//!
+//! Responsible for: one [`GatewayDenialRecord`] for each authorization decision the gateway made
+//! at an enforcing stage that ended the request with a refusal (`Deny` or `Indeterminate`), after
+//! the legacy audit switch and the legacy denial rate limiter have both admitted it.
+//! NOT responsible for: the authoritative authorization decision. On the gateway stack that is the
+//! legacy access check in owned form, run by the handler before the use case, and it writes its own
+//! denial record (`DenialContext::record`, `rustfs/src/storage/access.rs`). The gateway
+//! `Authorizer` is only an earlier gate that may refuse what the legacy check certainly refuses, so
+//! a request it lets through is not decided yet, and a request it refuses never reaches the check.
+//! Nor is this module responsible for building the `rustfs_audit::AuditEntry`, filling its
+//! request-line fields, or dispatching it: that is the binary's [`DenialAuditSink`].
+//! Upstream: the gateway service, once per decision (`AuthzAuditSink::on_decision`). Downstream:
+//! the binary's [`DenialAuditSink`].
+//!
+//! # What is recorded, and what is not
+//!
+//! - An `Allow` is never recorded: at the early gate it is not the final decision, and a handler
+//!   denial after it would contradict the record.
+//! - The missing-object visibility probe is never recorded. For `GetObject` the gateway also asks
+//!   `s3:ListBucket`, only to choose between `404` and `403` for an absent object; that answer
+//!   refuses nothing, yet it reaches the sink as an `Input` event of its own. Legacy keeps the same
+//!   probe out of the audit (`suppress_denial_audit`, `rustfs/src/app/object/head.rs`).
+//! - Every other `Deny` or `Indeterminate` ended its request: a route-stage refusal is answered
+//!   before the body is read, and an input-stage refusal either fails the stage or prevents the
+//!   gateway core from constructing the authorized request. One refused request therefore yields
+//!   exactly one record.
+//!
+//! # Telling the visibility probe apart
+//!
+//! At the pinned gateway rev every event's `resources` starts with the request the decision was
+//! about. For the route and input events that is the route target, whose `action` equals its
+//! `route_action` (`crates/gateway/src/service.rs`, the route request and every asked question,
+//! and the input stage's route request). The visibility event alone is built from the route
+//! target with only `action` replaced by `s3:ListBucket`, keeping the target's `route_action`
+//! (`crates/gateway/src/ext/authz_audit.rs`, `emit_input_safely`; `crates/gateway/src/ext/authorizer.rs`,
+//! `InputAuthzRequest::new`). An `Input` event whose first resource asks an action other than its
+//! route action is the visibility event.
+//!
+//! # Known limit: the denied action and resource
+//!
+//! `trigger` (and `api.name`), `api.bucket` and `api.object` are the event's `action`, `bucket` and
+//! `key`: the operation's declared action and primary target. The event carries no per-resource
+//! decision, so the sink cannot name a derived resource that was refused. This is exact only while
+//! every early-gate refusal concerns the declared action and target. An operation whose denial can
+//! concern a second resource or a version-specific action (a copy source, a versioned read) must
+//! therefore be decided by the handler, never refused by the early gate: legacy records those with
+//! the source bucket and object, and with `s3:GetObjectVersion`.
+//!
+//! # What may not reach a record
+//!
+//! The record is built field by field from the event's `action`, `bucket`, `key`, `request_id`,
+//! `decision` and the verified identity's access key id. Nothing is copied from `resources`,
+//! `subject`, `auth_scheme`, a version id or a copy-source identity, and nothing is formatted from a
+//! `Debug` implementation. The event carries no secret key, session token or signature.
+
+use rustfs_gateway::{AuthzAuditEvent, AuthzAuditSink, AuthzStage, Decision};
+
+/// `tags.denialReason` of a `Deny` the gateway enforced. The event carries the decision, not one of
+/// the legacy reasons (`iam_explicit_deny`, `no_policy_allows_action`, ...).
+pub const DENY_REASON: &str = "authorization_denied";
+/// `tags.denialReason` of an `Indeterminate` the gateway enforced: no policy reading could answer.
+pub const INDETERMINATE_REASON: &str = "authorization_indeterminate";
+
+/// What the legacy denial rate limiter answered for one refusal.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum DenialAdmission {
+    /// Record it; `suppressed` refusals of the same caller class were dropped since the previous
+    /// record, and are reported as `tags.suppressedDenials` when non-zero.
+    Emit {
+        /// Refusals dropped since the previous record of this caller class.
+        suppressed: u64,
+    },
+    /// Drop it.
+    Suppress,
+}
+
+/// One refusal the gateway enforced, in the fields of the legacy authorization-denial entry the
+/// event can supply.
+///
+/// The binary builds the entry as legacy `authorization_denial_entry` (`rustfs/src/storage/helper.rs`)
+/// does: `version` `1.0`, `event` `AuditS3Operation`, `trigger` and `api.name` [`Self::action`],
+/// `api.status` `failure`, `api.status_code` `403`, `error` `Access Denied`, `tags.denialReason`
+/// [`Self::denial_reason`], and `tags.suppressedDenials` only when [`Self::suppressed_denials`] is
+/// non-zero.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct GatewayDenialRecord {
+    /// `trigger` and `api.name`: the operation's declared IAM action, `s3:GetObject`.
+    pub action: String,
+    /// `api.bucket`, when the request addressed one.
+    pub bucket: Option<String>,
+    /// `api.object`, when the request addressed one.
+    pub object: Option<String>,
+    /// `requestID`: the identifier the caller received.
+    pub request_id: String,
+    /// `accessKey`: the verified caller; `None` for an anonymous caller, as legacy omits it.
+    pub access_key: Option<String>,
+    /// `tags.denialReason`: [`DENY_REASON`] or [`INDETERMINATE_REASON`].
+    pub denial_reason: &'static str,
+    /// `tags.suppressedDenials`, from [`DenialAdmission::Emit`].
+    pub suppressed_denials: u64,
+}
+
+/// Where admitted refusals go. Implemented by the binary over the legacy denial auditor.
+///
+/// The three methods are called in order, at most once each per refusal, on the request's task,
+/// and must neither block nor panic.
+///
+/// # What the implementation owes the legacy entry
+///
+/// - [`Self::enabled`] is the legacy audit switch (`is_audit_module_enabled`), and [`Self::admit`]
+///   the process-wide legacy `DenialRateLimiter` itself, not a second instance: gateway and handler
+///   refusals share one token bucket per caller class.
+/// - `time`, `parentUser` and `requestClaims` are filled as legacy fills them.
+/// - `remotehost`, `userAgent`, `requestHost`, `requestPath` and `requestQuery` come from the
+///   request the outer layer saw. **`requestQuery` is not the URI query.** Legacy fills it with
+///   `rustfs_targets::net::extract_params_header` over the request *headers*, which replaces the
+///   value of every sensitive header (`authorization`, `x-amz-security-token`, the SSE-C key
+///   headers, ...) with the redaction marker. The raw URI query must never be put there: on a
+///   presigned request it carries `X-Amz-Signature`, `X-Amz-Credential` and
+///   `X-Amz-Security-Token`.
+pub trait DenialAuditSink: Send + Sync + 'static {
+    /// Whether denial auditing is switched on. Asked first; `false` drops the refusal before the
+    /// rate limiter is consulted.
+    fn enabled(&self) -> bool;
+    /// Asks the rate limiter for one refusal of this caller class. Asked before any record exists.
+    fn admit(&self, anonymous: bool) -> DenialAdmission;
+    /// Receives one admitted record.
+    fn record(&self, record: GatewayDenialRecord);
+}
+
+/// An [`AuthzAuditSink`] recording only the refusals the gateway enforced.
+pub struct RustfsAuthzAudit<S> {
+    sink: S,
+}
+
+impl<S: DenialAuditSink> RustfsAuthzAudit<S> {
+    /// An audit sink reporting into `sink`.
+    #[must_use]
+    pub fn new(sink: S) -> Self {
+        Self { sink }
+    }
+}
+
+impl<S: DenialAuditSink> AuthzAuditSink for RustfsAuthzAudit<S> {
+    fn on_decision(&self, event: &AuthzAuditEvent<'_>) {
+        let denial_reason = match event.decision {
+            Decision::Allow => return,
+            Decision::Deny => DENY_REASON,
+            Decision::Indeterminate => INDETERMINATE_REASON,
+        };
+        if is_visibility_probe(event) || !self.sink.enabled() {
+            return;
+        }
+        let anonymous = event.identity.is_none();
+        let DenialAdmission::Emit { suppressed } = self.sink.admit(anonymous) else {
+            return;
+        };
+        self.sink.record(GatewayDenialRecord {
+            action: event.action.to_owned(),
+            bucket: event.bucket.map(|bucket| bucket.as_str().to_owned()),
+            object: event.key.map(|key| key.as_str().to_owned()),
+            request_id: event.request_id.as_str().to_owned(),
+            access_key: event.identity.map(|identity| identity.access_key_id().to_owned()),
+            denial_reason,
+            suppressed_denials: suppressed,
+        });
+    }
+}
+
+/// Whether `event` is the missing-object visibility probe, which refuses nothing. See the module
+/// documentation for why the first resource identifies it.
+fn is_visibility_probe(event: &AuthzAuditEvent<'_>) -> bool {
+    event.stage == AuthzStage::Input
+        && event
+            .resources
+            .first()
+            .is_some_and(|first| first.action != first.route_action)
+}
+
+#[cfg(test)]
+#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
+mod tests {
+    use super::*;
+    use core::time::Duration;
+    use rustfs_gateway::{
+        AuthSchemeRef, AuthzRequest, BucketName, Identity, ObjectKey, PolicySnapshot, RequestId, ResourceShape, SessionBinding,
+        Subject, TargetOrigin,
+    };
+    use std::sync::Mutex;
+
+    const ACCESS_KEY: &str = "AKIDCALLER0001";
+    const BUCKET: &str = "audit-bucket";
+    const KEY: &str = "reports/q3.csv";
+
+    /// A sink with a scripted switch and limiter that records every call it receives.
+    struct Probe {
+        enabled: bool,
+        admission: DenialAdmission,
+        switch_asked: Mutex<usize>,
+        limiter_asked: Mutex<Vec<bool>>,
+        records: Mutex<Vec<GatewayDenialRecord>>,
+    }
+
+    impl Probe {
+        fn new(enabled: bool, admission: DenialAdmission) -> Self {
+            Self {
+                enabled,
+                admission,
+                switch_asked: Mutex::new(0),
+                limiter_asked: Mutex::new(Vec::new()),
+                records: Mutex::new(Vec::new()),
+            }
+        }
+
+        fn open() -> Self {
+            Self::new(true, DenialAdmission::Emit { suppressed: 0 })
+        }
+
+        fn records(&self) -> Vec<GatewayDenialRecord> {
+            self.records.lock().expect("not poisoned").clone()
+        }
+
+        fn switch_asked(&self) -> usize {
+            *self.switch_asked.lock().expect("not poisoned")
+        }
+
+        fn limiter_asked(&self) -> Vec<bool> {
+            self.limiter_asked.lock().expect("not poisoned").clone()
+        }
+    }
+
+    impl DenialAuditSink for Probe {
+        fn enabled(&self) -> bool {
+            *self.switch_asked.lock().expect("not poisoned") += 1;
+            self.enabled
+        }
+
+        fn admit(&self, anonymous: bool) -> DenialAdmission {
+            self.limiter_asked.lock().expect("not poisoned").push(anonymous);
+            self.admission
+        }
+
+        fn record(&self, record: GatewayDenialRecord) {
+            self.records.lock().expect("not poisoned").push(record);
+        }
+    }
+
+    struct Fixture {
+        request_id: RequestId,
+        bucket: BucketName,
+        key: ObjectKey,
+        identity: Identity,
+        snapshot: PolicySnapshot,
+    }
+
+    fn fixture() -> Fixture {
+        Fixture {
+            request_id: RequestId::from_bits(0x5EED),
+            bucket: BucketName::new(BUCKET).expect("a valid bucket"),
+            key: ObjectKey::new(KEY).expect("a valid key"),
+            identity: Identity::new(ACCESS_KEY).expect("a valid access key"),
+            snapshot: PolicySnapshot::empty(),
+        }
+    }
+
+    /// The route target of a GetObject, as the gateway builds it: its action is its route action.
+    fn target<'a>(f: &'a Fixture, identity: Option<&'a Identity>) -> AuthzRequest<'a> {
+        AuthzRequest {
+            operation: "GetObject",
+            action: "s3:GetObject",
+            resource: ResourceShape::Object,
+            bucket: Some(&f.bucket),
+            key: Some(&f.key),
+            copy_source_identity: None,
+            version_id: None,
+            route_action: "s3:GetObject",
+            route_bucket: Some(&f.bucket),
+            route_key: Some(&f.key),
+            identity,
+            target_origin: TargetOrigin::Path,
+            subject: None,
+        }
+    }
+
+    /// The visibility question, built from the route target as `InputAuthzRequest::new` builds it.
+    fn visibility<'a>(route: &AuthzRequest<'a>) -> AuthzRequest<'a> {
+        AuthzRequest {
+            action: "s3:ListBucket",
+            resource: ResourceShape::Bucket,
+            copy_source_identity: None,
+            version_id: None,
+            ..*route
+        }
+    }
+
+    /// A route or input event of a GetObject about `resources`.
+    fn event<'a>(
+        f: &'a Fixture,
+        stage: AuthzStage,
+        decision: Decision,
+        resources: &'a [AuthzRequest<'a>],
+        identity: Option<&'a Identity>,
+    ) -> AuthzAuditEvent<'a> {
+        AuthzAuditEvent {
+            request_id: &f.request_id,
+            stage,
+            operation: "GetObject",
+            action: "s3:GetObject",
+            resource: ResourceShape::Object,
+            bucket: Some(&f.bucket),
+            key: Some(&f.key),
+            resources,
+            auth_scheme: if identity.is_some() {
+                AuthSchemeRef::SigV4Header
+            } else {
+                AuthSchemeRef::Anonymous
+            },
+            identity,
+            target_origin: TargetOrigin::Path,
+            policy_snapshot: Some(f.snapshot.id()),
+            decision,
+            elapsed: Duration::ZERO,
+        }
+    }
+
+    /// The visibility event, as `emit_input_safely` builds it from the input event.
+    fn visibility_event<'a>(
+        input: &AuthzAuditEvent<'a>,
+        request: &'a AuthzRequest<'a>,
+        decision: Decision,
+    ) -> AuthzAuditEvent<'a> {
+        AuthzAuditEvent {
+            stage: AuthzStage::Input,
+            action: request.action,
+            resource: request.resource,
+            bucket: request.bucket,
+            key: request.key,
+            resources: std::slice::from_ref(request),
+            decision,
+            ..*input
+        }
+    }
+
+    /// Drives the events of one GetObject through `audit` in the gateway's order: the route stage,
+    /// then (when the route allowed) the input stage followed by its visibility question.
+    fn get_object(
+        audit: &RustfsAuthzAudit<Probe>,
+        f: &Fixture,
+        identity: Option<&Identity>,
+        route: Decision,
+        input: Decision,
+        probe: Decision,
+    ) {
+        let route_target = target(f, identity);
+        let asked = [route_target];
+        audit.on_decision(&event(f, AuthzStage::Route, route, &asked, identity));
+        if route != Decision::Allow {
+            return;
+        }
+        let input_event = event(f, AuthzStage::Input, input, &asked, identity);
+        audit.on_decision(&input_event);
+        let question = visibility(&route_target);
+        audit.on_decision(&visibility_event(&input_event, &question, probe));
+    }
+
+    fn expected(denial_reason: &'static str, access_key: Option<&str>, suppressed_denials: u64) -> GatewayDenialRecord {
+        GatewayDenialRecord {
+            action: "s3:GetObject".to_owned(),
+            bucket: Some(BUCKET.to_owned()),
+            object: Some(KEY.to_owned()),
+            request_id: RequestId::from_bits(0x5EED).as_str().to_owned(),
+            access_key: access_key.map(str::to_owned),
+            denial_reason,
+            suppressed_denials,
+        }
+    }
+
+    /// Positive — a route-stage refusal is one record carrying the legacy denial entry's values,
+    /// and `Indeterminate` keeps a reason of its own.
+    #[test]
+    fn p_route_refusal_is_one_record_with_the_legacy_values() {
+        for (decision, reason) in [(Decision::Deny, DENY_REASON), (Decision::Indeterminate, INDETERMINATE_REASON)] {
+            let f = fixture();
+            let audit = RustfsAuthzAudit::new(Probe::open());
+            get_object(&audit, &f, Some(&f.identity), decision, Decision::Allow, Decision::Allow);
+            assert_eq!(audit.sink.records(), [expected(reason, Some(ACCESS_KEY), 0)], "{decision:?}");
+            assert_eq!(audit.sink.limiter_asked(), [false], "{decision:?}");
+        }
+    }
+
+    /// Positive — an input-stage refusal of a GetObject is exactly one record, though its
+    /// visibility question was refused as well.
+    #[test]
+    fn p_enforcing_input_refusal_is_recorded_once() {
+        let f = fixture();
+        let audit = RustfsAuthzAudit::new(Probe::open());
+        get_object(&audit, &f, Some(&f.identity), Decision::Allow, Decision::Deny, Decision::Deny);
+        assert_eq!(audit.sink.records(), [expected(DENY_REASON, Some(ACCESS_KEY), 0)]);
+        assert_eq!(audit.sink.limiter_asked(), [false]);
+    }
+
+    /// Positive — the refusals the limiter dropped before this one are carried on the record.
+    #[test]
+    fn p_suppressed_count_is_carried() {
+        let f = fixture();
+        let audit = RustfsAuthzAudit::new(Probe::new(true, DenialAdmission::Emit { suppressed: 7 }));
+        get_object(&audit, &f, Some(&f.identity), Decision::Deny, Decision::Allow, Decision::Allow);
+        assert_eq!(audit.sink.records(), [expected(DENY_REASON, Some(ACCESS_KEY), 7)]);
+    }
+
+    /// Negative — an `Allow` at either stage records nothing and asks neither the switch nor the
+    /// limiter, so it cannot spend a token a real refusal needs.
+    #[test]
+    fn n_allow_is_never_recorded() {
+        let f = fixture();
+        let audit = RustfsAuthzAudit::new(Probe::open());
+        let asked = [target(&f, Some(&f.identity))];
+        for stage in [AuthzStage::Route, AuthzStage::Input] {
+            audit.on_decision(&event(&f, stage, Decision::Allow, &asked, Some(&f.identity)));
+        }
+        assert!(audit.sink.records().is_empty());
+        assert_eq!((audit.sink.switch_asked(), audit.sink.limiter_asked()), (0, Vec::new()));
+    }
+
+    /// Negative — a GetObject that succeeds for a caller refused `s3:ListBucket` records nothing:
+    /// the visibility question refuses no request, whatever it answered.
+    #[test]
+    fn n_visibility_probe_is_never_recorded() {
+        for probe in [Decision::Deny, Decision::Indeterminate] {
+            let f = fixture();
+            let audit = RustfsAuthzAudit::new(Probe::open());
+            get_object(&audit, &f, Some(&f.identity), Decision::Allow, Decision::Allow, probe);
+            assert!(audit.sink.records().is_empty(), "{probe:?}");
+            assert_eq!((audit.sink.switch_asked(), audit.sink.limiter_asked()), (0, Vec::new()), "{probe:?}");
+        }
+    }
+
+    /// Negative — with the audit switch off a refusal records nothing and the limiter is never
+    /// asked, as legacy checks the switch first.
+    #[test]
+    fn n_audit_switch_off_records_nothing() {
+        let f = fixture();
+        let audit = RustfsAuthzAudit::new(Probe::new(false, DenialAdmission::Emit { suppressed: 0 }));
+        get_object(&audit, &f, Some(&f.identity), Decision::Deny, Decision::Allow, Decision::Allow);
+        assert!(audit.sink.records().is_empty());
+        assert_eq!((audit.sink.switch_asked(), audit.sink.limiter_asked()), (1, Vec::new()));
+    }
+
+    /// Negative — a refusal the limiter suppresses records nothing.
+    #[test]
+    fn n_rate_limiter_suppression_records_nothing() {
+        let f = fixture();
+        let audit = RustfsAuthzAudit::new(Probe::new(true, DenialAdmission::Suppress));
+        get_object(&audit, &f, Some(&f.identity), Decision::Deny, Decision::Allow, Decision::Allow);
+        assert!(audit.sink.records().is_empty());
+        assert_eq!(audit.sink.limiter_asked(), [false]);
+    }
+
+    /// Negative — an anonymous caller has no access key, not even an empty one, and is limited as
+    /// the anonymous class; the authenticated refusal beside it is limited as the other class.
+    #[test]
+    fn n_anonymous_caller_has_no_access_key() {
+        let f = fixture();
+        let audit = RustfsAuthzAudit::new(Probe::open());
+        get_object(&audit, &f, None, Decision::Deny, Decision::Allow, Decision::Allow);
+        get_object(&audit, &f, Some(&f.identity), Decision::Deny, Decision::Allow, Decision::Allow);
+        assert_eq!(
+            audit.sink.records(),
+            [expected(DENY_REASON, None, 0), expected(DENY_REASON, Some(ACCESS_KEY), 0)]
+        );
+        assert_eq!(audit.sink.limiter_asked(), [true, false]);
+    }
+
+    /// Negative — values planted in the event's `resources`, version ids, subject, auth scheme,
+    /// target origin and session binding leave the record exactly as it is without them, and none
+    /// of their text reaches its `Debug` output.
+    #[test]
+    fn n_event_fields_outside_the_record_do_not_reach_it() {
+        const SECRETS: [&str; 3] = [
+            "wJalrXUtnFEMI/K7MDENG/bPxRfiCYSECRETKEY",
+            "AQoDYXdzEJrSESSIONTOKEN",
+            "fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024",
+        ];
+        const PLANTED_BUCKET: &str = "planted-source-bucket";
+        const PLANTED_KEY: &str = "planted/source-key";
+        let f = fixture();
+        let source_bucket = BucketName::new(PLANTED_BUCKET).expect("a valid bucket");
+        let source_key = ObjectKey::new(PLANTED_KEY).expect("a valid key");
+        let binding = SessionBinding::new(SECRETS[1], 4_000_000_000).expect("a binding");
+        let session = Identity::new(ACCESS_KEY).expect("a key").with_session_binding(binding);
+        let subject = Subject::Caller;
+        let planted_target = AuthzRequest {
+            version_id: Some(SECRETS[0]),
+            target_origin: TargetOrigin::Host,
+            subject: Some(&subject),
+            ..target(&f, Some(&session))
+        };
+        let derived = AuthzRequest {
+            action: "s3:GetObjectVersion",
+            bucket: Some(&source_bucket),
+            key: Some(&source_key),
+            version_id: Some(SECRETS[2]),
+            ..planted_target
+        };
+        let planted_resources = [planted_target, derived];
+        let plain_resources = [target(&f, Some(&f.identity))];
+        for stage in [AuthzStage::Route, AuthzStage::Input] {
+            let plain = RustfsAuthzAudit::new(Probe::open());
+            plain.on_decision(&event(&f, stage, Decision::Deny, &plain_resources, Some(&f.identity)));
+            let planted = RustfsAuthzAudit::new(Probe::open());
+            planted.on_decision(&AuthzAuditEvent {
+                auth_scheme: AuthSchemeRef::SigV4Presigned,
+                target_origin: TargetOrigin::Host,
+                ..event(&f, stage, Decision::Deny, &planted_resources, Some(&session))
+            });
+            let records = planted.sink.records();
+            assert_eq!(records, plain.sink.records(), "{stage:?}");
+            assert_eq!(records.len(), 1, "{stage:?}");
+            let rendered = format!("{records:?}");
+            for planted_text in SECRETS.into_iter().chain([
+                PLANTED_BUCKET,
+                PLANTED_KEY,
+                "GetObjectVersion",
+                "Presigned",
+                "Host",
+                "Caller",
+                "Authz",
+            ]) {
+                assert!(!rendered.contains(planted_text), "{planted_text} reached the record: {rendered}");
+            }
+        }
+    }
+}

crates/gateway-adapter/src/bucket_owner.rs

@@ -0,0 +1,215 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! The gateway's [`BucketOwnerSource`]: the one owner RustFS reports, for a bucket that exists.
+//!
+//! Responsible for: confirming through a [`BucketLookup`] that a bucket exists, answering the
+//! single owner id the deployment reports ([`RustfsBucketOwners::new`]) for it, and turning every
+//! outcome that is not a usable owner into the gateway's one refusal, [`BucketOwnerError`].
+//! NOT responsible for: choosing the owner id (the binary passes the constant it already reports
+//! as the owner in listings), comparing it with `x-amz-expected-bucket-owner` (the gateway does,
+//! byte for byte), or authorizing anything.
+//! Upstream: the gateway's expected-owner check, which consults this source only when a request
+//! carries the header, after authorization allowed it. Downstream: the lookup the binary supplies
+//! over the ecstore bucket metadata system.
+//!
+//! # What an owner is in RustFS
+//!
+//! RustFS stores no per-bucket owner: bucket ACL writes are no-ops and nothing writes the ACL
+//! document, so the only owner RustFS reports anywhere (ListBuckets, ACL and multipart listings)
+//! is one constant. The source answers exactly that constant. A caller therefore passes the
+//! gateway's check only by presenting the owner id RustFS lists, which separates no tenants; it is
+//! a compatibility answer, and the constant's own documentation says it must not decide
+//! authorization.
+//!
+//! # Difference from legacy (ledger entry R-071)
+//!
+//! Legacy RustFS ignores `x-amz-expected-bucket-owner` entirely. On the gateway stack a request
+//! carrying it is served only when the value equals the reported owner id; otherwise it is `403
+//! AccessDenied`. A missing bucket and a backend failure with the header present are also `403`
+//! at the pinned gateway, where legacy answers `404` and `5xx`: [`BucketOwnerError`] has one value,
+//! so it cannot express either. [`RustfsBucketOwners::resolve`] keeps the two apart at this
+//! adapter's boundary so the gateway can surface them once it grows the classes.
+
+use std::sync::Arc;
+
+use rustfs_gateway::{BoxFuture, BucketName, BucketOwnerError, BucketOwnerSource};
+
+/// Why a bucket yielded no owner.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum OwnerLookupError {
+    /// The bucket does not exist.
+    NoSuchBucket,
+    /// The metadata backend failed, could not be reached, or the owner is not usable (an empty
+    /// owner id is a misconfiguration), so the bucket's state is unknown.
+    Backend,
+}
+
+/// The narrow slice of RustFS bucket metadata an owner answer needs: whether a bucket exists.
+///
+/// Exists so the source can be tested against a fake. The binary implements it over the ecstore
+/// bucket metadata system, mapping `BucketNotFound` to [`OwnerLookupError::NoSuchBucket`] and every
+/// other error to [`OwnerLookupError::Backend`].
+pub trait BucketLookup: Send + Sync + 'static {
+    /// `Ok(())` when `bucket` exists.
+    fn check<'a>(&'a self, bucket: &'a str) -> BoxFuture<'a, Result<(), OwnerLookupError>>;
+}
+
+/// A [`BucketOwnerSource`] answering one owner id for every bucket that exists.
+pub struct RustfsBucketOwners<L> {
+    lookup: L,
+    owner: Arc<str>,
+}
+
+impl<L: BucketLookup> RustfsBucketOwners<L> {
+    /// A source answering `owner` for every bucket `lookup` finds.
+    ///
+    /// An empty `owner` is no owner: every answer is then [`OwnerLookupError::Backend`].
+    #[must_use]
+    pub fn new(lookup: L, owner: impl Into<Arc<str>>) -> Self {
+        Self {
+            lookup,
+            owner: owner.into(),
+        }
+    }
+
+    /// The owner of `bucket`, or why there is none. Missing and failing stay distinct here.
+    ///
+    /// # Errors
+    ///
+    /// [`OwnerLookupError::NoSuchBucket`] when the bucket does not exist;
+    /// [`OwnerLookupError::Backend`] on any lookup failure or when the configured owner is empty.
+    pub async fn resolve(&self, bucket: &str) -> Result<Arc<str>, OwnerLookupError> {
+        self.lookup.check(bucket).await?;
+        if self.owner.is_empty() {
+            return Err(OwnerLookupError::Backend);
+        }
+        Ok(Arc::clone(&self.owner))
+    }
+}
+
+impl<L: BucketLookup> BucketOwnerSource for RustfsBucketOwners<L> {
+    fn owner<'a>(&'a self, bucket: &'a BucketName) -> BoxFuture<'a, Result<Arc<str>, BucketOwnerError>> {
+        Box::pin(async move {
+            // The pinned gateway collapses both classes (R-071), so both are the one refusal.
+            self.resolve(bucket.as_str())
+                .await
+                .map_err(|_: OwnerLookupError| BucketOwnerError::unavailable())
+        })
+    }
+}
+
+#[cfg(test)]
+#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
+mod tests {
+    use super::*;
+    use std::sync::Mutex;
+
+    /// Stands for the owner id RustFS reports; the binary passes its own constant.
+    const OWNER: &str = "reported-owner-id";
+
+    struct Fake {
+        answer: Result<(), OwnerLookupError>,
+        asked: Mutex<Vec<String>>,
+    }
+
+    impl Fake {
+        fn answering(answer: Result<(), OwnerLookupError>) -> Self {
+            Self {
+                answer,
+                asked: Mutex::new(Vec::new()),
+            }
+        }
+    }
+
+    impl BucketLookup for Fake {
+        fn check<'a>(&'a self, bucket: &'a str) -> BoxFuture<'a, Result<(), OwnerLookupError>> {
+            self.asked.lock().unwrap().push(bucket.to_owned());
+            let answer = self.answer;
+            Box::pin(async move { answer })
+        }
+    }
+
+    fn source(answer: Result<(), OwnerLookupError>) -> RustfsBucketOwners<Fake> {
+        RustfsBucketOwners::new(Fake::answering(answer), OWNER)
+    }
+
+    async fn gateway_answer(answer: Result<(), OwnerLookupError>) -> Result<Arc<str>, BucketOwnerError> {
+        source(answer).owner(&BucketName::new("photos").expect("a bucket name")).await
+    }
+
+    /// Positive: an existing bucket's owner is the configured id, byte for byte.
+    #[tokio::test]
+    async fn an_existing_bucket_has_the_configured_owner() {
+        assert_eq!(gateway_answer(Ok(())).await.as_deref(), Ok(OWNER));
+    }
+
+    /// Positive: every bucket gets the same owner, since RustFS stores none per bucket.
+    #[tokio::test]
+    async fn every_bucket_has_the_same_owner() {
+        let source = source(Ok(()));
+        assert_eq!(source.resolve("photos").await.as_deref(), Ok(OWNER));
+        assert_eq!(source.resolve("backups").await.as_deref(), Ok(OWNER));
+    }
+
+    /// Positive: the lookup is asked about exactly the bucket the gateway named.
+    #[tokio::test]
+    async fn the_lookup_is_asked_about_the_named_bucket() {
+        let source = source(Ok(()));
+        source
+            .owner(&BucketName::new("photos").expect("a bucket name"))
+            .await
+            .expect("an owner");
+        assert_eq!(*source.lookup.asked.lock().unwrap(), vec!["photos".to_owned()]);
+    }
+
+    /// Negative: a missing bucket is reported as missing at the adapter boundary.
+    #[tokio::test]
+    async fn a_missing_bucket_is_reported_as_missing() {
+        assert_eq!(
+            source(Err(OwnerLookupError::NoSuchBucket)).resolve("photos").await,
+            Err(OwnerLookupError::NoSuchBucket)
+        );
+    }
+
+    /// Negative: a failing backend is reported as a failure, never as an owner and never as missing.
+    #[tokio::test]
+    async fn a_backend_failure_is_reported_as_a_failure() {
+        assert_eq!(
+            source(Err(OwnerLookupError::Backend)).resolve("photos").await,
+            Err(OwnerLookupError::Backend)
+        );
+    }
+
+    /// Negative: the pinned gateway sees one refusal for both, and never an owner.
+    #[tokio::test]
+    async fn the_gateway_is_refused_in_both_cases() {
+        assert_eq!(
+            gateway_answer(Err(OwnerLookupError::NoSuchBucket)).await,
+            Err(BucketOwnerError::unavailable())
+        );
+        assert_eq!(gateway_answer(Err(OwnerLookupError::Backend)).await, Err(BucketOwnerError::unavailable()));
+    }
+
+    /// Negative: an empty configured owner is no owner; it must not equal an empty assertion.
+    #[tokio::test]
+    async fn an_empty_owner_is_refused() {
+        let source = RustfsBucketOwners::new(Fake::answering(Ok(())), "");
+        assert_eq!(source.resolve("photos").await, Err(OwnerLookupError::Backend));
+        assert_eq!(
+            source.owner(&BucketName::new("photos").expect("a bucket name")).await,
+            Err(BucketOwnerError::unavailable())
+        );
+    }
+}

crates/gateway-adapter/src/credentials.rs

@@ -0,0 +1,446 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! The gateway's [`CredentialProvider`], answered by RustFS IAM.
+//!
+//! Responsible for: resolving one access key id to the secret the gateway verifies a signature
+//! with, and classifying every other outcome as either a business result
+//! ([`CredentialLookup::NotFound`], eligible for the gateway's negative cache) or an operational
+//! fault ([`ProviderError`], counted and never cached).
+//! NOT responsible for: authorization of any kind, session-token claim validation, or deciding
+//! what the client is answered with. The gateway answers every unusable credential identically.
+//!
+//! # Secret handling
+//!
+//! Secrets travel only inside the gateway's [`Credentials`], whose `Debug` redacts them. No type
+//! in this module derives or implements `Debug`, and no error produced here carries text: the
+//! IAM error is classified and dropped. A secret that fails the crate's effective-length rule
+//! (`crate::MIN_SECRET_KEY_LEN` bytes once trailing NULs are removed) is never handed to the
+//! gateway, whatever the lookup says about its validity.
+
+use std::sync::Arc;
+
+use rustfs_gateway::{BoxFuture, CredentialLookup, CredentialProvider, Credentials, ProviderError, SessionBinding};
+use rustfs_iam::error::Error as IamError;
+use rustfs_iam::store::Store;
+use rustfs_iam::sys::IamSys;
+use rustfs_policy::auth::{ACCOUNT_OFF, UserIdentity};
+
+use crate::root::{RootSource, is_usable_secret};
+
+/// The opaque issuer recorded in the [`rustfs_gateway::SessionBinding`] of every RustFS STS
+/// credential.
+pub const STS_SESSION_ISSUER: &str = "rustfs-sts";
+
+/// Access keys with this prefix belong to Keystone token authentication, which never verifies an
+/// AWS signature.
+const KEYSTONE_ACCESS_KEY_PREFIX: &str = "keystone:";
+
+/// The narrow slice of IAM that a credential lookup needs.
+///
+/// Exists so the adapter can be tested against a fake: IAM exports no in-memory backend.
+/// Implemented for [`IamSys`], whose `check_key` loads a user from storage on a cache miss.
+pub trait IamKeyLookup: Send + Sync + 'static {
+    /// Whether the IAM store has finished loading.
+    fn is_ready(&self) -> bool;
+
+    /// Resolves an access key, with [`IamSys::check_key`]'s contract: `Ok((None, _))` for an
+    /// unknown key, `Ok((Some(identity), valid))` otherwise, where `valid` is false for a
+    /// disabled, expired, or malformed credential.
+    fn check_key<'a>(&'a self, access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>>;
+}
+
+impl<T: Store> IamKeyLookup for IamSys<T> {
+    fn is_ready(&self) -> bool {
+        IamSys::is_ready(self)
+    }
+
+    fn check_key<'a>(&'a self, access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
+        Box::pin(IamSys::check_key(self, access_key))
+    }
+}
+
+/// A [`CredentialProvider`] over RustFS IAM and the live root credentials.
+///
+/// Built only by [`crate::IamAdapters::new`], so it always reads the same [`RootSource`] as the
+/// policy source it was assembled with.
+pub struct IamCredentials {
+    iam: Arc<dyn IamKeyLookup>,
+    root: Arc<dyn RootSource>,
+}
+
+impl IamCredentials {
+    pub(crate) fn new(iam: Arc<dyn IamKeyLookup>, root: Arc<dyn RootSource>) -> Self {
+        Self { iam, root }
+    }
+
+    async fn resolve(&self, access_key: &str) -> Result<CredentialLookup, ProviderError> {
+        // Deliberate change from the legacy s3s path (`rustfs/src/auth.rs` `get_secret_key`),
+        // which answered a `keystone:` key with an empty secret. Keystone authenticates by token,
+        // so there is no AWS secret to verify a signature against: the key is unknown here.
+        if access_key.starts_with(KEYSTONE_ACCESS_KEY_PREFIX) {
+            return Ok(CredentialLookup::NotFound);
+        }
+
+        // Legacy answers every key with an internal error while the root credentials are not
+        // initialized; without them no key can be classified as root or not.
+        let Some(root) = self.root.current() else {
+            return Err(ProviderError::Unavailable);
+        };
+        // Byte-exact, as the trait requires: the access key id is signed material.
+        if root.access_key() == access_key {
+            return Ok(CredentialLookup::Found(root.credentials().clone_credentials()));
+        }
+
+        if !self.iam.is_ready() {
+            return Err(ProviderError::Unavailable);
+        }
+
+        match self.iam.check_key(access_key).await {
+            // IAM refuses such a secret through `valid`, but this provider does not rely on every
+            // lookup doing so: a short secret is never usable, so it is not a credential at all.
+            Ok((Some(identity), _)) if !is_usable_secret(identity.credentials.secret_key.as_bytes()) => {
+                Ok(CredentialLookup::NotFound)
+            }
+            Ok((Some(identity), valid)) => to_gateway_credentials(access_key, &identity, valid).map(CredentialLookup::Found),
+            Ok((None, _)) => Ok(CredentialLookup::NotFound),
+            Err(error) => classify_iam_error(&error),
+        }
+    }
+}
+
+/// Classifies an IAM failure without keeping any of its text.
+fn classify_iam_error(error: &IamError) -> Result<CredentialLookup, ProviderError> {
+    match error {
+        IamError::NoSuchUser(_)
+        | IamError::NoSuchServiceAccount(_)
+        | IamError::NoSuchTempAccount(_)
+        | IamError::NoSuchAccount(_) => Ok(CredentialLookup::NotFound),
+        IamError::IamSysNotInitialized => Err(ProviderError::Unavailable),
+        _ => Err(ProviderError::Backend),
+    }
+}
+
+/// Builds the gateway credential for one IAM identity. No authorization happens here: a
+/// disabled or invalid principal still resolves, marked disabled, and the gateway refuses it
+/// exactly as it refuses an unknown key.
+fn to_gateway_credentials(access_key: &str, identity: &UserIdentity, valid: bool) -> Result<Credentials, ProviderError> {
+    let cred = &identity.credentials;
+    let mut credentials = Credentials::new(access_key, cred.secret_key.as_bytes()).map_err(|_| ProviderError::Backend)?;
+
+    // A service account carries a claims token internally, but its clients never present it, so
+    // it authenticates as a long-term credential (as `check_session_token` treats it in legacy).
+    if !cred.session_token.is_empty() && !cred.is_service_account() {
+        // A temporary credential without an expiry cannot become a gateway session, which always
+        // ends. Refusing it as a fault is the fail-closed answer; a never-ending session is not.
+        let expiration = cred.expiration.ok_or(ProviderError::Backend)?;
+        let binding = SessionBinding::new(STS_SESSION_ISSUER, expiration.unix_timestamp()).map_err(|_| ProviderError::Backend)?;
+        credentials = credentials
+            .with_session(&cred.session_token, binding)
+            .map_err(|_| ProviderError::Backend)?;
+    }
+
+    if !valid || cred.status == ACCOUNT_OFF {
+        credentials = credentials.disable();
+    }
+    Ok(credentials)
+}
+
+impl CredentialProvider for IamCredentials {
+    fn lookup<'a>(&'a self, access_key_id: &'a str) -> BoxFuture<'a, Result<CredentialLookup, ProviderError>> {
+        Box::pin(self.resolve(access_key_id))
+    }
+}
+
+#[cfg(test)]
+#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
+mod tests {
+    use super::*;
+    use crate::root::{RootCell, RootCredentials};
+    use rustfs_credentials::Credentials as IamCredential;
+    use rustfs_gateway::RequestNow;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use time::OffsetDateTime;
+
+    const ACCESS_KEY: &str = "AKIDUSER0001";
+    const SECRET: &str = "user-secret-key-0001";
+
+    type Answer = Box<dyn Fn() -> Result<(Option<UserIdentity>, bool), IamError> + Send + Sync>;
+
+    struct FakeIam {
+        ready: bool,
+        answer: Answer,
+        calls: AtomicUsize,
+    }
+
+    impl IamKeyLookup for FakeIam {
+        fn is_ready(&self) -> bool {
+            self.ready
+        }
+
+        fn check_key<'a>(&'a self, _access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
+            self.calls.fetch_add(1, Ordering::SeqCst);
+            let answer = (self.answer)();
+            Box::pin(async move { answer })
+        }
+    }
+
+    fn fake(
+        ready: bool,
+        answer: impl Fn() -> Result<(Option<UserIdentity>, bool), IamError> + Send + Sync + 'static,
+    ) -> Arc<FakeIam> {
+        Arc::new(FakeIam {
+            ready,
+            answer: Box::new(answer),
+            calls: AtomicUsize::new(0),
+        })
+    }
+
+    fn user(status: &str) -> IamCredential {
+        IamCredential {
+            access_key: ACCESS_KEY.to_owned(),
+            secret_key: SECRET.to_owned(),
+            status: status.to_owned(),
+            ..Default::default()
+        }
+    }
+
+    fn found(cred: IamCredential, valid: bool) -> Result<(Option<UserIdentity>, bool), IamError> {
+        Ok((Some(UserIdentity::new(cred)), valid))
+    }
+
+    fn root_cell() -> Arc<RootCell> {
+        let root = RootCredentials::new("rootadmin", "root-secret-key", "sts-signing-key").expect("valid root credentials");
+        Arc::new(RootCell::new(Some(root)))
+    }
+
+    async fn lookup(iam: Arc<FakeIam>, access_key: &str) -> Result<CredentialLookup, ProviderError> {
+        IamCredentials::new(iam, root_cell()).lookup(access_key).await
+    }
+
+    fn expect_found(result: Result<CredentialLookup, ProviderError>) -> Credentials {
+        match result {
+            Ok(CredentialLookup::Found(credentials)) => credentials,
+            Ok(CredentialLookup::NotFound) => panic!("expected Found, got NotFound"),
+            Err(error) => panic!("expected Found, got {error:?}"),
+        }
+    }
+
+    fn is_not_found(result: &Result<CredentialLookup, ProviderError>) -> bool {
+        matches!(result, Ok(CredentialLookup::NotFound))
+    }
+
+    /// Positive — an enabled user resolves to exactly its secret, as a long-term credential.
+    #[tokio::test]
+    async fn credentials_found_user_resolves_to_its_secret() {
+        let iam = fake(true, || found(user("on"), true));
+        let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
+        assert_eq!(credentials.secret().expose(), SECRET.as_bytes());
+        assert_eq!(credentials.identity().access_key_id(), ACCESS_KEY);
+        assert!(!credentials.is_disabled());
+        assert!(credentials.session_binding().is_none());
+    }
+
+    /// Negative — a key IAM does not know is a business result, not a fault.
+    #[tokio::test]
+    async fn credentials_unknown_key_is_not_found() {
+        let iam = fake(true, || Ok((None, false)));
+        assert!(is_not_found(&lookup(iam, "AKIDUNKNOWN").await));
+    }
+
+    /// Negative — every "no such principal" error IAM can surface is `NotFound`. Answering these
+    /// as `Backend` would keep them out of the negative cache and count every guessed key as a
+    /// backend fault.
+    #[tokio::test]
+    async fn credentials_no_such_principal_errors_are_not_found() {
+        let errors: [fn() -> IamError; 4] = [
+            || IamError::NoSuchUser(ACCESS_KEY.to_owned()),
+            || IamError::NoSuchServiceAccount(ACCESS_KEY.to_owned()),
+            || IamError::NoSuchTempAccount(ACCESS_KEY.to_owned()),
+            || IamError::NoSuchAccount(ACCESS_KEY.to_owned()),
+        ];
+        for error in errors {
+            let iam = fake(true, move || Err(error()));
+            let result = lookup(iam, ACCESS_KEY).await;
+            assert!(is_not_found(&result), "{:?} must be NotFound", error());
+        }
+    }
+
+    /// Negative — any other IAM failure is a backend fault, never `NotFound` and never `Found`.
+    #[tokio::test]
+    async fn credentials_backend_error_is_backend() {
+        let iam = fake(true, || Err(IamError::StringError("storage read failed".to_owned())));
+        assert!(matches!(lookup(iam, ACCESS_KEY).await, Err(ProviderError::Backend)));
+    }
+
+    /// Negative — a `keystone:` key never reaches IAM and never resolves to a secret.
+    ///
+    /// Deliberate change from the legacy s3s path, which answered such a key with an empty
+    /// secret: Keystone authenticates by token, so there is no AWS secret to verify against.
+    #[tokio::test]
+    async fn credentials_keystone_prefix_is_not_found_without_iam() {
+        let iam = fake(true, || found(user("on"), true));
+        let result = lookup(iam.clone(), "keystone:project-user").await;
+        assert!(is_not_found(&result));
+        assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
+    }
+
+    /// Negative — a switched-off or otherwise invalid user resolves, but disabled, and the same
+    /// user enabled does not.
+    #[tokio::test]
+    async fn credentials_disabled_user_is_disabled() {
+        let off = expect_found(lookup(fake(true, || found(user("off"), false)), ACCESS_KEY).await);
+        assert!(off.is_disabled());
+        assert!(off.admit(None, RequestNow::from_unix_seconds(0)).is_err());
+
+        let invalid = expect_found(lookup(fake(true, || found(user("on"), false)), ACCESS_KEY).await);
+        assert!(invalid.is_disabled());
+
+        let on = expect_found(lookup(fake(true, || found(user("on"), true)), ACCESS_KEY).await);
+        assert!(!on.is_disabled());
+    }
+
+    /// Positive — an STS credential carries its token and its lifetime as one session binding.
+    #[tokio::test]
+    async fn credentials_sts_carries_session_binding() {
+        let expires_at = 4_000_000_000_i64;
+        let iam = fake(true, move || {
+            let mut cred = user("on");
+            cred.session_token = "sts-session-token".to_owned();
+            cred.parent_user = "parent".to_owned();
+            cred.expiration = Some(OffsetDateTime::from_unix_timestamp(expires_at).expect("valid timestamp"));
+            found(cred, true)
+        });
+        let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
+        let binding = credentials.session_binding().expect("an STS credential has a session");
+        assert_eq!(binding.expires_at_unix_seconds(), expires_at);
+        assert_eq!(binding.issuer(), STS_SESSION_ISSUER);
+        let now = RequestNow::from_unix_seconds(expires_at - 1);
+        assert!(credentials.admit(Some(b"sts-session-token"), now).is_ok());
+        assert!(credentials.admit(None, now).is_err());
+    }
+
+    /// Negative — an STS credential with no expiry cannot be expressed as a session the gateway
+    /// would ever expire, so it is a backend fault rather than a session that never ends.
+    #[tokio::test]
+    async fn credentials_sts_without_expiry_is_backend() {
+        let iam = fake(true, || {
+            let mut cred = user("on");
+            cred.session_token = "sts-session-token".to_owned();
+            found(cred, true)
+        });
+        assert!(matches!(lookup(iam, ACCESS_KEY).await, Err(ProviderError::Backend)));
+    }
+
+    /// Positive — a service account carries a token internally but authenticates as a long-term
+    /// credential: clients never present that token.
+    #[tokio::test]
+    async fn credentials_service_account_is_long_term() {
+        let iam = fake(true, || {
+            let mut cred = user("on");
+            cred.session_token = "service-account-claims-token".to_owned();
+            cred.parent_user = "parent".to_owned();
+            cred.claims = Some([(rustfs_credentials::IAM_POLICY_CLAIM_NAME_SA.to_owned(), "inherited-policy".into())].into());
+            found(cred, true)
+        });
+        let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
+        assert!(credentials.session_binding().is_none());
+        assert!(credentials.admit(None, RequestNow::from_unix_seconds(0)).is_ok());
+    }
+
+    /// Negative — before IAM is ready a lookup is `Unavailable`, not `NotFound`: a key must not
+    /// be negatively cached because the store had not loaded yet.
+    #[tokio::test]
+    async fn credentials_iam_not_ready_is_unavailable() {
+        let iam = fake(false, || found(user("on"), true));
+        assert!(matches!(lookup(iam.clone(), ACCESS_KEY).await, Err(ProviderError::Unavailable)));
+        assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
+
+        let uninitialized = fake(true, || Err(IamError::IamSysNotInitialized));
+        assert!(matches!(lookup(uninitialized, ACCESS_KEY).await, Err(ProviderError::Unavailable)));
+    }
+
+    /// Positive — the injected root credentials resolve without IAM, even before it is ready, and
+    /// nothing else is answered with the root secret.
+    #[tokio::test]
+    async fn credentials_root_resolves_without_iam() {
+        let iam = fake(false, || found(user("on"), true));
+        let provider = IamCredentials::new(iam.clone(), root_cell());
+        let root = expect_found(provider.lookup("rootadmin").await);
+        assert_eq!(root.secret().expose(), b"root-secret-key");
+        assert!(matches!(provider.lookup("ROOTADMIN").await, Err(ProviderError::Unavailable)));
+        assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
+    }
+
+    /// Negative — a lookup that calls an identity valid but holds an empty or short secret is not
+    /// trusted: such a secret is not a credential, so the key is `NotFound`, never `Found`.
+    #[tokio::test]
+    async fn credentials_short_secret_is_never_found() {
+        for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0"] {
+            let iam = fake(true, move || {
+                let mut cred = user("on");
+                cred.secret_key = weak.to_owned();
+                found(cred, true)
+            });
+            let result = lookup(iam, ACCESS_KEY).await;
+            assert!(is_not_found(&result), "{weak:?} must be NotFound");
+        }
+        for usable in ["12345678", "abc\0defg"] {
+            let iam = fake(true, move || {
+                let mut cred = user("on");
+                cred.secret_key = usable.to_owned();
+                found(cred, true)
+            });
+            let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
+            assert_eq!(credentials.secret().expose(), usable.as_bytes());
+        }
+    }
+
+    /// Negative — a switched-off account is disabled even when the lookup calls it valid.
+    #[tokio::test]
+    async fn credentials_off_status_alone_disables() {
+        let off = expect_found(lookup(fake(true, || found(user("off"), true)), ACCESS_KEY).await);
+        assert!(off.is_disabled());
+        assert!(off.admit(None, RequestNow::from_unix_seconds(0)).is_err());
+    }
+
+    /// Negative — an STS credential admits only its own session's token: the token of another
+    /// session, or none, is refused.
+    #[tokio::test]
+    async fn credentials_sts_refuses_another_sessions_token() {
+        let expires_at = 4_000_000_000_i64;
+        let iam = fake(true, move || {
+            let mut cred = user("on");
+            cred.session_token = "session-token-a".to_owned();
+            cred.parent_user = "parent".to_owned();
+            cred.expiration = Some(OffsetDateTime::from_unix_timestamp(expires_at).expect("valid timestamp"));
+            found(cred, true)
+        });
+        let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
+        let now = RequestNow::from_unix_seconds(expires_at - 1);
+        assert!(credentials.admit(Some(b"session-token-b"), now).is_err());
+        assert!(credentials.admit(Some(b"session-token-a"), now).is_ok());
+    }
+
+    /// Negative — without root credentials no key can be classified, so every lookup is
+    /// `Unavailable` and IAM is not asked.
+    #[tokio::test]
+    async fn credentials_without_root_are_unavailable() {
+        let iam = fake(true, || found(user("on"), true));
+        let provider = IamCredentials::new(iam.clone(), Arc::new(RootCell::new(None)));
+        assert!(matches!(provider.lookup(ACCESS_KEY).await, Err(ProviderError::Unavailable)));
+        assert!(matches!(provider.lookup("rootadmin").await, Err(ProviderError::Unavailable)));
+        assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
+    }
+}

crates/gateway-adapter/src/host.rs

@@ -0,0 +1,306 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! The gateway's [`HostResolver`], built from RustFS's configured domain list.
+//!
+//! Responsible for: turning the `RUSTFS_SERVER_DOMAINS` list into the gateway's legacy RustFS
+//! virtual-host resolver.
+//! NOT responsible for: parsing a `Host` header, matching a domain, validating bucket names (all
+//! done by [`LegacyRustfsVirtualHosts`]), reading the environment, or deciding that a console
+//! deployment serves no domains (the caller passes an empty list then, as `s3_host_domains` in
+//! `rustfs/src/server/http.rs` returns `None`). An empty list needs no branch here:
+//! [`LegacyRustfsVirtualHosts`] built from no domain reads every host path-style, exactly as the
+//! gateway's `PathStyleOnly` does.
+//! Upstream: the RustFS configuration (`Config::server_domains`). Downstream: the gateway's
+//! `ServiceBuilder`, which holds the resolver as `Arc<dyn HostResolver>`.
+//!
+//! # What "the same as `MultiDomain`" means
+//!
+//! RustFS today hands the same list to the s3s framework's multi-domain host, which ignores ports
+//! on both sides when matching a domain, takes the whole prefix in front of a matched domain as
+//! the bucket, compares a domain exactly (so a host is matched on the case it arrives in), and,
+//! for a host no domain matches, falls back to CNAME-style: a host that is itself a bucket name
+//! (lower-cased) is that bucket. [`LegacyRustfsVirtualHosts`] reproduces exactly that.
+//!
+//! What reads **path-style**: an empty domain list; a configured domain itself, with or without a
+//! port; an IP address, or a socket address (`10.0.0.1`, `[::1]:9000`); a host that is a valid
+//! domain but, lower-cased, is no bucket name (it carries a port, is over 63 bytes, or starts
+//! with `xn--`). What reads as a **bucket**: anything under a configured domain, and any other
+//! bucket-shaped host, which is a bucket of its own (`localhost`, `badexample.com`, `EXAMPLE.COM`).
+//! What is **refused** before routing: a host that is no domain (`under_score.org`, a bare
+//! `[::1]`, a trailing dot), and a bucket under a domain that the bucket rules refuse
+//! (`PHOTOS.example.com`, `ab.example.com`). The tests below pin each of those readings.
+
+use std::sync::Arc;
+
+use rustfs_gateway::{HostResolver, LegacyDomainError, LegacyRustfsVirtualHosts};
+
+/// Builds the host resolver for RustFS's configured domain list.
+///
+/// An empty list reads every host path-style. Otherwise the list is read as legacy RustFS reads
+/// it: a later entry whose name without its port repeats an earlier one is dropped, and a list
+/// that legacy RustFS refuses at start-up is refused here.
+///
+/// # Errors
+///
+/// [`LegacyDomainError`] for an entry that is not a domain, or for two entries where one equals or
+/// is a subdomain of the other.
+pub fn rustfs_host_resolver<I, S>(domains: I) -> Result<Arc<dyn HostResolver>, LegacyDomainError>
+where
+    I: IntoIterator<Item = S>,
+    S: AsRef<str>,
+{
+    Ok(Arc::new(LegacyRustfsVirtualHosts::new(domains)?))
+}
+
+#[cfg(test)]
+#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
+mod tests {
+    use super::*;
+    use http::Method;
+    use rustfs_gateway::{Addressing, HostQuery, HostRefusal, TargetKind};
+
+    /// What the resolver reads `GET path` with `Host: host` as: the host-named bucket (`None` for
+    /// path style), the routed target, and the refusal, if any.
+    fn read(resolver: &dyn HostResolver, path: &str, host: &str) -> (Option<String>, TargetKind, Option<HostRefusal>) {
+        let method = Method::GET;
+        let request = http::Request::builder()
+            .method(method.clone())
+            .uri(path)
+            .header("host", host)
+            .body(())
+            .expect("a valid request");
+        let effective = rustfs_gateway_http::effective_host(&request).expect("an acceptable host");
+        let query = HostQuery {
+            host: &effective,
+            path,
+            method: &method,
+        };
+        let resolved = resolver.resolve(&query);
+        let bucket = match &resolved.addressing {
+            Addressing::VirtualHosted { bucket, .. } => Some(bucket.as_str().to_owned()),
+            Addressing::Path => None,
+        };
+        (bucket, resolved.target, resolver.refusal(&query))
+    }
+
+    fn with_domains(domains: &[&str]) -> Arc<dyn HostResolver> {
+        rustfs_host_resolver(domains).expect("a list RustFS accepts")
+    }
+
+    fn bucket(domains: &[&str], host: &str) -> Option<String> {
+        read(&*with_domains(domains), "/key", host).0
+    }
+
+    /// Positive: a host under a configured domain names the bucket in front of it.
+    #[test]
+    fn host_under_domain_names_the_bucket() {
+        assert_eq!(
+            read(&*with_domains(&["example.com"]), "/key", "photos.example.com"),
+            (Some("photos".to_owned()), TargetKind::Object, None)
+        );
+        assert_eq!(read(&*with_domains(&["example.com"]), "/", "photos.example.com").1, TargetKind::Bucket);
+    }
+
+    /// Positive: the port on the host, or on the configured domain, does not take part in matching.
+    #[test]
+    fn ports_are_ignored_on_both_sides() {
+        assert_eq!(bucket(&["example.com"], "photos.example.com:9000").as_deref(), Some("photos"));
+        assert_eq!(bucket(&["example.com:9000"], "photos.example.com").as_deref(), Some("photos"));
+        assert_eq!(bucket(&["example.com:9000"], "photos.example.com:1234").as_deref(), Some("photos"));
+    }
+
+    /// Positive: the whole prefix in front of the domain is the bucket, dots kept.
+    #[test]
+    fn whole_prefix_is_the_bucket() {
+        assert_eq!(bucket(&["example.com"], "my.dotted.bkt.example.com").as_deref(), Some("my.dotted.bkt"));
+    }
+
+    /// Positive: with several domains, each one resolves its own subdomains.
+    #[test]
+    fn every_configured_domain_resolves() {
+        let domains = ["example.com", "s3.local"];
+        assert_eq!(bucket(&domains, "abc.example.com").as_deref(), Some("abc"));
+        assert_eq!(bucket(&domains, "xyz.s3.local").as_deref(), Some("xyz"));
+    }
+
+    /// Positive: the domain itself is the endpoint, so the bucket comes from the path.
+    #[test]
+    fn the_domain_itself_is_path_style() {
+        assert_eq!(bucket(&["example.com"], "example.com"), None);
+        assert_eq!(bucket(&["example.com"], "example.com:9000"), None);
+    }
+
+    /// Negative: no configured domain means path style only, for every host shape.
+    #[test]
+    fn no_domain_is_path_style_only() {
+        for host in ["photos.example.com", "localhost", "localhost:9000", "10.0.0.1"] {
+            assert_eq!(bucket(&[], host), None, "{host}");
+        }
+    }
+
+    /// Negative: an address, and a host carrying a port that no domain matches, are path style.
+    #[test]
+    fn addresses_and_unmatched_ports_are_path_style() {
+        for host in ["10.0.0.1", "10.0.0.1:9000", "localhost:9000", "other.org:9000"] {
+            assert_eq!(bucket(&["example.com"], host), None, "{host}");
+        }
+    }
+
+    /// Negative: a host that only ends in the same letters as a domain is not under it. It is not
+    /// stripped to `bad`; the only reading it gets is the CNAME-style one, the whole host.
+    #[test]
+    fn suffix_lookalike_is_not_under_the_domain() {
+        assert_eq!(bucket(&["example.com"], "badexample.com").as_deref(), Some("badexample.com"));
+        assert_eq!(bucket(&["example.com"], "example.com.evil.org").as_deref(), Some("example.com.evil.org"));
+    }
+
+    /// Negative: matching a configured domain is exact on case, as the legacy multi-domain host does it, so a
+    /// mixed-case host is not stripped to its prefix; it is the lower-cased whole host.
+    #[test]
+    fn mixed_case_host_is_not_stripped_to_a_prefix() {
+        assert_eq!(bucket(&["example.com"], "Photos.Example.COM").as_deref(), Some("photos.example.com"));
+    }
+
+    /// Negative: a host that is no domain at all is refused, not routed.
+    #[test]
+    fn a_host_that_is_no_domain_is_refused() {
+        let resolver = with_domains(&["example.com"]);
+        assert_eq!(read(&*resolver, "/k", "under_score.org").2, Some(HostRefusal::UnusableHost));
+    }
+
+    /// Negative: a bucket the host names that bucket rules refuse is refused.
+    #[test]
+    fn a_host_named_bucket_the_rules_refuse_is_refused() {
+        let resolver = with_domains(&["example.com"]);
+        assert_eq!(read(&*resolver, "/k", "ab.example.com").2, Some(HostRefusal::RefusedBucket));
+        assert_eq!(read(&*resolver, "/k", "bad_host.example.com").2, Some(HostRefusal::RefusedBucket));
+    }
+
+    /// Negative: a list legacy RustFS refuses at start-up is refused.
+    #[test]
+    fn domain_lists_legacy_rustfs_refuses_are_refused() {
+        for list in [
+            vec!["example.com", "a.example.com"],
+            vec!["example.com", "com"],
+            vec!["bad_domain"],
+            vec![""],
+            vec!["example.com:notaport"],
+        ] {
+            assert!(rustfs_host_resolver(&list).is_err(), "{list:?}");
+        }
+    }
+
+    /// Positive: a later repeat of a name, whatever its port, is dropped rather than refused, as
+    /// `s3_host_domains` drops it before the legacy multi-domain host sees the list.
+    #[test]
+    fn a_repeat_of_a_domain_is_dropped() {
+        assert_eq!(
+            bucket(&["example.com:9411", "example.com:9000"], "xyz.example.com").as_deref(),
+            Some("xyz")
+        );
+    }
+
+    /// The refusal alone, for hosts whose bucket reading is not the point.
+    fn refusal(domains: &[&str], host: &str) -> Option<HostRefusal> {
+        read(&*with_domains(domains), "/k", host).2
+    }
+
+    /// Positive: a bucket-shaped host no domain matches is a bucket of its own, `localhost` and
+    /// an upper-case spelling of the domain among them.
+    #[test]
+    fn bucket_shaped_hosts_outside_every_domain_are_their_own_bucket() {
+        assert_eq!(bucket(&["example.com"], "localhost").as_deref(), Some("localhost"));
+        assert_eq!(bucket(&["example.com"], "EXAMPLE.COM").as_deref(), Some("example.com"));
+        assert_eq!(bucket(&["example.com"], "cdn.other.org").as_deref(), Some("cdn.other.org"));
+    }
+
+    /// Negative: a bucket the rules refuse under a domain is refused, and reads no bucket.
+    #[test]
+    fn an_upper_case_bucket_under_the_domain_is_refused() {
+        assert_eq!(bucket(&["example.com"], "PHOTOS.example.com"), None);
+        assert_eq!(refusal(&["example.com"], "PHOTOS.example.com"), Some(HostRefusal::RefusedBucket));
+    }
+
+    /// Negative: an IPv6 literal. The bare form is no domain and is refused; with a port it is a
+    /// socket address and reads path-style.
+    #[test]
+    fn ipv6_literals_follow_the_address_rules() {
+        assert_eq!(refusal(&["example.com"], "[::1]"), Some(HostRefusal::UnusableHost));
+        assert_eq!(bucket(&["example.com"], "[::1]:9000"), None);
+        assert_eq!(refusal(&["example.com"], "[::1]:9000"), None);
+    }
+
+    /// Negative: a trailing root dot makes the host no domain, so it is refused, not stripped.
+    #[test]
+    fn a_trailing_dot_is_refused() {
+        assert_eq!(refusal(&["example.com"], "photos.example.com."), Some(HostRefusal::UnusableHost));
+        assert_eq!(bucket(&["example.com"], "photos.example.com."), None);
+    }
+
+    /// Negative: an empty label never reaches the resolver; the gateway's wire layer refuses the
+    /// host first (legacy answers 400 as well).
+    #[test]
+    fn an_empty_label_is_refused_before_resolution() {
+        for host in ["a..b.example.com", ".example.com"] {
+            let request = http::Request::builder()
+                .uri("/k")
+                .header("host", host)
+                .body(())
+                .expect("a valid request");
+            assert!(rustfs_gateway_http::effective_host(&request).is_err(), "{host}");
+        }
+    }
+
+    /// Negative: a punycode label is no bucket under a domain; outside every domain the host is
+    /// a valid domain that is no bucket name, so it reads path-style.
+    #[test]
+    fn a_punycode_label_is_never_a_bucket() {
+        assert_eq!(refusal(&["example.com"], "xn--bcher-kva.example.com"), Some(HostRefusal::RefusedBucket));
+        assert_eq!(bucket(&["example.com"], "xn--bcher-kva.de"), None);
+        assert_eq!(refusal(&["example.com"], "xn--bcher-kva.de"), None);
+    }
+
+    /// Negative: a label over 63 bytes is no bucket, under a domain or outside one.
+    #[test]
+    fn a_label_over_63_bytes_is_never_a_bucket() {
+        let long = "a".repeat(64);
+        let under = format!("{long}.example.com");
+        assert_eq!(refusal(&["example.com"], &under), Some(HostRefusal::RefusedBucket));
+        let outside = format!("{long}.org");
+        assert_eq!(bucket(&["example.com"], &outside), None);
+        assert_eq!(refusal(&["example.com"], &outside), None);
+        let at_limit = "a".repeat(63);
+        assert_eq!(bucket(&["example.com"], &format!("{at_limit}.example.com")), Some(at_limit));
+    }
+
+    /// Positive and negative: two configured domains where one merely ends in the other's letters
+    /// both work, and each domain itself is path-style.
+    #[test]
+    fn a_domain_and_its_suffix_lookalike_are_both_served() {
+        let domains = ["example.com", "badexample.com"];
+        assert_eq!(bucket(&domains, "photos.example.com").as_deref(), Some("photos"));
+        assert_eq!(bucket(&domains, "photos.badexample.com").as_deref(), Some("photos"));
+        assert_eq!(bucket(&domains, "badexample.com"), None);
+        assert_eq!(bucket(&domains, "bad.example.com").as_deref(), Some("bad"));
+    }
+
+    /// Negative: a configured domain is matched exactly, case included.
+    #[test]
+    fn a_configured_domain_is_matched_on_its_case() {
+        assert_eq!(bucket(&["Example.COM"], "photos.Example.COM").as_deref(), Some("photos"));
+        assert_eq!(bucket(&["Example.COM"], "photos.example.com").as_deref(), Some("photos.example.com"));
+        assert_eq!(bucket(&["Example.COM"], "Example.COM"), None);
+    }
+}

crates/gateway-adapter/src/lib.rs

@@ -0,0 +1,107 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! RustFS implementations of the `rustfs-gateway` extension traits.
+//!
+//! This is the gateway's ring 2: it may depend on RustFS business crates, and nothing in the
+//! gateway depends on it. Each module adapts one extension point to the RustFS subsystem that
+//! owns the answer.
+
+pub mod authz_audit;
+pub mod bucket_owner;
+pub mod credentials;
+pub mod host;
+pub mod policy;
+pub mod root;
+pub mod trace;
+
+use std::sync::Arc;
+
+use rustfs_gateway::{Authenticator, CredentialProvider, ServiceBuilder};
+
+pub use authz_audit::{DenialAdmission, DenialAuditSink, GatewayDenialRecord, RustfsAuthzAudit};
+pub use bucket_owner::{BucketLookup, OwnerLookupError, RustfsBucketOwners};
+pub use credentials::{IamCredentials, IamKeyLookup, STS_SESSION_ISSUER};
+pub use host::rustfs_host_resolver;
+pub use policy::{IamPolicySource, IamPrincipalLookup, PrincipalKind, RustfsPrincipal};
+pub use root::{MIN_SECRET_KEY_LEN, RootCredentials, RootCredentialsError, RootSource};
+pub use trace::{AttachOutcome, RequestIdCarrier, attach_host_request_id, host_request_id};
+
+/// The IAM-backed credential provider and policy source, assembled together.
+///
+/// Both halves read the same IAM handle and the same live [`RootSource`], so a key cannot be root
+/// to one and an ordinary user to the other. Neither half can be taken out on its own: the fields
+/// are private, and [`IamAdapters::install`] consumes the pair and hands both to one builder.
+///
+/// Mixing the halves of two assemblies does not compile:
+///
+/// ```compile_fail
+/// fn mix(a: rustfs_gateway_adapter::IamAdapters, b: rustfs_gateway_adapter::IamAdapters) {
+///     let _mixed = (a.credentials, b.policy);
+/// }
+/// ```
+///
+/// Nor does building a half outside the pair:
+///
+/// ```compile_fail
+/// use std::sync::Arc;
+/// use rustfs_gateway_adapter::{IamPolicySource, IamPrincipalLookup, RootSource};
+/// fn build(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>) {
+///     let _alone = IamPolicySource::new(iam, root);
+/// }
+/// ```
+///
+/// Installing the pair is the one supported path:
+///
+/// ```no_run
+/// use std::sync::Arc;
+/// use rustfs_gateway::{RegionSet, ServiceBuilder, SigV4Authenticator};
+/// use rustfs_gateway_adapter::{IamAdapters, IamPrincipalLookup, RootSource};
+/// fn assemble(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>, regions: RegionSet) -> ServiceBuilder {
+///     IamAdapters::new(iam, root).install(ServiceBuilder::new(), |provider| SigV4Authenticator::new(provider, regions))
+/// }
+/// ```
+pub struct IamAdapters {
+    credentials: IamCredentials,
+    policy: IamPolicySource,
+}
+
+impl IamAdapters {
+    /// Assembles both adapters over one IAM handle and one root source. The binary passes a
+    /// source that reads its live root credentials on every call (see [`RootSource`]).
+    #[must_use]
+    pub fn new(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>) -> Self {
+        Self {
+            credentials: IamCredentials::new(iam.clone(), root.clone()),
+            policy: IamPolicySource::new(iam, root),
+        }
+    }
+
+    /// Installs both halves into `builder`: the credential provider through the authenticator
+    /// `authenticator` builds over it (for example `SigV4Authenticator::new(provider, regions)`
+    /// with the RustFS profile applied), and the policy source.
+    ///
+    /// Consumes the pair and gives both halves to this one builder. The `authenticator` closure
+    /// receives the provider handle and may keep a clone of it; nothing here prevents that, so
+    /// the closure must not hand it to a second builder.
+    #[must_use]
+    pub fn install<A: Authenticator>(
+        self,
+        builder: ServiceBuilder,
+        authenticator: impl FnOnce(Arc<dyn CredentialProvider>) -> A,
+    ) -> ServiceBuilder {
+        let authenticator = authenticator(Arc::new(self.credentials));
+        builder.authenticator(authenticator).policy_source(self.policy)
+    }
+}

crates/gateway-adapter/src/policy.rs

@@ -0,0 +1,779 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! The gateway's [`PolicySource`], answered by RustFS IAM.
+//!
+//! Responsible for: the one reading of the caller's IAM facts a request is judged against — which
+//! principal kind it is, the user it acts for, its groups, its token claims, and whether it owns
+//! the deployment — packed as a [`RustfsPrincipal`] into the gateway's [`PolicySnapshot`].
+//! NOT responsible for: evaluating any policy, choosing which policy documents apply (that stays
+//! in `IamSys::is_allowed`, which owns the STS, OIDC, session-policy and service-account rules), or
+//! verifying the signature or the session token (the gateway did both before calling this).
+//! Upstream: the gateway's verified [`Identity`]. Downstream: the RustFS authorizer, which reads
+//! the payload back with `PolicySnapshot::get::<RustfsPrincipal>()`.
+//!
+//! # Failure classes
+//!
+//! Every refusal is a [`PolicyError`] with a constant label and nothing from IAM or the request:
+//! IAM not ready, not initialized, or failing, or root credentials not initialized, is
+//! [`PolicyError::unavailable`]; a principal IAM no longer knows is [`UNKNOWN_PRINCIPAL`]; a
+//! disabled one, or one whose secret fails the effective-length rule, is
+//! [`DISABLED_PRINCIPAL`]; an identity
+//! whose credential kind disagrees with IAM's record is [`SESSION_MISMATCH`]; a token whose claims
+//! cannot be read is [`PolicyError::malformed`]. None of them is ever an empty snapshot: an empty
+//! snapshot is what an anonymous caller gets, and answering a fault with it would judge an
+//! authenticated caller as anonymous.
+//!
+//! # Secret handling
+//!
+//! [`RustfsPrincipal`] holds no secret key and no session token, and implements no `Debug`, so a
+//! snapshot formatted into a log cannot print its claims either. The root credentials and the STS
+//! token-signing key are read per request from the shared [`RootSource`] and never stored here.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use rustfs_credentials::Credentials as IamCredential;
+use rustfs_gateway::{BoxFuture, Identity, PolicyError, PolicySnapshot, PolicySource};
+use rustfs_iam::error::Error as IamError;
+use rustfs_iam::store::Store;
+use rustfs_iam::sys::{
+    IamSys, SESSION_POLICY_NAME, get_claims_from_token_with_secret, get_claims_from_token_with_secret_allow_missing_exp,
+    is_rustfs_oidc_claims,
+};
+use rustfs_policy::auth::ACCOUNT_OFF;
+use serde_json::Value;
+
+use crate::credentials::IamKeyLookup;
+use crate::root::{RootCredentials, RootSource, is_usable_secret};
+
+/// [`PolicyError`] label: IAM does not know the authenticated principal (any longer).
+pub const UNKNOWN_PRINCIPAL: &str = "unknown-principal";
+/// [`PolicyError`] label: IAM holds the principal disabled, expired, or otherwise invalid.
+pub const DISABLED_PRINCIPAL: &str = "disabled-principal";
+/// [`PolicyError`] label: the identity's credential kind (session or long-term) disagrees with
+/// what IAM holds for its access key.
+pub const SESSION_MISMATCH: &str = "session-mismatch";
+
+/// The narrow slice of IAM a policy reading needs, on top of [`IamKeyLookup`].
+///
+/// Implemented for [`IamSys`]; exists so the source can be tested against a fake.
+pub trait IamPrincipalLookup: IamKeyLookup {
+    /// The built-in IAM groups `user` belongs to now, as `IamSys::user_group_memberships`
+    /// answers. These may include groups granted after a credential was issued.
+    fn group_memberships(&self, user: &str) -> Vec<String>;
+}
+
+impl<T: Store> IamPrincipalLookup for IamSys<T> {
+    fn group_memberships(&self, user: &str) -> Vec<String> {
+        IamSys::user_group_memberships(self, user)
+    }
+}
+
+/// Which kind of credential the principal authenticated with.
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub enum PrincipalKind {
+    /// The deployment's root credentials.
+    Root,
+    /// A long-term IAM user.
+    User,
+    /// A service account, acting for its parent user.
+    ServiceAccount,
+    /// An STS temporary credential, acting for its parent user.
+    Temporary,
+}
+
+/// The caller's IAM facts: the payload of every [`PolicySnapshot`] [`IamPolicySource`] produces.
+///
+/// Carries no secret key and no session token, and deliberately implements no `Debug`.
+pub struct RustfsPrincipal {
+    kind: PrincipalKind,
+    account: String,
+    username: String,
+    parent_user: String,
+    credential_groups: Option<Vec<String>>,
+    group_memberships: Vec<String>,
+    claims: Option<HashMap<String, Value>>,
+    is_owner: bool,
+}
+
+impl RustfsPrincipal {
+    /// The kind of credential the caller authenticated with.
+    #[must_use]
+    pub fn kind(&self) -> PrincipalKind {
+        self.kind
+    }
+
+    /// The access key the caller signed with (IAM `Args::account`).
+    #[must_use]
+    pub fn account(&self) -> &str {
+        &self.account
+    }
+
+    /// The user the caller acts as: the parent user for an STS credential or a service account,
+    /// the access key otherwise.
+    #[must_use]
+    pub fn username(&self) -> &str {
+        &self.username
+    }
+
+    /// The parent user IAM records for the credential; empty for root and long-term users.
+    #[must_use]
+    pub fn parent_user(&self) -> &str {
+        &self.parent_user
+    }
+
+    /// The groups recorded on the credential itself (IAM `Args::groups`).
+    #[must_use]
+    pub fn credential_groups(&self) -> &Option<Vec<String>> {
+        &self.credential_groups
+    }
+
+    /// The groups IAM currently lists for [`Self::username`], read once with the snapshot.
+    #[must_use]
+    pub fn group_memberships(&self) -> &[String] {
+        &self.group_memberships
+    }
+
+    /// The claims the credential's token carries; `None` when it has none.
+    #[must_use]
+    pub fn claims(&self) -> Option<&HashMap<String, Value>> {
+        self.claims.as_ref()
+    }
+
+    /// Whether the caller owns the deployment (IAM `Args::is_owner`): root, or a credential of
+    /// root that no session policy or external OIDC identity narrows.
+    #[must_use]
+    pub fn is_owner(&self) -> bool {
+        self.is_owner
+    }
+}
+
+/// A [`PolicySource`] over RustFS IAM and the live root credentials.
+///
+/// Built only by [`crate::IamAdapters::new`], so it always reads the same [`RootSource`] as the
+/// credential provider it was assembled with.
+pub struct IamPolicySource {
+    iam: Arc<dyn IamPrincipalLookup>,
+    root: Arc<dyn RootSource>,
+}
+
+impl IamPolicySource {
+    pub(crate) fn new(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>) -> Self {
+        Self { iam, root }
+    }
+
+    async fn read(&self, identity: Option<&Identity>) -> Result<PolicySnapshot, PolicyError> {
+        let Some(identity) = identity else {
+            return Ok(PolicySnapshot::empty());
+        };
+        let principal = self.principal(identity).await?;
+        Ok(PolicySnapshot::of(Arc::new(principal)))
+    }
+
+    async fn principal(&self, identity: &Identity) -> Result<RustfsPrincipal, PolicyError> {
+        let access_key = identity.access_key_id();
+        let has_session = identity.session().is_some();
+        // Without root credentials neither ownership nor STS claims can be decided.
+        let Some(root) = self.root.current() else {
+            return Err(PolicyError::unavailable());
+        };
+        let root_access_key = root.access_key();
+
+        if access_key == root_access_key {
+            // Root has no session token, so a session identity for its key is not one IAM issued.
+            if has_session {
+                return Err(PolicyError::because(SESSION_MISMATCH));
+            }
+            return Ok(RustfsPrincipal {
+                kind: PrincipalKind::Root,
+                account: access_key.to_owned(),
+                username: access_key.to_owned(),
+                parent_user: String::new(),
+                credential_groups: None,
+                group_memberships: Vec::new(),
+                claims: None,
+                is_owner: true,
+            });
+        }
+
+        if !self.iam.is_ready() {
+            return Err(PolicyError::unavailable());
+        }
+        let cred = match self.iam.check_key(access_key).await {
+            Ok((Some(identity), true)) => identity.credentials,
+            Ok((Some(_), false)) => return Err(PolicyError::because(DISABLED_PRINCIPAL)),
+            Ok((None, _)) => return Err(PolicyError::because(UNKNOWN_PRINCIPAL)),
+            Err(error) => return Err(classify_iam_error(&error)),
+        };
+        if cred.status == ACCOUNT_OFF || !is_usable_secret(cred.secret_key.as_bytes()) {
+            return Err(PolicyError::because(DISABLED_PRINCIPAL));
+        }
+
+        let kind = if cred.is_service_account() {
+            PrincipalKind::ServiceAccount
+        } else if !cred.session_token.is_empty() {
+            PrincipalKind::Temporary
+        } else {
+            PrincipalKind::User
+        };
+        // The gateway admitted this identity against the credential IAM returned, so the kinds
+        // agree unless IAM's record changed in between; refuse rather than judge a stale reading.
+        if has_session != (kind == PrincipalKind::Temporary) {
+            return Err(PolicyError::because(SESSION_MISMATCH));
+        }
+
+        let claims = read_claims(kind, &cred, &root)?;
+        let username = match kind {
+            PrincipalKind::Temporary | PrincipalKind::ServiceAccount => cred.parent_user.clone(),
+            PrincipalKind::Root | PrincipalKind::User => cred.access_key.clone(),
+        };
+        let membership_subject = if username.is_empty() { access_key } else { username.as_str() };
+        let group_memberships = self.iam.group_memberships(membership_subject);
+        let is_owner = acts_for_root(&cred, claims.as_ref(), root_access_key);
+
+        Ok(RustfsPrincipal {
+            kind,
+            account: access_key.to_owned(),
+            username,
+            parent_user: cred.parent_user,
+            credential_groups: cred.groups,
+            group_memberships,
+            claims,
+            is_owner,
+        })
+    }
+}
+
+/// The claims of the credential's own token. The gateway admitted the presented token only when it
+/// equals the stored one, so reading the stored token reads the presented one.
+fn read_claims(
+    kind: PrincipalKind,
+    cred: &IamCredential,
+    root: &RootCredentials,
+) -> Result<Option<HashMap<String, Value>>, PolicyError> {
+    // As in legacy `check_claims_from_token_with_context`: no token, no claims.
+    if cred.session_token.is_empty() {
+        return Ok(None);
+    }
+    let claims = match kind {
+        PrincipalKind::ServiceAccount => {
+            get_claims_from_token_with_secret_allow_missing_exp(&cred.session_token, &cred.secret_key)
+                .map_err(|_| PolicyError::malformed())?
+        }
+        PrincipalKind::Temporary => get_claims_from_token_with_secret(&cred.session_token, root.sts_signing_key())
+            .map_err(|_| PolicyError::malformed())?,
+        PrincipalKind::Root | PrincipalKind::User => return Ok(None),
+    };
+    Ok(if claims.is_empty() { None } else { Some(claims) })
+}
+
+/// Classifies an IAM failure without keeping any of its text.
+fn classify_iam_error(error: &IamError) -> PolicyError {
+    match error {
+        IamError::NoSuchUser(_)
+        | IamError::NoSuchServiceAccount(_)
+        | IamError::NoSuchTempAccount(_)
+        | IamError::NoSuchAccount(_) => PolicyError::because(UNKNOWN_PRINCIPAL),
+        _ => PolicyError::unavailable(),
+    }
+}
+
+/// Root ownership as the legacy edge decides it (`rustfs/src/auth.rs` `has_root_access`): the
+/// credential is root's or acts for root, and neither a session policy nor an external OIDC
+/// identity narrows it.
+fn acts_for_root(cred: &IamCredential, claims: Option<&HashMap<String, Value>>, root_access_key: &str) -> bool {
+    (cred.access_key == root_access_key || cred.parent_user == root_access_key)
+        && !claims.is_some_and(|claims| claims.contains_key(SESSION_POLICY_NAME) || is_rustfs_oidc_claims(claims))
+}
+
+impl PolicySource for IamPolicySource {
+    fn snapshot<'a>(&'a self, identity: Option<&'a Identity>) -> BoxFuture<'a, Result<PolicySnapshot, PolicyError>> {
+        Box::pin(self.read(identity))
+    }
+}
+
+#[cfg(test)]
+#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
+mod tests {
+    use super::*;
+    use crate::root::RootCell;
+    use rustfs_credentials::{Credentials as IamCredential, IAM_POLICY_CLAIM_NAME_SA};
+    use rustfs_gateway::SessionBinding;
+    use rustfs_iam::error::Error as IamError;
+    use rustfs_iam::sys::SESSION_POLICY_NAME;
+    use rustfs_iam::utils::generate_jwt;
+    use rustfs_policy::auth::UserIdentity;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use time::OffsetDateTime;
+
+    const ROOT_ACCESS_KEY: &str = "rootadmin";
+    const ROOT_SECRET: &str = "root-secret-key-0001";
+    /// Deliberately not the root secret, so every STS test proves which key claims are read with.
+    const SIGNING_KEY: &str = "sts-signing-key-0001";
+    const ACCESS_KEY: &str = "AKIDUSER0001";
+    const SECRET: &str = "user-secret-key-0001";
+    const PARENT: &str = "alice";
+    const EXPIRES_AT: i64 = 4_000_000_000;
+
+    type KeyAnswer = Result<(Option<UserIdentity>, bool), IamError>;
+    type Answer = Box<dyn Fn() -> KeyAnswer + Send + Sync>;
+
+    struct FakeIam {
+        ready: bool,
+        answer: Answer,
+        calls: AtomicUsize,
+    }
+
+    impl IamKeyLookup for FakeIam {
+        fn is_ready(&self) -> bool {
+            self.ready
+        }
+
+        fn check_key<'a>(&'a self, _access_key: &'a str) -> BoxFuture<'a, KeyAnswer> {
+            self.calls.fetch_add(1, Ordering::SeqCst);
+            let answer = (self.answer)();
+            Box::pin(async move { answer })
+        }
+    }
+
+    impl IamPrincipalLookup for FakeIam {
+        /// Distinct groups per name, so a test can tell which name the source asked about.
+        fn group_memberships(&self, user: &str) -> Vec<String> {
+            match user {
+                PARENT => vec!["parent-group".to_owned()],
+                ACCESS_KEY => vec!["key-group".to_owned()],
+                ROOT_ACCESS_KEY => vec!["root-group".to_owned()],
+                _ => Vec::new(),
+            }
+        }
+    }
+
+    fn fake(ready: bool, answer: impl Fn() -> KeyAnswer + Send + Sync + 'static) -> Arc<FakeIam> {
+        Arc::new(FakeIam {
+            ready,
+            answer: Box::new(answer),
+            calls: AtomicUsize::new(0),
+        })
+    }
+
+    fn root() -> RootCredentials {
+        RootCredentials::new(ROOT_ACCESS_KEY, ROOT_SECRET, SIGNING_KEY).expect("valid root credentials")
+    }
+
+    fn source(iam: Arc<FakeIam>) -> IamPolicySource {
+        IamPolicySource::new(iam, Arc::new(RootCell::new(Some(root()))))
+    }
+
+    fn user() -> IamCredential {
+        IamCredential {
+            access_key: ACCESS_KEY.to_owned(),
+            secret_key: SECRET.to_owned(),
+            status: "on".to_owned(),
+            groups: Some(vec!["cred-group".to_owned()]),
+            ..Default::default()
+        }
+    }
+
+    fn sts_token(claims: &[(&str, Value)], secret: &str) -> String {
+        let mut map: HashMap<String, Value> = claims.iter().map(|(k, v)| ((*k).to_owned(), v.clone())).collect();
+        map.insert("exp".to_owned(), Value::from(EXPIRES_AT));
+        generate_jwt(&map, secret).expect("a token")
+    }
+
+    fn sts(parent: &str, token: String) -> IamCredential {
+        IamCredential {
+            session_token: token,
+            parent_user: parent.to_owned(),
+            expiration: Some(OffsetDateTime::from_unix_timestamp(EXPIRES_AT).expect("valid timestamp")),
+            ..user()
+        }
+    }
+
+    fn found(cred: IamCredential) -> KeyAnswer {
+        Ok((Some(UserIdentity::new(cred)), true))
+    }
+
+    fn long_term(access_key: &str) -> Identity {
+        Identity::new(access_key).expect("valid access key")
+    }
+
+    fn session(access_key: &str) -> Identity {
+        let binding = SessionBinding::new(crate::STS_SESSION_ISSUER, EXPIRES_AT).expect("valid binding");
+        long_term(access_key).with_session_binding(binding)
+    }
+
+    async fn principal_of(source: &IamPolicySource, identity: &Identity) -> PolicySnapshot {
+        source.snapshot(Some(identity)).await.expect("a snapshot")
+    }
+
+    fn payload(snapshot: &PolicySnapshot) -> &RustfsPrincipal {
+        snapshot.get::<RustfsPrincipal>().expect("the payload is a RustfsPrincipal")
+    }
+
+    fn refusal(result: Result<PolicySnapshot, PolicyError>) -> PolicyError {
+        match result {
+            Ok(snapshot) => panic!("expected a refusal, got a snapshot (empty: {})", snapshot.is_empty()),
+            Err(error) => error,
+        }
+    }
+
+    /// Compile-time: `RustfsPrincipal` has no `Debug`, so no formatting of a snapshot payload can
+    /// reach its claims. The two blanket impls make the path ambiguous, and so a compile error, as
+    /// soon as `RustfsPrincipal: Debug` holds.
+    #[test]
+    fn policy_principal_has_no_debug() {
+        trait AmbiguousIfDebug<A> {
+            fn some_item() {}
+        }
+        impl<T: ?Sized> AmbiguousIfDebug<()> for T {}
+        impl<T: ?Sized + core::fmt::Debug> AmbiguousIfDebug<u8> for T {}
+        let _ = <RustfsPrincipal as AmbiguousIfDebug<_>>::some_item;
+    }
+
+    /// Positive — a long-term user's facts: itself as username, both group sources, no claims,
+    /// not the owner.
+    #[tokio::test]
+    async fn policy_user_snapshot_carries_iam_facts() {
+        let iam = fake(true, || found(user()));
+        let snapshot = principal_of(&source(iam), &long_term(ACCESS_KEY)).await;
+        let principal = payload(&snapshot);
+        assert!(principal.kind() == PrincipalKind::User);
+        assert_eq!(principal.account(), ACCESS_KEY);
+        assert_eq!(principal.username(), ACCESS_KEY);
+        assert_eq!(principal.parent_user(), "");
+        assert_eq!(principal.credential_groups(), &Some(vec!["cred-group".to_owned()]));
+        assert_eq!(principal.group_memberships(), ["key-group".to_owned()]);
+        assert!(principal.claims().is_none());
+        assert!(!principal.is_owner());
+    }
+
+    /// Positive — an STS credential acts for its parent user, and its claims are the ones its
+    /// token carries, read with the root secret that signed it.
+    #[tokio::test]
+    async fn policy_sts_snapshot_carries_parent_user_and_claims() {
+        let token = sts_token(&[("sub", Value::from("alice-session"))], SIGNING_KEY);
+        let iam = fake(true, move || found(sts(PARENT, token.clone())));
+        let snapshot = principal_of(&source(iam), &session(ACCESS_KEY)).await;
+        let principal = payload(&snapshot);
+        assert!(principal.kind() == PrincipalKind::Temporary);
+        assert_eq!(principal.account(), ACCESS_KEY);
+        assert_eq!(principal.parent_user(), PARENT);
+        assert_eq!(principal.username(), PARENT);
+        let claims = principal.claims().expect("an STS principal has claims");
+        assert_eq!(claims.get("sub"), Some(&Value::from("alice-session")));
+        assert_eq!(principal.group_memberships(), ["parent-group".to_owned()]);
+        assert!(!principal.is_owner());
+    }
+
+    /// Positive — a service account's claims are read with its own secret, and it acts for its
+    /// parent.
+    #[tokio::test]
+    async fn policy_service_account_snapshot_reads_its_own_token() {
+        let mut token_claims: HashMap<String, Value> = HashMap::new();
+        token_claims.insert(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"));
+        token_claims.insert("parent".to_owned(), Value::from(PARENT));
+        let token = generate_jwt(&token_claims, SECRET).expect("a token");
+        let iam = fake(true, move || {
+            let mut cred = user();
+            cred.session_token = token.clone();
+            cred.parent_user = PARENT.to_owned();
+            cred.claims = Some([(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"))].into());
+            found(cred)
+        });
+        let snapshot = principal_of(&source(iam), &long_term(ACCESS_KEY)).await;
+        let principal = payload(&snapshot);
+        assert!(principal.kind() == PrincipalKind::ServiceAccount);
+        assert_eq!(principal.username(), PARENT);
+        assert_eq!(principal.claims().and_then(|c| c.get("parent")), Some(&Value::from(PARENT)));
+        assert_eq!(principal.group_memberships(), ["parent-group".to_owned()]);
+        assert!(!principal.is_owner());
+    }
+
+    /// Positive and negative — root is the owner without asking IAM, even before IAM is ready;
+    /// an STS credential of root is the owner too, but not once a session policy narrows it.
+    #[tokio::test]
+    async fn policy_owner_is_root_or_unrestricted_root_session() {
+        let not_ready = fake(false, || found(user()));
+        let snapshot = principal_of(&source(not_ready.clone()), &long_term(ROOT_ACCESS_KEY)).await;
+        let root = payload(&snapshot);
+        assert!(root.kind() == PrincipalKind::Root);
+        assert!(root.is_owner());
+        assert_eq!(not_ready.calls.load(Ordering::SeqCst), 0);
+
+        let plain = sts_token(&[], SIGNING_KEY);
+        let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, plain.clone())));
+        assert!(payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
+
+        let narrowed = sts_token(&[(SESSION_POLICY_NAME, Value::from("e30"))], SIGNING_KEY);
+        let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, narrowed.clone())));
+        assert!(!payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
+    }
+
+    /// Negative — an anonymous caller is an empty snapshot, and IAM is not asked.
+    #[tokio::test]
+    async fn policy_anonymous_is_empty_snapshot() {
+        let iam = fake(true, || found(user()));
+        let snapshot = source(iam.clone()).snapshot(None).await.expect("an anonymous snapshot");
+        assert!(snapshot.is_empty());
+        assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
+    }
+
+    /// Negative — IAM that is not ready, not initialized, or failing is `unavailable`: never an
+    /// empty snapshot, which an authorizer would judge as an anonymous caller.
+    #[tokio::test]
+    async fn policy_iam_unavailable_is_unavailable() {
+        let not_ready = fake(false, || found(user()));
+        let error = refusal(source(not_ready.clone()).snapshot(Some(&long_term(ACCESS_KEY))).await);
+        assert_eq!(error, PolicyError::unavailable());
+        assert_eq!(not_ready.calls.load(Ordering::SeqCst), 0);
+
+        let failures: [fn() -> IamError; 2] = [
+            || IamError::IamSysNotInitialized,
+            || IamError::StringError("storage read failed".to_owned()),
+        ];
+        for failure in failures {
+            let iam = fake(true, move || Err(failure()));
+            let error = refusal(source(iam).snapshot(Some(&long_term(ACCESS_KEY))).await);
+            assert_eq!(error, PolicyError::unavailable(), "{:?}", failure());
+        }
+    }
+
+    /// Negative — a principal IAM no longer knows is refused as unknown, not as unavailable and
+    /// not as an empty snapshot.
+    #[tokio::test]
+    async fn policy_unknown_principal_is_refused() {
+        let answers: [fn() -> KeyAnswer; 2] = [|| Ok((None, false)), || Err(IamError::NoSuchUser(ACCESS_KEY.to_owned()))];
+        for answer in answers {
+            let error = refusal(source(fake(true, answer)).snapshot(Some(&long_term(ACCESS_KEY))).await);
+            assert_eq!(error, PolicyError::because(UNKNOWN_PRINCIPAL));
+        }
+    }
+
+    /// Negative — a disabled or invalid principal is refused rather than given a snapshot.
+    #[tokio::test]
+    async fn policy_disabled_principal_is_refused() {
+        let invalid = fake(true, || Ok((Some(UserIdentity::new(user())), false)));
+        let error = refusal(source(invalid).snapshot(Some(&long_term(ACCESS_KEY))).await);
+        assert_eq!(error, PolicyError::because(DISABLED_PRINCIPAL));
+
+        let off = fake(true, || {
+            let mut cred = user();
+            cred.status = "off".to_owned();
+            found(cred)
+        });
+        let error = refusal(source(off).snapshot(Some(&long_term(ACCESS_KEY))).await);
+        assert_eq!(error, PolicyError::because(DISABLED_PRINCIPAL));
+    }
+
+    /// Negative — an STS token not signed by root yields no claims at all: it is `malformed`.
+    #[tokio::test]
+    async fn policy_sts_token_with_foreign_signature_is_malformed() {
+        let forged = sts_token(&[("sub", Value::from("alice-session"))], "not-the-root-secret");
+        let iam = fake(true, move || found(sts(PARENT, forged.clone())));
+        let error = refusal(source(iam).snapshot(Some(&session(ACCESS_KEY))).await);
+        assert_eq!(error, PolicyError::malformed());
+    }
+
+    /// Negative — the identity's credential kind must match what IAM holds: a session identity for
+    /// a long-term key, or a long-term identity for an STS key, is refused.
+    #[tokio::test]
+    async fn policy_session_kind_mismatch_is_refused() {
+        let token = sts_token(&[], SIGNING_KEY);
+        let iam = fake(true, move || found(sts(PARENT, token.clone())));
+        let error = refusal(source(iam).snapshot(Some(&long_term(ACCESS_KEY))).await);
+        assert_eq!(error, PolicyError::because(SESSION_MISMATCH));
+
+        let iam = fake(true, || found(user()));
+        let error = refusal(source(iam).snapshot(Some(&session(ACCESS_KEY))).await);
+        assert_eq!(error, PolicyError::because(SESSION_MISMATCH));
+
+        let iam = fake(false, || found(user()));
+        let error = refusal(source(iam).snapshot(Some(&session(ROOT_ACCESS_KEY))).await);
+        assert_eq!(error, PolicyError::because(SESSION_MISMATCH));
+    }
+
+    fn service_account(parent: &str, extra_claims: &[(&str, Value)]) -> KeyAnswer {
+        let mut token_claims: HashMap<String, Value> = extra_claims.iter().map(|(k, v)| ((*k).to_owned(), v.clone())).collect();
+        token_claims.insert(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"));
+        let mut cred = user();
+        cred.session_token = generate_jwt(&token_claims, SECRET).expect("a token");
+        cred.parent_user = parent.to_owned();
+        cred.claims = Some([(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"))].into());
+        found(cred)
+    }
+
+    /// Negative — a valid-flagged identity whose secret is empty or short gets no snapshot.
+    #[tokio::test]
+    async fn policy_short_secret_principal_is_refused() {
+        for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0"] {
+            let iam = fake(true, move || {
+                let mut cred = user();
+                cred.secret_key = weak.to_owned();
+                found(cred)
+            });
+            let error = refusal(source(iam).snapshot(Some(&long_term(ACCESS_KEY))).await);
+            assert_eq!(error, PolicyError::because(DISABLED_PRINCIPAL), "{weak:?}");
+        }
+        for usable in ["12345678", "abc\0defg"] {
+            let iam = fake(true, move || {
+                let mut cred = user();
+                cred.secret_key = usable.to_owned();
+                found(cred)
+            });
+            assert!(payload(&principal_of(&source(iam), &long_term(ACCESS_KEY)).await).kind() == PrincipalKind::User);
+        }
+    }
+
+    /// Negative — a root session issued through external OIDC is not the owner, though its parent
+    /// is root; the same session without the OIDC issuer is.
+    #[tokio::test]
+    async fn policy_oidc_root_session_is_not_owner() {
+        let oidc = sts_token(
+            &[
+                ("iss", Value::from("rustfs-oidc")),
+                ("oidc_provider", Value::from("corp-idp")),
+                ("sub", Value::from("alice@corp")),
+            ],
+            SIGNING_KEY,
+        );
+        let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, oidc.clone())));
+        assert!(!payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
+
+        let local = sts_token(
+            &[
+                ("iss", Value::from("rustfs-sts")),
+                ("oidc_provider", Value::from("corp-idp")),
+                ("sub", Value::from("alice@corp")),
+            ],
+            SIGNING_KEY,
+        );
+        let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, local.clone())));
+        assert!(payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
+    }
+
+    /// Positive and negative — a service account of root is the owner; one narrowed by a session
+    /// policy is not, and neither is a service account of an ordinary user.
+    #[tokio::test]
+    async fn policy_service_account_of_root_is_owner() {
+        let iam = fake(true, || service_account(ROOT_ACCESS_KEY, &[]));
+        let principal = principal_of(&source(iam), &long_term(ACCESS_KEY)).await;
+        assert!(payload(&principal).kind() == PrincipalKind::ServiceAccount);
+        assert!(payload(&principal).is_owner());
+
+        let iam = fake(true, || service_account(ROOT_ACCESS_KEY, &[(SESSION_POLICY_NAME, Value::from("e30"))]));
+        assert!(!payload(&principal_of(&source(iam), &long_term(ACCESS_KEY)).await).is_owner());
+
+        let iam = fake(true, || service_account(PARENT, &[]));
+        assert!(!payload(&principal_of(&source(iam), &long_term(ACCESS_KEY)).await).is_owner());
+    }
+
+    /// Negative — STS claims are read with the STS signing key, not with the root secret: a token
+    /// signed with the root secret while the signing key differs is `malformed`.
+    #[tokio::test]
+    async fn policy_sts_claims_use_the_signing_key() {
+        let root_signed = sts_token(&[("sub", Value::from("alice-session"))], ROOT_SECRET);
+        let iam = fake(true, move || found(sts(PARENT, root_signed.clone())));
+        let error = refusal(source(iam).snapshot(Some(&session(ACCESS_KEY))).await);
+        assert_eq!(error, PolicyError::malformed());
+    }
+
+    /// Negative — without root credentials no principal can be judged: `unavailable`, never an
+    /// empty snapshot, and IAM is not asked.
+    #[tokio::test]
+    async fn policy_without_root_is_unavailable() {
+        let iam = fake(true, || found(user()));
+        let source = IamPolicySource::new(iam.clone(), Arc::new(RootCell::new(None)));
+        for identity in [long_term(ACCESS_KEY), long_term(ROOT_ACCESS_KEY)] {
+            assert_eq!(refusal(source.snapshot(Some(&identity)).await), PolicyError::unavailable());
+        }
+        assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
+    }
+
+    type LiveRoot = Arc<std::sync::Mutex<Option<(String, String)>>>;
+
+    /// The primary path: a source that reads a live accessor on every call, as the binary's does,
+    /// with nothing cached in between.
+    fn live_root_source(live: &LiveRoot) -> Arc<dyn RootSource> {
+        let accessor = live.clone();
+        Arc::new(move || {
+            let guard = accessor.lock().ok()?;
+            let (access_key, secret_key) = guard.as_ref()?;
+            RootCredentials::new(access_key, secret_key, SIGNING_KEY).ok().map(Arc::new)
+        })
+    }
+
+    fn set_live(live: &LiveRoot, value: Option<(&str, &str)>) {
+        *live.lock().expect("live root lock") = value.map(|(ak, sk)| (ak.to_owned(), sk.to_owned()));
+    }
+
+    /// Positive and negative — the two adapters read one live root accessor: a root change is seen
+    /// by both on the next request with no second step, the old root key stops being root to
+    /// both, and an uninitialized or unusable root makes both unavailable.
+    #[tokio::test]
+    async fn adapters_share_one_live_root() {
+        use rustfs_gateway::{CredentialLookup, CredentialProvider, ProviderError};
+
+        let live: LiveRoot = Arc::default();
+        set_live(&live, Some((ROOT_ACCESS_KEY, ROOT_SECRET)));
+        let iam = fake(true, || Ok((None, false)));
+        let adapters = crate::IamAdapters::new(iam, live_root_source(&live));
+
+        let found = adapters.credentials.lookup(ROOT_ACCESS_KEY).await;
+        assert!(matches!(found, Ok(CredentialLookup::Found(_))));
+        let snapshot = principal_of(&adapters.policy, &long_term(ROOT_ACCESS_KEY)).await;
+        assert!(payload(&snapshot).is_owner());
+
+        set_live(&live, Some(("newroot", "new-root-secret-01")));
+        match adapters.credentials.lookup("newroot").await {
+            Ok(CredentialLookup::Found(credentials)) => assert_eq!(credentials.secret().expose(), b"new-root-secret-01"),
+            _ => panic!("the new root key must resolve to the new secret"),
+        }
+        assert!(matches!(
+            adapters.credentials.lookup(ROOT_ACCESS_KEY).await,
+            Ok(CredentialLookup::NotFound)
+        ));
+        assert!(payload(&principal_of(&adapters.policy, &long_term("newroot")).await).is_owner());
+        let old = refusal(adapters.policy.snapshot(Some(&long_term(ROOT_ACCESS_KEY))).await);
+        assert_eq!(old, PolicyError::because(UNKNOWN_PRINCIPAL));
+
+        for unusable in [None, Some(("newroot", "\0\0\0\0\0\0\0\0"))] {
+            set_live(&live, unusable);
+            assert!(matches!(adapters.credentials.lookup("newroot").await, Err(ProviderError::Unavailable)));
+            assert_eq!(
+                refusal(adapters.policy.snapshot(Some(&long_term("newroot"))).await),
+                PolicyError::unavailable()
+            );
+        }
+    }
+
+    /// Positive — `install` builds the authenticator exactly once, over a provider that reads the
+    /// pair's root source.
+    #[tokio::test]
+    async fn adapters_install_hands_over_the_paired_provider_once() {
+        use rustfs_gateway::{CredentialLookup, CredentialProvider, RegionSet, ServiceBuilder, SigV4Authenticator};
+
+        let live: LiveRoot = Arc::default();
+        set_live(&live, Some((ROOT_ACCESS_KEY, ROOT_SECRET)));
+        let adapters = crate::IamAdapters::new(fake(true, || Ok((None, false))), live_root_source(&live));
+        let mut handed = Vec::new();
+        let _builder = adapters.install(ServiceBuilder::new(), |provider| {
+            handed.push(provider.clone());
+            SigV4Authenticator::new(provider, RegionSet::new(["us-east-1"]).expect("a region"))
+        });
+        assert_eq!(handed.len(), 1);
+        let found = handed[0].lookup(ROOT_ACCESS_KEY).await;
+        assert!(matches!(found, Ok(CredentialLookup::Found(_))));
+    }
+}

crates/gateway-adapter/src/root.rs

@@ -0,0 +1,216 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! The deployment's root credentials, as every adapter in this crate reads them.
+//!
+//! Responsible for: [`RootCredentials`], the validated root key pair plus the key STS session
+//! tokens are signed with; [`RootSource`], the one live handle both the credential provider and
+//! the policy source read on every request; and the effective-secret-length rule every secret in
+//! this crate is held to.
+//! NOT responsible for: where the root keys come from or when they change. The binary implements
+//! [`RootSource`] directly over its live root accessor — a closure is enough — so there is no
+//! second copy of root to keep in step.
+//! Upstream: the binary's runtime credentials. Downstream: `crate::credentials`, `crate::policy`.
+//!
+//! # Why one source
+//!
+//! Legacy reads the current root credentials on every request. Two copies taken at assembly could
+//! disagree, or go stale after a root change, and a key one adapter still treats as root would be
+//! answered with the root secret or judged as the owner. Both adapters therefore hold the same
+//! `Arc<dyn RootSource>`, handed to them together by [`crate::IamAdapters::new`].
+
+use std::sync::Arc;
+
+use rustfs_gateway::Credentials;
+
+/// The shortest effective secret key, in bytes, that any credential in this crate may verify a
+/// signature with. The same bound IAM applies to its own credentials; a signature over a shorter
+/// (or empty) key is one anybody can compute.
+pub const MIN_SECRET_KEY_LEN: usize = 8;
+
+/// The number of bytes of `secret` a signature actually depends on: its length after trailing NUL
+/// bytes are removed.
+///
+/// HMAC pads a key shorter than its block with zero bytes, and SigV4's first key is `"AWS4"`
+/// followed by the secret, so trailing NULs add nothing: eight NUL bytes sign exactly like the
+/// empty secret, and `"abc"` followed by five NULs signs like `"abc"`. A secret of NULs only has
+/// an effective length of zero. An interior NUL is followed by real key bytes, which change the
+/// padded key, so it counts like any other byte.
+pub(crate) fn effective_secret_len(secret: &[u8]) -> usize {
+    secret.iter().rposition(|byte| *byte != 0).map_or(0, |last| last + 1)
+}
+
+/// Whether `secret` may verify a signature: its [`effective_secret_len`] is at least
+/// [`MIN_SECRET_KEY_LEN`]. The one rule for root, IAM credentials and the STS signing key.
+pub(crate) fn is_usable_secret(secret: &[u8]) -> bool {
+    effective_secret_len(secret) >= MIN_SECRET_KEY_LEN
+}
+
+/// Why [`RootCredentials::new`] refused its input. Carries no input text.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum RootCredentialsError {
+    /// The access key id is not one the gateway can hold.
+    InvalidAccessKey,
+    /// The root secret key fails the effective-length rule.
+    WeakSecretKey,
+    /// The STS token-signing key fails the effective-length rule.
+    WeakSigningKey,
+}
+
+impl core::fmt::Display for RootCredentialsError {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        f.write_str(match self {
+            Self::InvalidAccessKey => "root access key is not usable",
+            Self::WeakSecretKey => "root secret key is shorter than the minimum length",
+            Self::WeakSigningKey => "STS token-signing key is shorter than the minimum length",
+        })
+    }
+}
+
+impl std::error::Error for RootCredentialsError {}
+
+/// The root key pair and the STS token-signing key, validated together.
+///
+/// Implements no `Debug`; the secrets are readable only inside this crate.
+pub struct RootCredentials {
+    credentials: Credentials,
+    sts_signing_key: Box<str>,
+}
+
+impl RootCredentials {
+    /// Validates the root credentials and the key IAM signs STS session tokens with (today the
+    /// root secret, `rustfs_iam::manager::get_token_signing_key`; pass it separately so a future
+    /// key separation changes no adapter).
+    ///
+    /// # Errors
+    ///
+    /// [`RootCredentialsError`] when either secret fails the effective-length rule (fewer than
+    /// [`MIN_SECRET_KEY_LEN`] bytes once trailing NUL bytes are removed) or the access key id is not
+    /// one the gateway can hold.
+    pub fn new(access_key: &str, secret_key: &str, sts_signing_key: &str) -> Result<Self, RootCredentialsError> {
+        if !is_usable_secret(secret_key.as_bytes()) {
+            return Err(RootCredentialsError::WeakSecretKey);
+        }
+        if !is_usable_secret(sts_signing_key.as_bytes()) {
+            return Err(RootCredentialsError::WeakSigningKey);
+        }
+        let credentials =
+            Credentials::new(access_key, secret_key.as_bytes()).map_err(|_| RootCredentialsError::InvalidAccessKey)?;
+        Ok(Self {
+            credentials,
+            sts_signing_key: sts_signing_key.into(),
+        })
+    }
+
+    /// The root access key id, a public identifier.
+    #[must_use]
+    pub fn access_key(&self) -> &str {
+        self.credentials.identity().access_key_id()
+    }
+
+    pub(crate) fn credentials(&self) -> &Credentials {
+        &self.credentials
+    }
+
+    pub(crate) fn sts_signing_key(&self) -> &str {
+        &self.sts_signing_key
+    }
+}
+
+/// The live root credentials, read on every request by both adapters.
+///
+/// The binary implements it directly over RustFS's live root accessor, so a root change needs no
+/// second step. Any `Fn() -> Option<Arc<RootCredentials>>` is a source; for example, over the
+/// server context:
+///
+/// ```ignore
+/// let root = move || {
+///     let cred = ctx.action_credentials().get()?;
+///     let signing_key = ctx.token_signing_key()?;
+///     RootCredentials::new(&cred.access_key, &cred.secret_key, &signing_key).ok().map(Arc::new)
+/// };
+/// let adapters = IamAdapters::new(iam, Arc::new(root));
+/// ```
+///
+/// Root credentials that fail validation read as `None`, which both adapters answer as
+/// unavailable.
+pub trait RootSource: Send + Sync + 'static {
+    /// The current root credentials; `None` while they are not initialized, which both adapters
+    /// answer as unavailable rather than guessing.
+    fn current(&self) -> Option<Arc<RootCredentials>>;
+}
+
+impl<F> RootSource for F
+where
+    F: Fn() -> Option<Arc<RootCredentials>> + Send + Sync + 'static,
+{
+    fn current(&self) -> Option<Arc<RootCredentials>> {
+        self()
+    }
+}
+
+/// Test helper: a [`RootSource`] holding fixed root credentials. Production code reads the live
+/// accessor instead, so that there is no copy of root to keep in step.
+#[cfg(test)]
+pub(crate) struct RootCell(Option<Arc<RootCredentials>>);
+
+#[cfg(test)]
+impl RootCell {
+    pub(crate) fn new(root: Option<RootCredentials>) -> Self {
+        Self(root.map(Arc::new))
+    }
+}
+
+#[cfg(test)]
+impl RootSource for RootCell {
+    fn current(&self) -> Option<Arc<RootCredentials>> {
+        self.0.clone()
+    }
+}
+
+#[cfg(test)]
+#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
+mod tests {
+    use super::*;
+
+    /// Negative — an empty or short root secret is refused at construction, so no source can ever
+    /// hand out a root key anybody could sign with; eight bytes is accepted.
+    #[test]
+    fn root_short_secret_is_refused() {
+        for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0", "abcdefg\0"] {
+            let refused = RootCredentials::new("rootadmin", weak, "signing-key-0001");
+            assert!(matches!(refused, Err(RootCredentialsError::WeakSecretKey)), "{} bytes", weak.len());
+        }
+        assert!(RootCredentials::new("rootadmin", "12345678", "signing-key-0001").is_ok());
+    }
+
+    /// Positive — an interior NUL is followed by real key bytes and counts: `"abc\0defg"` has an
+    /// effective length of eight and is accepted, as is the same secret with trailing NULs.
+    #[test]
+    fn root_interior_nul_counts_toward_length() {
+        for secret in ["abc\0defg", "abc\0defg\0\0"] {
+            assert!(RootCredentials::new("rootadmin", secret, "signing-key-0001").is_ok(), "{secret:?}");
+            assert!(RootCredentials::new("rootadmin", "root-secret-key-0001", secret).is_ok(), "{secret:?}");
+        }
+    }
+
+    /// Negative — an empty or short STS signing key is refused at construction.
+    #[test]
+    fn root_short_signing_key_is_refused() {
+        for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0"] {
+            let refused = RootCredentials::new("rootadmin", "root-secret-key-0001", weak);
+            assert!(matches!(refused, Err(RootCredentialsError::WeakSigningKey)), "{} bytes", weak.len());
+        }
+    }
+}

crates/gateway-adapter/src/trace.rs

@@ -0,0 +1,230 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! The request id RustFS's outer layer minted, handed to the gateway as its own when that is safe.
+//!
+//! Responsible for: reading the id from the context the outer layer put in the request's
+//! extensions ([`RequestIdCarrier`]) and, when it is inside the gateway's identifier alphabet and
+//! length, inserting it as the gateway's [`HostRequestId`].
+//! NOT responsible for: minting any id, refusing any request (nothing here can fail a request),
+//! choosing the response headers (the gateway writes `x-amz-request-id` from the id it was
+//! handed, and RustFS's outer layer overwrites that header on S3 paths), or the layer that mints
+//! (`rustfs/src/server/layer.rs`). It never reads a header itself.
+//! Upstream: RustFS's `RequestContext`, which the binary makes a [`RequestIdCarrier`]. Downstream:
+//! the gateway service, which reads the [`HostRequestId`] extension once, at the top.
+//!
+//! # Whose id it is
+//!
+//! On S3 paths the outer layer mints a server-owned UUID. On admin, STS and console paths the
+//! context copies the caller's `x-request-id` (or `x-amz-request-id`), as legacy does, so there
+//! the id can be caller-chosen. Either way the id is attached only when it is inert, that is 1 to
+//! 64 bytes of ASCII letters, digits, `-`, `.`, `_` and `~`, which is safe in a header, an XML
+//! document and a log line. Anything else is not attached and the request is **forwarded
+//! unchanged**: the gateway then mints its own id. No request is ever refused because of its id.
+//!
+//! # Known difference when nothing is attached
+//!
+//! The gateway answers with the id it minted (error body `<RequestId>`, its events and audit
+//! records) while, on S3 paths, the outer layer overwrites the response header with RustFS's id.
+//! So for such a request the header and the error body name different ids. That needs a caller-
+//! chosen id outside the alphabet, or a request that never passed the outer layer.
+//!
+//! # Why this is not a `TraceSource`
+//!
+//! The gateway's `TraceSource::mint` takes no request, so a source cannot read the id RustFS
+//! already minted; the gateway's documented path for a host that identifies its own requests is
+//! the [`HostRequestId`] extension, and this module fills it.
+//!
+//! # Why a carrier trait
+//!
+//! `RequestContext` lives in the `rustfs` binary crate, which a library crate cannot depend on.
+//! The binary implements [`RequestIdCarrier`] for it, and this module looks it up in the
+//! extensions by that type.
+
+use rustfs_gateway::HostRequestId;
+
+/// A type in the request extensions that holds the id the outer layer minted or propagated.
+pub trait RequestIdCarrier: Send + Sync + 'static {
+    /// The canonical request id: the value RustFS logs and the response should carry.
+    fn request_id(&self) -> &str;
+}
+
+/// What [`attach_host_request_id`] did. Ignoring it is harmless: every outcome forwards the request.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum AttachOutcome {
+    /// The carrier's id was inserted as the request's [`HostRequestId`].
+    Attached,
+    /// The request carries no context from the outer layer; nothing was inserted.
+    NoContext,
+    /// The carrier's id is empty, too long, or outside the gateway's alphabet; nothing was
+    /// inserted and the id was not repaired.
+    UnusableId,
+}
+
+/// The gateway's [`HostRequestId`] for the id `carrier` holds, or `None` when it is not inert.
+#[must_use]
+pub fn host_request_id<C: RequestIdCarrier + ?Sized>(carrier: &C) -> Option<HostRequestId> {
+    HostRequestId::new(carrier.request_id()).ok()
+}
+
+/// Reads the `C` the outer layer put in `request`'s extensions and, when its id is inert, inserts
+/// it as the request's [`HostRequestId`].
+///
+/// Never refuses the request: the caller forwards it whatever the outcome.
+pub fn attach_host_request_id<C: RequestIdCarrier, B>(request: &mut http::Request<B>) -> AttachOutcome {
+    let Some(carrier) = request.extensions().get::<C>() else {
+        return AttachOutcome::NoContext;
+    };
+    let Some(id) = host_request_id(carrier) else {
+        return AttachOutcome::UnusableId;
+    };
+    request.extensions_mut().insert(id);
+    AttachOutcome::Attached
+}
+
+#[cfg(test)]
+#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
+mod tests {
+    use super::*;
+
+    #[derive(Clone)]
+    struct Ctx(String);
+
+    impl RequestIdCarrier for Ctx {
+        fn request_id(&self) -> &str {
+            &self.0
+        }
+    }
+
+    const UUID: &str = "7c9e6679-7425-40de-944b-e07fc1f90ae7";
+
+    fn request_with(id: &str) -> http::Request<()> {
+        let mut request = http::Request::new(());
+        request.extensions_mut().insert(Ctx(id.to_owned()));
+        request
+    }
+
+    fn handed_over(request: &http::Request<()>) -> Option<String> {
+        request
+            .extensions()
+            .get::<HostRequestId>()
+            .map(|id| id.request_id().as_str().to_owned())
+    }
+
+    fn attach(request: &mut http::Request<()>) -> AttachOutcome {
+        attach_host_request_id::<Ctx, _>(request)
+    }
+
+    /// Positive: the id RustFS minted is the id the gateway is handed, byte for byte.
+    #[test]
+    fn the_minted_id_is_handed_over_unchanged() {
+        let mut request = request_with(UUID);
+        assert_eq!(attach(&mut request), AttachOutcome::Attached);
+        assert_eq!(handed_over(&request).as_deref(), Some(UUID));
+    }
+
+    /// Positive: every shape RustFS mints is accepted, including its two fallbacks.
+    #[test]
+    fn every_shape_rustfs_mints_is_accepted() {
+        for id in [UUID, "trace-0123456789abcdef", "req-0123456789abcdef"] {
+            let mut request = request_with(id);
+            assert_eq!(attach(&mut request), AttachOutcome::Attached, "{id}");
+            assert_eq!(handed_over(&request).as_deref(), Some(id));
+        }
+    }
+
+    /// Positive: a caller-supplied id that is inert (admin and STS paths copy it) is attached, and
+    /// the longest allowed one is accepted.
+    #[test]
+    fn an_inert_caller_supplied_id_is_attached() {
+        let longest = "a".repeat(64);
+        for id in ["abc", "my.trace_id~1-2", longest.as_str()] {
+            let mut request = request_with(id);
+            assert_eq!(attach(&mut request), AttachOutcome::Attached, "{id}");
+            assert_eq!(handed_over(&request).as_deref(), Some(id));
+        }
+    }
+
+    /// Positive: two requests keep their own ids; nothing is shared or minted between them.
+    #[test]
+    fn each_request_keeps_its_own_id() {
+        let mut first = request_with("first-id");
+        let mut second = request_with("second-id");
+        attach(&mut first);
+        attach(&mut second);
+        assert_eq!(handed_over(&first).as_deref(), Some("first-id"));
+        assert_eq!(handed_over(&second).as_deref(), Some("second-id"));
+    }
+
+    /// Positive: with both a carrier and an `x-amz-request-id` header, the carrier's id wins.
+    #[test]
+    fn the_carrier_wins_over_a_request_id_header() {
+        let mut request = http::Request::builder()
+            .header("x-amz-request-id", "caller-chosen")
+            .header("x-request-id", "caller-chosen")
+            .body(())
+            .expect("a request");
+        request.extensions_mut().insert(Ctx(UUID.to_owned()));
+        assert_eq!(attach(&mut request), AttachOutcome::Attached);
+        assert_eq!(handed_over(&request).as_deref(), Some(UUID));
+    }
+
+    /// Negative: a request the outer layer never saw gets no id, and the request is not refused.
+    #[test]
+    fn a_missing_context_hands_over_nothing() {
+        let mut request = http::Request::new(());
+        assert_eq!(attach(&mut request), AttachOutcome::NoContext);
+        assert_eq!(handed_over(&request), None);
+    }
+
+    /// Negative: a header named like the id is not the id; only the context is read.
+    #[test]
+    fn a_request_id_header_alone_is_not_read() {
+        let mut request = http::Request::builder()
+            .header("x-amz-request-id", "caller-chosen")
+            .header("x-request-id", "caller-chosen")
+            .body(())
+            .expect("a request");
+        assert_eq!(attach(&mut request), AttachOutcome::NoContext);
+        assert_eq!(handed_over(&request), None);
+    }
+
+    /// Negative: an id outside the alphabet or the length is never attached or repaired, and the
+    /// outcome says so rather than failing the request.
+    #[test]
+    fn an_id_the_gateway_cannot_carry_is_not_attached() {
+        let too_long = "a".repeat(65);
+        for id in [
+            "",
+            "has space",
+            "line\r\nbreak",
+            "<RequestId>",
+            "quo\"te",
+            "Root=1-5759e988",
+            too_long.as_str(),
+        ] {
+            let mut request = request_with(id);
+            assert_eq!(attach(&mut request), AttachOutcome::UnusableId, "{id:?}");
+            assert_eq!(handed_over(&request), None, "{id:?}");
+        }
+    }
+
+    /// Negative: the unusable id is not repaired into a shorter or cleaned-up one.
+    #[test]
+    fn host_request_id_does_not_repair() {
+        assert!(host_request_id(&Ctx("has space".to_owned())).is_none());
+        assert!(host_request_id(&Ctx(format!("{UUID} "))).is_none());
+        assert!(host_request_id(&Ctx(UUID.to_owned())).is_some());
+    }
+}

crates/heal/src/heal/resume/bucket_pass.rs

@@ -385,9 +385,15 @@ mod tests {
                 .is_err(),
             "a stale worker cannot mutate a recreated bucket"
         );
+        // Name targets as the store's endpoints render them (`C:/...` on Windows),
+        // not as raw OS paths.
         let targets = directories
             .iter()
-            .map(|dir| dir.path().to_string_lossy().into_owned())
+            .map(|dir| {
+                crate::heal::Endpoint::try_from(dir.path().to_str().expect("UTF-8 temp disk path"))
+                    .expect("temp disk endpoint")
+                    .to_string()
+            })
             .collect::<Vec<_>>();
         assert!(
             storage

crates/kms/src/api_types.rs

@@ -917,9 +917,10 @@ mod tests {
 
     #[test]
     fn test_configure_request_development_defaults_require_opt_in() {
+        let key_dir = std::env::temp_dir().join("kms-key-dir");
         let local_raw = serde_json::json!({
             "backend_type": "local",
-            "key_dir": "/tmp/kms-key-dir"
+            "key_dir": key_dir
         });
         let request: ConfigureKmsRequest = serde_json::from_value(local_raw).expect("local request should deserialize");
         let config = request.to_kms_config();
@@ -927,7 +928,7 @@ mod tests {
 
         let local_opt_in_raw = serde_json::json!({
             "backend_type": "local",
-            "key_dir": "/tmp/kms-key-dir",
+            "key_dir": key_dir,
             "allow_insecure_dev_defaults": true
         });
         let request: ConfigureKmsRequest = serde_json::from_value(local_opt_in_raw).expect("local request should deserialize");

crates/kms/src/config.rs

@@ -1608,10 +1608,14 @@ mod tests {
         assert!(config.validate().is_err());
         assert!(config.with_insecure_development_defaults().validate().is_ok());
 
+        #[cfg(windows)]
+        let production_key_dir = PathBuf::from(r"C:\rustfs\kms");
+        #[cfg(not(windows))]
+        let production_key_dir = PathBuf::from("/var/lib/rustfs/kms");
         let production_config = KmsConfig {
             backend: KmsBackend::Local,
             backend_config: BackendConfig::Local(LocalConfig {
-                key_dir: PathBuf::from("/var/lib/rustfs/kms"),
+                key_dir: production_key_dir,
                 master_key: Some("production-master-key".to_string()),
                 file_permissions: Some(0o600),
             }),

crates/kms/tests/behavior_lifecycle.rs

@@ -336,7 +336,11 @@ async fn invalid_configurations_are_rejected_before_anything_starts() {
     assert_configuration_error(relative.validate(), "must be an absolute path");
 
     // Production-shaped Local config: no master key is refused without opt-in.
-    let production_without_master_key = KmsConfig::local("/var/lib/rustfs/kms-behavior".into());
+    #[cfg(windows)]
+    let production_key_dir = r"C:\rustfs\kms-behavior";
+    #[cfg(not(windows))]
+    let production_key_dir = "/var/lib/rustfs/kms-behavior";
+    let production_without_master_key = KmsConfig::local(production_key_dir.into());
     assert_configuration_error(production_without_master_key.validate(), "requires a master key");
 
     // ...and so is a key directory under the process temp dir.

crates/lock/src/client/local.rs

@@ -143,6 +143,49 @@ struct LocalGuardEntry {
     ttl: Duration,
 }
 
+/// The shared acquisition algorithm reports facts before either API formats them.
+enum LocalAcquireFailure {
+    Timeout,
+    Conflict {
+        current_owner: Arc<str>,
+        current_mode: crate::LockMode,
+    },
+    InvalidResource,
+}
+
+impl LocalAcquireFailure {
+    fn into_response(self, request: &LockRequest) -> LockResponse {
+        match self {
+            Self::Timeout => LockResponse::failure("Lock acquisition timeout", request.acquire_timeout),
+            Self::Conflict {
+                current_owner,
+                current_mode,
+            } => LockResponse::failure(
+                format!("Lock conflict: resource held by {current_owner} in {current_mode:?} mode"),
+                Duration::ZERO,
+            ),
+            Self::InvalidResource => {
+                LockResponse::failure("Lock id resource does not match the requested resource", Duration::ZERO)
+            }
+        }
+    }
+
+    fn into_batch_outcome(self, request: &LockRequest) -> super::BatchAcquireOutcome {
+        use super::BatchAcquireOutcome;
+        match self {
+            Self::Timeout => BatchAcquireOutcome::RetryableNoGrant {
+                error: crate::LockError::timeout(request.resource.to_string(), request.acquire_timeout),
+            },
+            Self::Conflict { current_owner, .. } => BatchAcquireOutcome::RetryableNoGrant {
+                error: crate::LockError::already_locked(request.resource.to_string(), current_owner.to_string()),
+            },
+            Self::InvalidResource => BatchAcquireOutcome::TerminalNoGrant {
+                error: crate::LockError::invalid_handle(request.lock_id.to_string()),
+            },
+        }
+    }
+}
+
 impl LocalGuardEntry {
     fn new(guard: FastLockGuard, ttl: Duration) -> Self {
         let acquired_at = SystemTime::now();
@@ -171,6 +214,158 @@ impl LocalGuardEntry {
 }
 
 impl LocalClient {
+    async fn acquire_lock_until(
+        &self,
+        request: &LockRequest,
+        acquire_deadline: Instant,
+    ) -> std::result::Result<LockInfo, LocalAcquireFailure> {
+        let started = Instant::now();
+        self.ensure_reaper();
+        if request.lock_id.resource != request.resource {
+            return Err(LocalAcquireFailure::InvalidResource);
+        }
+        let lock_manager = self.get_lock_manager();
+        let initial_remaining = acquire_deadline.saturating_duration_since(Instant::now());
+        let reclaimed_before_acquire = if initial_remaining.is_zero() {
+            0
+        } else {
+            timeout(initial_remaining, self.reclaim_expired_guards_for_resource(&request.resource))
+                .await
+                .unwrap_or(0)
+        };
+        let reclaim_elapsed = started.elapsed();
+
+        let build_lock_request = |acquire_timeout| match request.lock_type {
+            LockType::Exclusive => crate::ObjectLockRequest::new_write(request.resource.clone(), request.owner.clone())
+                .with_acquire_timeout(acquire_timeout),
+            LockType::Shared => crate::ObjectLockRequest::new_read(request.resource.clone(), request.owner.clone())
+                .with_acquire_timeout(acquire_timeout),
+        };
+
+        let mut retried_after_reclaim = reclaimed_before_acquire > 0;
+        loop {
+            let remaining = acquire_deadline.saturating_duration_since(Instant::now());
+            if remaining.is_zero() {
+                return Err(LocalAcquireFailure::Timeout);
+            }
+            let manager_started = Instant::now();
+            // The backend uses its own relative wait; enforce the caller's
+            // absolute budget across that entire future as well. Dropping an
+            // uncompleted manager acquisition releases its publication ticket.
+            let acquired = match timeout_at(acquire_deadline, lock_manager.acquire_lock(build_lock_request(remaining))).await {
+                Ok(acquired) => acquired,
+                Err(_) => return Err(LocalAcquireFailure::Timeout),
+            };
+            match acquired {
+                Ok(guard) => {
+                    let manager_elapsed = manager_started.elapsed();
+                    let registration_started = Instant::now();
+                    let physical_guard_id = guard.guard_id();
+                    let lock_id = request.lock_id.clone();
+                    let entry = LocalGuardEntry::new(guard, request.ttl);
+                    let acquired_at = entry.acquired_at;
+                    let expires_at = entry.expires_at;
+
+                    let (replaced_entry, registration_wait, registration_mutation, registry_entries, registry_capacity) = {
+                        let shard = self.get_shard(&lock_id);
+                        let Ok(mut guards) = timeout_at(acquire_deadline, shard.write()).await else {
+                            return Err(LocalAcquireFailure::Timeout);
+                        };
+                        // A ready table cannot publish a grant after the peer
+                        // budget expired while its acquisition task was idle.
+                        if Instant::now() >= acquire_deadline {
+                            drop(guards);
+                            return Err(LocalAcquireFailure::Timeout);
+                        }
+                        let registration_wait = registration_started.elapsed();
+                        let mutation_started = Instant::now();
+                        let replaced_entry = guards.insert(lock_id.clone(), entry);
+                        (
+                            replaced_entry,
+                            registration_wait,
+                            mutation_started.elapsed(),
+                            guards.len(),
+                            guards.capacity(),
+                        )
+                    };
+                    // Repeated shared requests can replace an entry. Releasing
+                    // its physical guard must not reenter locks under the table.
+                    drop(replaced_entry);
+
+                    let registration_elapsed = registration_started.elapsed();
+                    if started.elapsed() >= Duration::from_millis(100) {
+                        tracing::trace!(
+                            target: "rustfs_lock::namespace_owner",
+                            event = EVENT_NAMESPACE_LOCK_ACQUIRE,
+                            component = LOG_COMPONENT_LOCK,
+                            subsystem = LOG_SUBSYSTEM_NAMESPACE,
+                            state = "local_completed",
+                            resource = %request.resource,
+                            lock_id = %lock_id,
+                            mode = ?request.lock_type,
+                            physical_guard_id,
+                            reclaim_ms = reclaim_elapsed.as_millis(),
+                            manager_ms = manager_elapsed.as_millis(),
+                            registration_ms = registration_elapsed.as_millis(),
+                            registration_wait_ms = registration_wait.as_millis(),
+                            registration_mutation_ms = registration_mutation.as_millis(),
+                            registry_entries,
+                            registry_capacity,
+                            acquire_ms = started.elapsed().as_millis(),
+                            "Namespace local lock acquisition completed slowly"
+                        );
+                    }
+
+                    let lock_info = LockInfo {
+                        id: lock_id,
+                        resource: request.resource.clone(),
+                        lock_type: request.lock_type,
+                        status: crate::types::LockStatus::Acquired,
+                        owner: request.owner.clone(),
+                        acquired_at,
+                        expires_at,
+                        last_refreshed: acquired_at,
+                        metadata: request.metadata.clone(),
+                        priority: request.priority,
+                        wait_start_time: None,
+                    };
+                    return Ok(lock_info);
+                }
+                Err(crate::fast_lock::LockResult::Timeout) => {
+                    if !retried_after_reclaim {
+                        let remaining = acquire_deadline.saturating_duration_since(Instant::now());
+                        if !remaining.is_zero() {
+                            let _ = timeout(remaining, self.reclaim_expired_guards_for_resource(&request.resource)).await;
+                        }
+                        retried_after_reclaim = true;
+                        continue;
+                    }
+                    return Err(LocalAcquireFailure::Timeout);
+                }
+                Err(crate::fast_lock::LockResult::Conflict {
+                    current_owner,
+                    current_mode,
+                }) => {
+                    if !retried_after_reclaim {
+                        let remaining = acquire_deadline.saturating_duration_since(Instant::now());
+                        if !remaining.is_zero() {
+                            let _ = timeout(remaining, self.reclaim_expired_guards_for_resource(&request.resource)).await;
+                        }
+                        retried_after_reclaim = true;
+                        continue;
+                    }
+                    return Err(LocalAcquireFailure::Conflict {
+                        current_owner,
+                        current_mode,
+                    });
+                }
+                Err(crate::fast_lock::LockResult::Acquired) => {
+                    unreachable!("Acquired should not be an error")
+                }
+            }
+        }
+    }
+
     /// Create new local client with default shard count
     pub fn new() -> Self {
         Self::with_shard_count(DEFAULT_GUARD_SHARD_COUNT)
@@ -320,149 +515,29 @@ impl Default for LocalClient {
 impl LockClient for LocalClient {
     async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
         let started = Instant::now();
-        self.ensure_reaper();
-        if request.lock_id.resource != request.resource {
-            return Ok(LockResponse::failure(
-                "Lock id resource does not match the requested resource",
-                Duration::ZERO,
-            ));
-        }
-        let acquire_deadline = Instant::now()
-            .checked_add(request.acquire_timeout)
-            .unwrap_or_else(Instant::now);
-        let lock_manager = self.get_lock_manager();
-        let initial_remaining = acquire_deadline.saturating_duration_since(Instant::now());
-        let reclaimed_before_acquire = if initial_remaining.is_zero() {
-            0
-        } else {
-            timeout(initial_remaining, self.reclaim_expired_guards_for_resource(&request.resource))
-                .await
-                .unwrap_or(0)
-        };
-        let reclaim_elapsed = started.elapsed();
+        let deadline = started.checked_add(request.acquire_timeout).unwrap_or(started);
+        Ok(match self.acquire_lock_until(request, deadline).await {
+            Ok(info) => LockResponse::success(info, Duration::ZERO),
+            Err(failure) => failure.into_response(request),
+        })
+    }
 
-        let build_lock_request = |acquire_timeout| match request.lock_type {
-            LockType::Exclusive => crate::ObjectLockRequest::new_write(request.resource.clone(), request.owner.clone())
-                .with_acquire_timeout(acquire_timeout),
-            LockType::Shared => crate::ObjectLockRequest::new_read(request.resource.clone(), request.owner.clone())
-                .with_acquire_timeout(acquire_timeout),
-        };
-
-        let mut retried_after_reclaim = reclaimed_before_acquire > 0;
-        loop {
-            let remaining = acquire_deadline.saturating_duration_since(Instant::now());
-            if remaining.is_zero() {
-                return Ok(LockResponse::failure("Lock acquisition timeout", request.acquire_timeout));
-            }
-            let manager_started = Instant::now();
-            match lock_manager.acquire_lock(build_lock_request(remaining)).await {
-                Ok(guard) => {
-                    let manager_elapsed = manager_started.elapsed();
-                    let registration_started = Instant::now();
-                    let physical_guard_id = guard.guard_id();
-                    let lock_id = request.lock_id.clone();
-                    let entry = LocalGuardEntry::new(guard, request.ttl);
-                    let acquired_at = entry.acquired_at;
-                    let expires_at = entry.expires_at;
-
-                    let (replaced_entry, registration_wait, registration_mutation, registry_entries, registry_capacity) = {
-                        let shard = self.get_shard(&lock_id);
-                        let Ok(mut guards) = timeout_at(acquire_deadline, shard.write()).await else {
-                            return Ok(LockResponse::failure("Lock acquisition timeout", request.acquire_timeout));
-                        };
-                        // A ready table cannot publish a grant after the peer
-                        // budget expired while its acquisition task was idle.
-                        if Instant::now() >= acquire_deadline {
-                            drop(guards);
-                            return Ok(LockResponse::failure("Lock acquisition timeout", request.acquire_timeout));
-                        }
-                        let registration_wait = registration_started.elapsed();
-                        let mutation_started = Instant::now();
-                        let replaced_entry = guards.insert(lock_id.clone(), entry);
-                        (
-                            replaced_entry,
-                            registration_wait,
-                            mutation_started.elapsed(),
-                            guards.len(),
-                            guards.capacity(),
-                        )
-                    };
-                    // Repeated shared requests can replace an entry. Releasing
-                    // its physical guard must not reenter locks under the table.
-                    drop(replaced_entry);
-
-                    let registration_elapsed = registration_started.elapsed();
-                    if started.elapsed() >= Duration::from_millis(100) {
-                        tracing::trace!(
-                            target: "rustfs_lock::namespace_owner",
-                            event = EVENT_NAMESPACE_LOCK_ACQUIRE,
-                            component = LOG_COMPONENT_LOCK,
-                            subsystem = LOG_SUBSYSTEM_NAMESPACE,
-                            state = "local_completed",
-                            resource = %request.resource,
-                            lock_id = %lock_id,
-                            mode = ?request.lock_type,
-                            physical_guard_id,
-                            reclaim_ms = reclaim_elapsed.as_millis(),
-                            manager_ms = manager_elapsed.as_millis(),
-                            registration_ms = registration_elapsed.as_millis(),
-                            registration_wait_ms = registration_wait.as_millis(),
-                            registration_mutation_ms = registration_mutation.as_millis(),
-                            registry_entries,
-                            registry_capacity,
-                            acquire_ms = started.elapsed().as_millis(),
-                            "Namespace local lock acquisition completed slowly"
-                        );
-                    }
-
-                    let lock_info = LockInfo {
-                        id: lock_id,
-                        resource: request.resource.clone(),
-                        lock_type: request.lock_type,
-                        status: crate::types::LockStatus::Acquired,
-                        owner: request.owner.clone(),
-                        acquired_at,
-                        expires_at,
-                        last_refreshed: acquired_at,
-                        metadata: request.metadata.clone(),
-                        priority: request.priority,
-                        wait_start_time: None,
-                    };
-                    return Ok(LockResponse::success(lock_info, Duration::ZERO));
-                }
-                Err(crate::fast_lock::LockResult::Timeout) => {
-                    if !retried_after_reclaim {
-                        let remaining = acquire_deadline.saturating_duration_since(Instant::now());
-                        if !remaining.is_zero() {
-                            let _ = timeout(remaining, self.reclaim_expired_guards_for_resource(&request.resource)).await;
-                        }
-                        retried_after_reclaim = true;
-                        continue;
-                    }
-                    return Ok(LockResponse::failure("Lock acquisition timeout", request.acquire_timeout));
-                }
-                Err(crate::fast_lock::LockResult::Conflict {
-                    current_owner,
-                    current_mode,
-                }) => {
-                    if !retried_after_reclaim {
-                        let remaining = acquire_deadline.saturating_duration_since(Instant::now());
-                        if !remaining.is_zero() {
-                            let _ = timeout(remaining, self.reclaim_expired_guards_for_resource(&request.resource)).await;
-                        }
-                        retried_after_reclaim = true;
-                        continue;
-                    }
-                    return Ok(LockResponse::failure(
-                        format!("Lock conflict: resource held by {current_owner} in {current_mode:?} mode"),
-                        Duration::ZERO,
-                    ));
-                }
-                Err(crate::fast_lock::LockResult::Acquired) => {
-                    unreachable!("Acquired should not be an error")
+    async fn acquire_locks_batch_until(&self, requests: &[LockRequest], deadline: Instant) -> Vec<super::BatchAcquireOutcome> {
+        // Every item shares the entry time, including items first polled later.
+        let started = Instant::now();
+        futures::future::join_all(requests.iter().map(|request| {
+            let item_deadline = deadline.min(started.checked_add(request.acquire_timeout).unwrap_or(started));
+            async move {
+                match self.acquire_lock_until(request, item_deadline).await {
+                    Ok(info) => super::BatchAcquireOutcome::Granted {
+                        lock_info: Some(Box::new(info)),
+                        wait_time: Duration::ZERO,
+                    },
+                    Err(failure) => failure.into_batch_outcome(request),
                 }
             }
-        }
+        }))
+        .await
     }
 
     async fn release(&self, lock_id: &LockId) -> Result<bool> {
@@ -609,6 +684,167 @@ mod tests {
     use futures::FutureExt;
     use std::task::Poll;
 
+    #[tokio::test(start_paused = true)]
+    async fn typed_batch_preserves_free_key_grants_beside_a_contended_key() {
+        let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new()));
+        let hot = crate::ObjectKey::new("bucket", "hot");
+        let holder = request(hot.clone(), "holder", Duration::from_secs(30));
+        assert!(client.acquire_lock(&holder).await.expect("hold contended key").success);
+        let requests = [
+            request(crate::ObjectKey::new("bucket", "free"), "batch", Duration::from_secs(30)),
+            request(hot, "batch", Duration::from_secs(30)),
+        ];
+        let outcomes = client
+            .acquire_locks_batch_until(&requests, Instant::now() + Duration::from_secs(1))
+            .await;
+        assert_eq!(outcomes.len(), 2);
+        assert!(matches!(outcomes[0], super::super::BatchAcquireOutcome::Granted { .. }));
+        assert!(matches!(outcomes[1], super::super::BatchAcquireOutcome::RetryableNoGrant { .. }));
+        assert!(
+            client
+                .check_status(&requests[0].lock_id)
+                .await
+                .expect("check free grant")
+                .is_some()
+        );
+        assert!(
+            client
+                .check_status(&requests[1].lock_id)
+                .await
+                .expect("check rejected attempt")
+                .is_none()
+        );
+        assert!(client.release(&holder.lock_id).await.expect("release holder"));
+        assert!(client.release(&requests[0].lock_id).await.expect("release free key"));
+        assert!(client.list_lock_leases().await.is_empty());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn typed_batch_manager_wait_obeys_absolute_deadline_without_releasing_holder() {
+        let manager = Arc::new(GlobalLockManager::new());
+        let client = LocalClient::with_manager(manager.clone());
+        let resource = crate::ObjectKey::new("bucket", "manager-deadline");
+        let holder = manager
+            .acquire_lock(crate::ObjectLockRequest::new_write(resource.clone(), "holder"))
+            .await
+            .expect("hold the physical lock independently of a backend lease");
+        let requests = [request(resource.clone(), "batch", Duration::from_secs(30)).with_acquire_timeout(Duration::from_secs(5))];
+        let started = Instant::now();
+        let deadline = started + Duration::from_millis(100);
+        let outcomes = timeout_at(deadline + Duration::from_millis(1), client.acquire_locks_batch_until(&requests, deadline))
+            .await
+            .expect("the manager wait must finish at the absolute caller deadline");
+        assert!(started.elapsed() <= Duration::from_millis(100));
+        assert!(matches!(
+            outcomes.as_slice(),
+            [super::super::BatchAcquireOutcome::RetryableNoGrant { .. }]
+        ));
+        assert!(client.list_lock_leases().await.is_empty(), "no failed attempt is registered");
+        assert_eq!(
+            manager
+                .as_fast_lock_manager()
+                .expect("enabled manager")
+                .get_shard(&resource)
+                .active_guard_count(),
+            1,
+            "the original holder is never released by a timed-out attempt"
+        );
+        drop(holder);
+        let successor = manager
+            .acquire_lock(crate::ObjectLockRequest::new_write(resource.clone(), "successor"))
+            .await
+            .expect("no cancelled manager waiter may take the released lock later");
+        drop(successor);
+        assert_eq!(
+            manager
+                .as_fast_lock_manager()
+                .expect("enabled manager")
+                .get_shard(&resource)
+                .active_guard_count(),
+            0
+        );
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn typed_batch_rejects_late_poll_without_resetting_the_caller_deadline() {
+        let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new()));
+        let requests = [
+            request(crate::ObjectKey::new("bucket", "late"), "batch", Duration::from_secs(30))
+                .with_acquire_timeout(Duration::from_secs(5)),
+        ];
+        let deadline = Instant::now() + Duration::from_millis(20);
+        let acquire = client.acquire_locks_batch_until(&requests, deadline);
+        tokio::time::advance(Duration::from_millis(21)).await;
+        assert!(matches!(
+            acquire.await.as_slice(),
+            [super::super::BatchAcquireOutcome::RetryableNoGrant {
+                error: crate::LockError::Timeout { .. }
+            }]
+        ));
+        assert!(client.list_lock_leases().await.is_empty(), "an expired item must never register a grant");
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn typed_batch_registration_timeout_drops_the_unpublished_physical_guard() {
+        let manager = Arc::new(GlobalLockManager::new());
+        let client = LocalClient::with_manager(manager.clone());
+        client.reaper_started.store(true, Ordering::Release);
+        let resource = crate::ObjectKey::new("bucket", "typed-registration-timeout");
+        let blocker = manager
+            .acquire_lock(crate::ObjectLockRequest::new_write(resource.clone(), "holder"))
+            .await
+            .expect("hold physical lock until preflight completes");
+        let requests = [request(resource.clone(), "batch", Duration::from_secs(30)).with_acquire_timeout(Duration::from_secs(5))];
+        let deadline = Instant::now() + Duration::from_millis(20);
+        let mut acquire = Box::pin(tokio::task::unconstrained(client.acquire_locks_batch_until(&requests, deadline)));
+        assert!(matches!(futures::poll!(&mut acquire), Poll::Pending));
+        let (release_table, table_owner) = hold_guard_shard(client.get_shard(&requests[0].lock_id).clone());
+        drop(blocker);
+        assert!(matches!(futures::poll!(&mut acquire), Poll::Pending));
+        assert_eq!(
+            manager
+                .as_fast_lock_manager()
+                .expect("enabled manager")
+                .get_shard(&resource)
+                .active_guard_count(),
+            1
+        );
+        tokio::time::advance(Duration::from_millis(21)).await;
+        let result = acquire.await;
+        assert!(matches!(result.as_slice(), [super::super::BatchAcquireOutcome::RetryableNoGrant { .. }]));
+        assert_eq!(
+            manager
+                .as_fast_lock_manager()
+                .expect("enabled manager")
+                .get_shard(&resource)
+                .active_guard_count(),
+            0,
+            "NoGrant must only be returned after the unpublished physical guard is dropped"
+        );
+        release_table.send(()).expect("release table");
+        table_owner.join().expect("table owner finishes");
+        assert!(client.list_lock_leases().await.is_empty());
+    }
+
+    #[test]
+    fn legacy_acquisition_messages_remain_compatible() {
+        let request = request(crate::ObjectKey::new("bucket", "legacy"), "owner", Duration::from_secs(30));
+        assert_eq!(
+            LocalAcquireFailure::Timeout.into_response(&request).error.as_deref(),
+            Some("Lock acquisition timeout")
+        );
+        assert_eq!(
+            LocalAcquireFailure::Conflict {
+                current_owner: Arc::from("holder"),
+                current_mode: crate::LockMode::Exclusive
+            }
+            .into_response(&request)
+            .error
+            .as_deref(),
+            Some("Lock conflict: resource held by holder in Exclusive mode")
+        );
+    }
+
     fn request(resource: crate::ObjectKey, owner: &str, ttl: Duration) -> LockRequest {
         LockRequest::new(resource, LockType::Exclusive, owner)
             .with_ttl(ttl)

crates/lock/src/client/mod.rs

@@ -15,10 +15,49 @@
 pub mod local;
 // pub mod remote;
 
-use crate::{LockId, LockInfo, LockLeaseInfo, LockRequest, LockResponse, LockStats, Result};
+use crate::{LockError, LockId, LockInfo, LockLeaseInfo, LockRequest, LockResponse, LockStats, Result};
 use async_trait::async_trait;
 use futures::future::join_all;
 use std::sync::Arc;
+use std::time::Duration;
+use tokio::time::Instant;
+
+/// A batch peer's result for one request, in the original request order.
+#[derive(Clone, Debug)]
+pub enum BatchAcquireOutcome {
+    /// The peer confirmed a grant. Missing legacy metadata uses the requested ID.
+    Granted {
+        lock_info: Option<Box<LockInfo>>,
+        wait_time: Duration,
+    },
+    /// This attempt cannot grant later, and another attempt may succeed.
+    RetryableNoGrant { error: LockError },
+    /// This attempt cannot grant later, and this peer must not be retried.
+    TerminalNoGrant { error: LockError },
+    /// A grant may exist or arrive later. Stop using this peer for this key.
+    Ambiguous {
+        error: LockError,
+        /// The client already retains exact-ID cleanup until the attempt settles.
+        /// Otherwise the coordinator must retain uncertain cleanup ownership.
+        cleanup_owned_by_client: bool,
+    },
+}
+
+impl BatchAcquireOutcome {
+    fn from_legacy_response(response: LockResponse) -> Self {
+        if response.success {
+            Self::Granted {
+                lock_info: response.lock_info.map(Box::new),
+                wait_time: response.wait_time,
+            }
+        } else {
+            Self::Ambiguous {
+                error: LockError::internal(response.error.unwrap_or_else(|| "Legacy batch acquisition failed".to_owned())),
+                cleanup_owned_by_client: false,
+            }
+        }
+    }
+}
 
 /// Outcome of cleaning up a lock whose acquisition has already completed.
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -45,6 +84,45 @@ pub trait LockClient: Send + Sync + std::fmt::Debug {
             .collect::<Result<Vec<_>>>()?)
     }
 
+    /// Acquire a batch within the caller's absolute deadline, preserving per-key outcomes.
+    ///
+    /// Legacy clients cannot prove that failed or missing responses did not grant.
+    /// Their calls retain completion ownership in the coordinator if the caller
+    /// stops waiting. Implementations may narrow each request's backend wait;
+    /// they must never reset the deadline after connection or queue delays.
+    async fn acquire_locks_batch_until(&self, requests: &[LockRequest], deadline: Instant) -> Vec<BatchAcquireOutcome> {
+        if Instant::now() >= deadline {
+            return requests
+                .iter()
+                .map(|request| BatchAcquireOutcome::RetryableNoGrant {
+                    error: LockError::timeout(request.resource.to_string(), request.acquire_timeout),
+                })
+                .collect();
+        }
+        match self.acquire_locks_batch(requests).await {
+            Ok(responses) => {
+                let mut responses = responses.into_iter();
+                requests
+                    .iter()
+                    .map(|_| match responses.next() {
+                        Some(response) => BatchAcquireOutcome::from_legacy_response(response),
+                        None => BatchAcquireOutcome::Ambiguous {
+                            error: LockError::internal("Legacy peer returned an incomplete batch response"),
+                            cleanup_owned_by_client: false,
+                        },
+                    })
+                    .collect()
+            }
+            Err(error) => requests
+                .iter()
+                .map(|_| BatchAcquireOutcome::Ambiguous {
+                    error: error.clone(),
+                    cleanup_owned_by_client: false,
+                })
+                .collect(),
+        }
+    }
+
     /// Release lock
     async fn release(&self, lock_id: &LockId) -> Result<bool>;
 
@@ -122,3 +200,39 @@ impl ClientFactory {
     //     Arc::new(remote::RemoteClient::new(endpoint))
     // }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn legacy_failure_text_never_proves_no_grant() {
+        for message in [
+            "Lock acquisition timeout",
+            "Lock conflict: resource held by holder",
+            "Remote lock RPC timed out",
+        ] {
+            assert!(matches!(
+                BatchAcquireOutcome::from_legacy_response(LockResponse::failure(message, Duration::ZERO)),
+                BatchAcquireOutcome::Ambiguous {
+                    cleanup_owned_by_client: false,
+                    ..
+                }
+            ));
+        }
+    }
+
+    #[test]
+    fn legacy_success_without_metadata_remains_a_grant() {
+        assert!(matches!(
+            BatchAcquireOutcome::from_legacy_response(LockResponse {
+                success: true,
+                lock_info: None,
+                error: None,
+                wait_time: Duration::ZERO,
+                position_in_queue: None,
+            }),
+            BatchAcquireOutcome::Granted { lock_info: None, .. }
+        ));
+    }
+}

crates/lock/src/distributed_lock/batch_acquire_tests.rs

@@ -0,0 +1,451 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::{DistributedLock, LockBatchAcquireRound};
+use crate::{
+    GlobalLockManager, LocalClient, LockCleanupOutcome, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats,
+    LockType, ObjectKey, client::LockClient,
+};
+use std::sync::{
+    Arc, Mutex,
+    atomic::{AtomicBool, AtomicUsize, Ordering},
+};
+use std::time::Duration;
+use tokio::sync::{Notify, Semaphore};
+use tokio::task::JoinSet;
+
+#[derive(Debug)]
+struct ObservedBatchClient {
+    backend: LocalClient,
+    deny_acquire: bool,
+    acquire_gate: Option<Arc<Semaphore>>,
+    cleanup_gate: Option<Arc<Semaphore>>,
+    granted: AtomicUsize,
+    grant_changed: Notify,
+    released: AtomicUsize,
+    release_changed: Notify,
+    cleanup_entered: Notify,
+    cleanup_calls: AtomicUsize,
+    cleanup_changed: Notify,
+    cleanup_ids: Mutex<Vec<LockId>>,
+    omit_lock_info: AtomicBool,
+    lose_release_ack: AtomicBool,
+    release_ack_gate: Semaphore,
+}
+
+impl ObservedBatchClient {
+    fn new(deny_acquire: bool, acquire_gate: Option<Arc<Semaphore>>, cleanup_gate: Option<Arc<Semaphore>>) -> Arc<Self> {
+        Arc::new(Self {
+            backend: LocalClient::with_manager(Arc::new(GlobalLockManager::new())),
+            deny_acquire,
+            acquire_gate,
+            cleanup_gate,
+            granted: AtomicUsize::new(0),
+            grant_changed: Notify::new(),
+            released: AtomicUsize::new(0),
+            release_changed: Notify::new(),
+            cleanup_entered: Notify::new(),
+            cleanup_calls: AtomicUsize::new(0),
+            cleanup_changed: Notify::new(),
+            cleanup_ids: Mutex::new(Vec::new()),
+            omit_lock_info: AtomicBool::new(false),
+            lose_release_ack: AtomicBool::new(false),
+            release_ack_gate: Semaphore::new(0),
+        })
+    }
+
+    async fn wait_for_count(counter: &AtomicUsize, changed: &Notify, expected: usize) {
+        tokio::time::timeout(Duration::from_secs(1), async {
+            loop {
+                let notified = changed.notified();
+                if counter.load(Ordering::SeqCst) >= expected {
+                    return;
+                }
+                notified.await;
+            }
+        })
+        .await
+        .expect("expected real backend grant/release event before the test deadline");
+    }
+
+    async fn wait_for_grants(&self, expected: usize) {
+        Self::wait_for_count(&self.granted, &self.grant_changed, expected).await;
+    }
+
+    async fn wait_for_releases(&self, expected: usize) {
+        Self::wait_for_count(&self.released, &self.release_changed, expected).await;
+    }
+}
+
+#[async_trait::async_trait]
+impl LockClient for ObservedBatchClient {
+    async fn acquire_lock(&self, request: &LockRequest) -> crate::Result<LockResponse> {
+        if self.deny_acquire {
+            return Ok(LockResponse::failure("injected peer refusal", Duration::ZERO));
+        }
+        if let Some(gate) = &self.acquire_gate {
+            gate.acquire().await.expect("test acquisition gate remains open").forget();
+        }
+        let mut response = self.backend.acquire_lock(request).await?;
+        if response.success {
+            self.granted.fetch_add(1, Ordering::SeqCst);
+            self.grant_changed.notify_one();
+            if self.omit_lock_info.load(Ordering::SeqCst) {
+                response.lock_info = None;
+            }
+        }
+        Ok(response)
+    }
+
+    async fn cleanup_grant(&self, lock_id: &LockId) -> crate::Result<LockCleanupOutcome> {
+        self.cleanup_ids
+            .lock()
+            .expect("test cleanup ID log is available")
+            .push(lock_id.clone());
+        self.cleanup_calls.fetch_add(1, Ordering::SeqCst);
+        self.cleanup_changed.notify_one();
+        self.cleanup_entered.notify_one();
+        if let Some(gate) = &self.cleanup_gate {
+            gate.acquire().await.expect("test cleanup gate remains open").forget();
+        }
+        let outcome = self.backend.cleanup_grant(lock_id).await?;
+        if outcome == LockCleanupOutcome::Released {
+            self.released.fetch_add(1, Ordering::SeqCst);
+            self.release_changed.notify_one();
+            if self.lose_release_ack.swap(false, Ordering::SeqCst) {
+                self.release_ack_gate
+                    .acquire()
+                    .await
+                    .expect("test ACK gate remains open")
+                    .forget();
+                return Err(LockError::internal("injected lost release ACK"));
+            }
+        }
+        Ok(outcome)
+    }
+
+    async fn release(&self, lock_id: &LockId) -> crate::Result<bool> {
+        self.cleanup_grant(lock_id)
+            .await
+            .map(|outcome| outcome == LockCleanupOutcome::Released)
+    }
+
+    async fn refresh(&self, lock_id: &LockId) -> crate::Result<bool> {
+        self.backend.refresh(lock_id).await
+    }
+
+    async fn force_release(&self, lock_id: &LockId) -> crate::Result<bool> {
+        self.backend.force_release(lock_id).await
+    }
+
+    async fn check_status(&self, lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
+        self.backend.check_status(lock_id).await
+    }
+
+    async fn get_stats(&self) -> crate::Result<LockStats> {
+        self.backend.get_stats().await
+    }
+
+    async fn close(&self) -> crate::Result<()> {
+        self.backend.close().await
+    }
+
+    async fn is_online(&self) -> bool {
+        true
+    }
+
+    async fn is_local(&self) -> bool {
+        true
+    }
+}
+
+#[tokio::test(start_paused = true)]
+async fn cancelled_batch_acquisition_cleans_completed_and_late_grants() {
+    let late_gate = Arc::new(Semaphore::new(0));
+    let completed = ObservedBatchClient::new(false, None, None);
+    let late = ObservedBatchClient::new(false, Some(late_gate.clone()), None);
+    let lock = DistributedLock::new("cancelled-batch".to_string(), vec![completed.clone(), late.clone()], 2);
+    let resources = (0..1000)
+        .map(|key| ObjectKey::new("cancelled-batch", format!("key-{key}")))
+        .collect::<Vec<_>>();
+    let operation = tokio::spawn(async move {
+        lock.acquire_guard_batch(&resources, "owner", Duration::from_secs(5), Duration::from_secs(30))
+            .await
+    });
+
+    completed.wait_for_grants(1000).await;
+    assert_eq!(completed.backend.list_lock_leases().await.len(), 1000);
+    assert!(!operation.is_finished(), "the other peer still cannot complete its batch");
+    operation.abort();
+    assert!(operation.await.expect_err("caller was cancelled").is_cancelled());
+    late_gate.add_permits(1000);
+
+    completed.wait_for_releases(1000).await;
+    late.wait_for_releases(1000).await;
+    assert!(completed.backend.list_lock_leases().await.is_empty());
+    assert!(late.backend.list_lock_leases().await.is_empty());
+}
+
+#[tokio::test(start_paused = true)]
+async fn cancelled_batch_rollback_keeps_its_cleanup_owner() {
+    let cleanup_gate = Arc::new(Semaphore::new(0));
+    let granted = ObservedBatchClient::new(false, None, Some(cleanup_gate.clone()));
+    let refused = ObservedBatchClient::new(true, None, None);
+    let lock = DistributedLock::new("cancelled-rollback".to_string(), vec![granted.clone(), refused], 2);
+    let operation = tokio::spawn(async move {
+        lock.acquire_guard_batch(
+            &[ObjectKey::new("cancelled-rollback", "object")],
+            "owner",
+            Duration::from_secs(5),
+            Duration::from_secs(30),
+        )
+        .await
+    });
+
+    tokio::time::timeout(Duration::from_secs(1), granted.cleanup_entered.notified())
+        .await
+        .expect("below-quorum rollback entered the real grant cleanup");
+    assert_eq!(granted.backend.list_lock_leases().await.len(), 1);
+    operation.abort();
+    assert!(
+        operation
+            .await
+            .expect_err("caller was cancelled during rollback")
+            .is_cancelled()
+    );
+    cleanup_gate.add_permits(1);
+
+    granted.wait_for_releases(1).await;
+    assert!(granted.backend.list_lock_leases().await.is_empty());
+}
+
+#[tokio::test(start_paused = true)]
+async fn cancelled_batch_round_drains_ready_unjoined_replies_with_original_ids() {
+    for omit_lock_info in [false, true] {
+        let first = ObservedBatchClient::new(false, None, None);
+        let second = ObservedBatchClient::new(false, None, None);
+        first.omit_lock_info.store(omit_lock_info, Ordering::SeqCst);
+        second.omit_lock_info.store(omit_lock_info, Ordering::SeqCst);
+        let clients: Arc<[Arc<dyn LockClient>]> = vec![first.clone() as Arc<dyn LockClient>, second.clone()].into();
+        let requests: Arc<[LockRequest]> = vec![
+            LockRequest::new(ObjectKey::new("ready-batch", "object"), LockType::Exclusive, "owner")
+                .with_acquire_timeout(Duration::from_secs(5))
+                .with_ttl(Duration::from_secs(30)),
+        ]
+        .into();
+        let mut round = LockBatchAcquireRound {
+            pending: JoinSet::new(),
+            grants: vec![Vec::new()],
+            clients,
+            span: tracing::Span::current(),
+        };
+        let mut tasks = Vec::new();
+        for (peer_index, client) in round.clients.iter().enumerate() {
+            let client = client.clone();
+            let requests = requests.clone();
+            tasks.push(round.pending.spawn(async move {
+                let started = tokio::time::Instant::now();
+                let outcomes = client
+                    .acquire_locks_batch_until(&requests, started + Duration::from_secs(5))
+                    .await;
+                super::LockBatchAcquireTaskResult {
+                    peer_index,
+                    key_indices: vec![0],
+                    requests,
+                    outcomes,
+                    started,
+                    elapsed: Duration::ZERO,
+                }
+            }));
+        }
+        first.wait_for_grants(1).await;
+        second.wait_for_grants(1).await;
+        assert!(tasks.iter().all(tokio::task::AbortHandle::is_finished));
+        assert_eq!(round.pending.len(), 2, "both ready replies remain unjoined");
+        assert!(round.grants.iter().all(Vec::is_empty), "no reply has been transferred yet");
+        drop(round);
+
+        first.wait_for_releases(1).await;
+        second.wait_for_releases(1).await;
+        assert!(first.backend.list_lock_leases().await.is_empty());
+        assert!(second.backend.list_lock_leases().await.is_empty());
+    }
+}
+
+#[tokio::test(start_paused = true)]
+async fn cancelled_batch_preserves_a_locally_partially_completed_batch() {
+    let client = ObservedBatchClient::new(false, None, None);
+    let blocked = ObjectKey::new("local-partial-batch", "blocked");
+    let blocker = LockRequest::new(blocked.clone(), LockType::Exclusive, "blocker")
+        .with_acquire_timeout(Duration::from_secs(5))
+        .with_ttl(Duration::from_secs(30));
+    assert!(
+        client
+            .backend
+            .acquire_lock(&blocker)
+            .await
+            .expect("hold one local key")
+            .success
+    );
+    let lock = DistributedLock::new("local-partial-batch".to_string(), vec![client.clone()], 1);
+    let operation = tokio::spawn(async move {
+        lock.acquire_guard_batch(
+            &[
+                blocked,
+                ObjectKey::new("local-partial-batch", "free-one"),
+                ObjectKey::new("local-partial-batch", "free-two"),
+            ],
+            "batch-owner",
+            Duration::from_secs(5),
+            Duration::from_secs(30),
+        )
+        .await
+    });
+    client.wait_for_grants(2).await;
+    assert_eq!(client.backend.list_lock_leases().await.len(), 3, "blocker plus two partial grants");
+    assert!(!operation.is_finished());
+    operation.abort();
+    assert!(
+        operation
+            .await
+            .expect_err("cancel the partially completed batch")
+            .is_cancelled()
+    );
+    assert_eq!(
+        client
+            .backend
+            .cleanup_grant(&blocker.lock_id)
+            .await
+            .expect("release the original holder"),
+        LockCleanupOutcome::Released
+    );
+
+    client.wait_for_grants(3).await;
+    client.wait_for_releases(3).await;
+    assert!(client.backend.list_lock_leases().await.is_empty());
+}
+
+#[tokio::test(start_paused = true)]
+async fn successful_batch_guards_retain_their_real_grants_until_drop() {
+    let client = ObservedBatchClient::new(false, None, None);
+    let lock = DistributedLock::new("successful-batch".to_string(), vec![client.clone()], 1);
+    let results = lock
+        .acquire_guard_batch(
+            &[
+                ObjectKey::new("successful-batch", "one"),
+                ObjectKey::new("successful-batch", "two"),
+            ],
+            "owner",
+            Duration::from_secs(5),
+            Duration::from_secs(30),
+        )
+        .await;
+    assert!(results.iter().all(Result::is_ok));
+    assert_eq!(client.backend.list_lock_leases().await.len(), 2);
+    assert_eq!(client.cleanup_calls.load(Ordering::SeqCst), 0, "round drop cannot clean live guards");
+    drop(results);
+
+    client.wait_for_releases(2).await;
+    assert!(client.backend.list_lock_leases().await.is_empty());
+}
+
+#[tokio::test(start_paused = true)]
+async fn batch_guard_and_rollback_use_the_request_id_when_lock_info_is_missing() {
+    for deny_other_peer in [false, true] {
+        let granted = ObservedBatchClient::new(false, None, None);
+        granted.omit_lock_info.store(true, Ordering::SeqCst);
+        let other = ObservedBatchClient::new(deny_other_peer, None, None);
+        let lock = DistributedLock::new("missing-batch-info".to_string(), vec![granted.clone(), other.clone()], 2);
+        let results = lock
+            .acquire_guard_batch(
+                &[ObjectKey::new("missing-batch-info", "object")],
+                "owner",
+                Duration::from_secs(5),
+                Duration::from_secs(30),
+            )
+            .await;
+        assert_eq!(results[0].is_err(), deny_other_peer);
+        if !deny_other_peer {
+            assert_eq!(granted.backend.list_lock_leases().await.len(), 1);
+            assert_eq!(granted.cleanup_calls.load(Ordering::SeqCst), 0);
+        }
+        drop(results);
+        granted.wait_for_releases(1).await;
+        if !deny_other_peer {
+            other.wait_for_releases(1).await;
+        }
+        assert!(granted.backend.list_lock_leases().await.is_empty());
+        assert!(other.backend.list_lock_leases().await.is_empty());
+    }
+}
+
+#[tokio::test(start_paused = true)]
+async fn cancelled_batch_rollback_retries_lost_ack_without_releasing_a_successor() {
+    let granted = ObservedBatchClient::new(false, None, None);
+    granted.lose_release_ack.store(true, Ordering::SeqCst);
+    let refused = ObservedBatchClient::new(true, None, None);
+    let resource = ObjectKey::new("batch-lost-ack", "object");
+    let lock = DistributedLock::new("batch-lost-ack".to_string(), vec![granted.clone(), refused], 2);
+    let acquire_resource = resource.clone();
+    let operation = tokio::spawn(async move {
+        lock.acquire_guard_batch(&[acquire_resource], "predecessor", Duration::from_secs(5), Duration::from_secs(30))
+            .await
+    });
+    granted.wait_for_releases(1).await;
+    let successor = LockRequest::new(resource, LockType::Exclusive, "successor")
+        .with_acquire_timeout(Duration::from_secs(5))
+        .with_ttl(Duration::from_secs(30));
+    assert!(
+        granted
+            .backend
+            .acquire_lock(&successor)
+            .await
+            .expect("successor acquires after physical release")
+            .success
+    );
+    operation.abort();
+    assert!(
+        operation
+            .await
+            .expect_err("cancel caller while release ACK is blocked")
+            .is_cancelled()
+    );
+    granted.release_ack_gate.add_permits(1);
+
+    ObservedBatchClient::wait_for_count(&granted.cleanup_calls, &granted.cleanup_changed, 2).await;
+    let cleanup_ids = granted.cleanup_ids.lock().expect("cleanup ID log").clone();
+    assert!(cleanup_ids.len() >= 2, "the uncertain ACK must be retried");
+    assert!(
+        cleanup_ids.iter().all(|id| id == &cleanup_ids[0]),
+        "cleanup retains the exact predecessor ID"
+    );
+    assert_ne!(cleanup_ids[0], successor.lock_id);
+    assert!(
+        granted
+            .backend
+            .check_status(&successor.lock_id)
+            .await
+            .expect("check successor")
+            .is_some()
+    );
+    assert_eq!(granted.backend.list_lock_leases().await.len(), 1);
+    assert_eq!(
+        granted
+            .backend
+            .cleanup_grant(&successor.lock_id)
+            .await
+            .expect("clean successor test grant"),
+        LockCleanupOutcome::Released
+    );
+}

crates/lock/src/distributed_lock/batch_retry_tests.rs

@@ -0,0 +1,898 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::{BatchLockPeerDiagnostics, DistributedLock};
+use crate::{
+    BatchAcquireOutcome, GlobalLockManager, LocalClient, LockCleanupOutcome, LockError, LockId, LockInfo, LockRequest,
+    LockResponse, LockStats, ObjectKey, client::LockClient,
+};
+use std::{
+    collections::VecDeque,
+    sync::{
+        Arc, Mutex,
+        atomic::{AtomicBool, AtomicUsize, Ordering},
+    },
+    time::Duration,
+};
+use tokio::{
+    sync::{Notify, Semaphore},
+    time::Instant,
+};
+
+#[derive(Clone, Debug)]
+enum Plan {
+    Acquire,
+    Retryable,
+    Terminal,
+    Ambiguous,
+    ReplyGate(Arc<Semaphore>),
+}
+
+#[derive(Debug)]
+struct TypedClient {
+    backend: LocalClient,
+    plans: Mutex<VecDeque<Plan>>,
+    calls: Mutex<Vec<Vec<LockId>>>,
+    deadlines: Mutex<Vec<Instant>>,
+    changed: Notify,
+    grants: AtomicUsize,
+    cleanup_calls: AtomicUsize,
+    released: AtomicUsize,
+    refreshes: AtomicUsize,
+    refresh_allowed: AtomicBool,
+    cleanup_failures: AtomicUsize,
+    cleanup_gate: Option<Arc<Semaphore>>,
+    cleanup_key_gate: Mutex<Option<(ObjectKey, Arc<Semaphore>)>>,
+    local: bool,
+    local_gate: Mutex<Option<Arc<Semaphore>>>,
+}
+
+impl TypedClient {
+    fn new(plans: Vec<Plan>, local: bool, cleanup_gate: Option<Arc<Semaphore>>) -> Arc<Self> {
+        Arc::new(Self {
+            backend: LocalClient::with_manager(Arc::new(GlobalLockManager::new())),
+            plans: Mutex::new(plans.into()),
+            calls: Mutex::new(Vec::new()),
+            deadlines: Mutex::new(Vec::new()),
+            changed: Notify::new(),
+            grants: AtomicUsize::new(0),
+            cleanup_calls: AtomicUsize::new(0),
+            released: AtomicUsize::new(0),
+            refreshes: AtomicUsize::new(0),
+            refresh_allowed: AtomicBool::new(true),
+            cleanup_failures: AtomicUsize::new(0),
+            cleanup_gate,
+            cleanup_key_gate: Mutex::new(None),
+            local,
+            local_gate: Mutex::new(None),
+        })
+    }
+
+    async fn wait_until(&self, predicate: impl Fn() -> bool) {
+        tokio::time::timeout(Duration::from_secs(1), async {
+            loop {
+                let notified = self.changed.notified();
+                if predicate() {
+                    return;
+                }
+                notified.await;
+            }
+        })
+        .await
+        .expect("expected a real acquisition or cleanup event");
+    }
+
+    async fn wait_calls(&self, count: usize) {
+        self.wait_until(|| self.calls.lock().expect("call log").len() >= count).await;
+    }
+
+    async fn wait_grants(&self, count: usize) {
+        self.wait_until(|| self.grants.load(Ordering::SeqCst) >= count).await;
+    }
+
+    async fn wait_releases(&self, count: usize) {
+        self.wait_until(|| self.released.load(Ordering::SeqCst) >= count).await;
+    }
+}
+
+#[async_trait::async_trait]
+impl LockClient for TypedClient {
+    async fn acquire_lock(&self, request: &LockRequest) -> crate::Result<LockResponse> {
+        self.backend.acquire_lock(request).await
+    }
+    async fn acquire_locks_batch_until(&self, requests: &[LockRequest], deadline: Instant) -> Vec<BatchAcquireOutcome> {
+        self.calls
+            .lock()
+            .expect("call log")
+            .push(requests.iter().map(|request| request.lock_id.clone()).collect());
+        self.deadlines.lock().expect("deadline log").push(deadline);
+        self.changed.notify_one();
+        let plan = self.plans.lock().expect("plan queue").pop_front().unwrap_or(Plan::Acquire);
+        match plan {
+            Plan::Retryable => requests
+                .iter()
+                .map(|_| BatchAcquireOutcome::RetryableNoGrant {
+                    error: LockError::internal("explicit no grant"),
+                })
+                .collect(),
+            Plan::Terminal => requests
+                .iter()
+                .map(|_| BatchAcquireOutcome::TerminalNoGrant {
+                    error: LockError::permission_denied("explicit refusal"),
+                })
+                .collect(),
+            Plan::Ambiguous => requests
+                .iter()
+                .map(|_| BatchAcquireOutcome::Ambiguous {
+                    error: LockError::internal("unknown outcome"),
+                    cleanup_owned_by_client: true,
+                })
+                .collect(),
+            Plan::Acquire | Plan::ReplyGate(_) => {
+                let outcomes = self.backend.acquire_locks_batch_until(requests, deadline).await;
+                self.grants.fetch_add(
+                    outcomes
+                        .iter()
+                        .filter(|outcome| matches!(outcome, BatchAcquireOutcome::Granted { .. }))
+                        .count(),
+                    Ordering::SeqCst,
+                );
+                self.changed.notify_one();
+                if let Plan::ReplyGate(gate) = plan {
+                    gate.acquire().await.expect("response gate").forget();
+                }
+                outcomes
+            }
+        }
+    }
+    async fn cleanup_grant(&self, id: &LockId) -> crate::Result<LockCleanupOutcome> {
+        self.cleanup_calls.fetch_add(1, Ordering::SeqCst);
+        self.changed.notify_one();
+        if let Some(gate) = &self.cleanup_gate {
+            gate.acquire().await.expect("cleanup gate").forget();
+        }
+        let key_gate = self
+            .cleanup_key_gate
+            .lock()
+            .expect("key cleanup gate")
+            .as_ref()
+            .filter(|(key, _)| *key == id.resource)
+            .map(|(_, gate)| gate.clone());
+        if let Some(gate) = key_gate {
+            gate.acquire().await.expect("key cleanup gate").forget();
+        }
+        if self
+            .cleanup_failures
+            .try_update(Ordering::SeqCst, Ordering::SeqCst, |count| count.checked_sub(1))
+            .is_ok()
+        {
+            return Err(LockError::internal("injected cleanup failure"));
+        }
+        let result = self.backend.cleanup_grant(id).await?;
+        if result == LockCleanupOutcome::Released {
+            self.released.fetch_add(1, Ordering::SeqCst);
+            self.changed.notify_one();
+        }
+        Ok(result)
+    }
+    async fn release(&self, id: &LockId) -> crate::Result<bool> {
+        self.cleanup_grant(id)
+            .await
+            .map(|outcome| outcome == LockCleanupOutcome::Released)
+    }
+    async fn refresh(&self, id: &LockId) -> crate::Result<bool> {
+        self.refreshes.fetch_add(1, Ordering::SeqCst);
+        self.changed.notify_one();
+        if self.refresh_allowed.load(Ordering::SeqCst) {
+            self.backend.refresh(id).await
+        } else {
+            Ok(false)
+        }
+    }
+    async fn force_release(&self, id: &LockId) -> crate::Result<bool> {
+        self.backend.force_release(id).await
+    }
+    async fn check_status(&self, id: &LockId) -> crate::Result<Option<LockInfo>> {
+        self.backend.check_status(id).await
+    }
+    async fn get_stats(&self) -> crate::Result<LockStats> {
+        self.backend.get_stats().await
+    }
+    async fn close(&self) -> crate::Result<()> {
+        self.backend.close().await
+    }
+    async fn is_online(&self) -> bool {
+        true
+    }
+    async fn is_local(&self) -> bool {
+        let gate = self.local_gate.lock().expect("local lookup gate").take();
+        if let Some(gate) = gate {
+            gate.acquire().await.expect("local lookup gate open").forget();
+        }
+        self.local
+    }
+}
+
+fn lock_for(clients: &[Arc<TypedClient>], quorum: usize) -> DistributedLock {
+    DistributedLock::new(
+        "batch-retry".to_owned(),
+        clients.iter().cloned().map(|client| client as Arc<dyn LockClient>).collect(),
+        quorum,
+    )
+}
+
+#[tokio::test(start_paused = true)]
+async fn healthy_thousand_key_batch_keeps_one_call_per_peer() {
+    let peers: Vec<_> = (0..4).map(|_| TypedClient::new(Vec::new(), true, None)).collect();
+    let resources: Vec<_> = (0..1000)
+        .map(|index| ObjectKey::new("healthy-batch", format!("{index}")))
+        .collect();
+    let results = lock_for(&peers, 3)
+        .acquire_guard_batch(&resources, "owner", Duration::from_secs(5), Duration::from_secs(30))
+        .await;
+    assert!(results.iter().all(Result::is_ok));
+    for peer in &peers {
+        peer.wait_calls(1).await;
+        let calls = peer.calls.lock().expect("call log");
+        assert_eq!(calls.len(), 1);
+        assert_eq!(calls[0].len(), 1000);
+    }
+    drop(results);
+}
+
+#[tokio::test(start_paused = true)]
+async fn third_pending_vote_is_waited_for_even_with_an_ambiguous_peer() {
+    for before_deadline in [true, false] {
+        let gate = Arc::new(Semaphore::new(0));
+        let peers = vec![
+            TypedClient::new(Vec::new(), true, None),
+            TypedClient::new(Vec::new(), true, None),
+            TypedClient::new(vec![Plan::ReplyGate(gate.clone())], false, None),
+            TypedClient::new(vec![Plan::Ambiguous], false, None),
+        ];
+        let lock = lock_for(&peers, 3);
+        let start = Instant::now();
+        let operation = tokio::spawn(async move {
+            lock.acquire_guard_batch(
+                &[ObjectKey::new("pending-third", "key")],
+                "owner",
+                Duration::from_secs(5),
+                Duration::from_secs(30),
+            )
+            .await
+        });
+        for peer in &peers[..3] {
+            peer.wait_grants(1).await;
+        }
+        assert!(!operation.is_finished(), "two votes do not discard a pending third vote");
+        if before_deadline {
+            gate.add_permits(1);
+            let result = operation.await.expect("batch task");
+            assert!(result[0].is_ok());
+            assert!(start.elapsed() < Duration::from_secs(5));
+            drop(result);
+        } else {
+            tokio::time::advance(Duration::from_secs(5)).await;
+            let result = operation.await.expect("deadline returns without the delayed reply");
+            assert!(result[0].is_err());
+            gate.add_permits(1);
+        }
+        for peer in &peers[..3] {
+            peer.wait_releases(1).await;
+            assert!(peer.backend.list_lock_leases().await.is_empty());
+        }
+    }
+}
+
+#[tokio::test(start_paused = true)]
+async fn ambiguous_peer_is_excluded_and_confirmed_subset_retries_with_fresh_ids() {
+    for terminal in [false, true] {
+        let peers = vec![
+            TypedClient::new(vec![Plan::Ambiguous], false, None),
+            TypedClient::new(Vec::new(), true, None),
+            TypedClient::new(Vec::new(), true, None),
+            TypedClient::new(vec![if terminal { Plan::Terminal } else { Plan::Retryable }], true, None),
+        ];
+        peers[1].cleanup_failures.store(1, Ordering::SeqCst);
+        let result = lock_for(&peers, 3)
+            .acquire_guard_batch(
+                &[ObjectKey::new("subset-retry", "key")],
+                "owner",
+                Duration::from_secs(5),
+                Duration::from_secs(30),
+            )
+            .await;
+        assert_eq!(result[0].is_err(), terminal);
+        assert_eq!(
+            peers[0].calls.lock().expect("ambiguous call log").len(),
+            1,
+            "uncertain peer never receives another attempt"
+        );
+        for peer in &peers[1..] {
+            let calls = peer.calls.lock().expect("healthy call log");
+            assert_eq!(calls.len(), if terminal { 1 } else { 2 });
+            if !terminal {
+                assert_ne!(calls[0][0], calls[1][0], "each retry uses a fresh exact ID");
+            }
+        }
+        assert!(
+            peers[1].cleanup_calls.load(Ordering::SeqCst) >= 2,
+            "first failed cleanup is retried before a new acquire"
+        );
+        assert_eq!(peers[1].released.load(Ordering::SeqCst), 1);
+        assert_eq!(peers[2].released.load(Ordering::SeqCst), 1);
+        drop(result);
+    }
+}
+
+#[tokio::test(start_paused = true)]
+async fn hot_key_retries_healthy_subset_without_waiting_for_silent_peer() {
+    let peers: Vec<_> = (0..3).map(|_| TypedClient::new(Vec::new(), true, None)).collect();
+    let holder_lock = lock_for(&peers, 3);
+    let hot = ObjectKey::new("hot-subset", "hot");
+    let holder = holder_lock
+        .lock_guard(hot.clone(), "holder", Duration::from_secs(5), Duration::from_secs(30))
+        .await
+        .expect("legitimate distributed holder")
+        .expect("holder reaches q3");
+    let late_gate = Arc::new(Semaphore::new(0));
+    let late = TypedClient::new(vec![Plan::ReplyGate(late_gate.clone())], false, None);
+    let mut batch_peers = peers.clone();
+    batch_peers.push(late.clone());
+    let lock = lock_for(&batch_peers, 3);
+    let start = Instant::now();
+    let resources: Vec<_> = std::iter::once(hot.clone())
+        .chain((0..999).map(|index| ObjectKey::new("hot-subset", format!("free-{index}"))))
+        .collect();
+    let release_holder = tokio::spawn(async move {
+        tokio::time::sleep_until(start + Duration::from_millis(1500)).await;
+        drop(holder);
+    });
+    let results = lock
+        .acquire_guard_batch(&resources, "owner", Duration::from_secs(5), Duration::from_secs(30))
+        .await;
+    release_holder.await.expect("holder release");
+    assert!(
+        results.iter().all(Result::is_ok),
+        "hot key recovers after the first local window, while the fourth peer is still silent"
+    );
+    assert!(start.elapsed() < Duration::from_secs(5));
+    assert_eq!(late.calls.lock().expect("silent peer call log").len(), 1);
+    for peer in &peers {
+        let calls = peer.calls.lock().expect("call log");
+        assert_eq!(calls[0].len(), 1000);
+        assert!(calls.len() >= 2);
+        assert!(
+            calls[1..].iter().all(|batch| batch.len() == 1 && batch[0].resource == hot),
+            "only the unresolved hot key is retried"
+        );
+    }
+    late_gate.add_permits(1);
+    late.wait_releases(1000).await;
+    drop(results);
+}
+
+#[tokio::test(start_paused = true)]
+async fn completed_eligible_quorum_retries_while_a_possible_vote_is_silent() {
+    let peers: Vec<_> = (0..3).map(|_| TypedClient::new(Vec::new(), true, None)).collect();
+    let resource = ObjectKey::new("completed-quorum", "hot");
+    let holder = LockRequest::new(resource.clone(), crate::LockType::Exclusive, "holder").with_ttl(Duration::from_secs(30));
+    assert!(
+        peers[2]
+            .backend
+            .acquire_lock(&holder)
+            .await
+            .expect("contended peer holder")
+            .success
+    );
+    let gate = Arc::new(Semaphore::new(0));
+    let late = TypedClient::new(vec![Plan::ReplyGate(gate.clone())], false, None);
+    let mut batch_peers = peers.clone();
+    batch_peers.push(late.clone());
+    let lock = lock_for(&batch_peers, 3);
+    let start = Instant::now();
+    let contended = peers[2].clone();
+    let release_holder = tokio::spawn(async move {
+        tokio::time::sleep_until(start + Duration::from_millis(1500)).await;
+        contended
+            .backend
+            .cleanup_grant(&holder.lock_id)
+            .await
+            .expect("release contention");
+    });
+    let results = lock
+        .acquire_guard_batch(&[resource], "owner", Duration::from_secs(5), Duration::from_secs(30))
+        .await;
+    release_holder.await.expect("holder release task");
+    assert!(
+        results[0].is_ok(),
+        "the three completed eligible peers retry before the silent fourth responds"
+    );
+    assert!(start.elapsed() < Duration::from_secs(5));
+    for peer in &peers {
+        let calls = peer.calls.lock().expect("reachable peer attempts");
+        assert!(calls.len() >= 2);
+        assert_ne!(calls[0][0], calls[1][0], "rollback precedes a fresh-ID attempt");
+    }
+    assert_eq!(late.calls.lock().expect("silent peer attempts").len(), 1);
+    gate.add_permits(1);
+    late.wait_releases(1).await;
+    drop(results);
+    for peer in &peers {
+        peer.wait_releases(if Arc::ptr_eq(peer, &peers[2]) { 1 } else { 2 }).await;
+        assert!(peer.backend.list_lock_leases().await.is_empty());
+    }
+}
+
+#[tokio::test(start_paused = true)]
+async fn cleaned_late_peer_rejoins_keys_with_distinct_peer_contention() {
+    let peers: Vec<_> = (0..3).map(|_| TypedClient::new(Vec::new(), true, None)).collect();
+    let resources = vec![
+        ObjectKey::new("late-rejoins", "first"),
+        ObjectKey::new("late-rejoins", "second"),
+    ];
+    let mut holders = Vec::new();
+    for (peer, key) in [(1, 0), (2, 1)] {
+        let holder = LockRequest::new(resources[key].clone(), crate::LockType::Exclusive, "distinct-holder")
+            .with_ttl(Duration::from_secs(30));
+        assert!(
+            peers[peer]
+                .backend
+                .acquire_lock(&holder)
+                .await
+                .expect("distinct holder")
+                .success
+        );
+        holders.push((peer, holder));
+    }
+    let gate = Arc::new(Semaphore::new(0));
+    let late = TypedClient::new(vec![Plan::ReplyGate(gate.clone())], false, None);
+    let mut batch_peers = peers.clone();
+    batch_peers.push(late.clone());
+    let start = Instant::now();
+    let response = tokio::spawn(async move {
+        tokio::time::sleep_until(start + Duration::from_millis(1500)).await;
+        gate.add_permits(1);
+    });
+    let results = lock_for(&batch_peers, 3)
+        .acquire_guard_batch(&resources, "owner", Duration::from_secs(5), Duration::from_secs(30))
+        .await;
+    response.await.expect("late response task");
+    assert!(
+        results.iter().all(Result::is_ok),
+        "confirmed late cleanup restores a useful peer for each key"
+    );
+    assert!(start.elapsed() < Duration::from_secs(5));
+    assert!(late.calls.lock().expect("late attempts").len() >= 2);
+    for (peer, holder) in holders {
+        assert!(
+            peers[peer]
+                .backend
+                .check_status(&holder.lock_id)
+                .await
+                .expect("holder status")
+                .is_some()
+        );
+        peers[peer]
+            .backend
+            .cleanup_grant(&holder.lock_id)
+            .await
+            .expect("release holder");
+    }
+    drop(results);
+    for peer in batch_peers {
+        peer.wait_until(|| peer.released.load(Ordering::SeqCst) >= peer.grants.load(Ordering::SeqCst))
+            .await;
+        assert!(peer.backend.list_lock_leases().await.is_empty());
+    }
+}
+
+#[tokio::test(start_paused = true)]
+async fn cleaned_key_retries_while_another_keys_rollback_is_stalled() {
+    let peers = vec![
+        TypedClient::new(Vec::new(), true, None),
+        TypedClient::new(Vec::new(), true, None),
+        TypedClient::new(Vec::new(), true, None),
+        TypedClient::new(vec![Plan::Ambiguous], false, None),
+    ];
+    let stalled = ObjectKey::new("independent-rollback", "stalled");
+    let ready = ObjectKey::new("independent-rollback", "ready");
+    let gate = Arc::new(Semaphore::new(0));
+    *peers[0].cleanup_key_gate.lock().expect("install per-key gate") = Some((stalled.clone(), gate.clone()));
+    let holders: Vec<_> = [stalled.clone(), ready.clone()]
+        .into_iter()
+        .map(|key| LockRequest::new(key, crate::LockType::Exclusive, "holder").with_ttl(Duration::from_secs(30)))
+        .collect();
+    for holder in &holders {
+        assert!(
+            peers[2]
+                .backend
+                .acquire_lock(holder)
+                .await
+                .expect("contention holder")
+                .success
+        );
+    }
+    let start = Instant::now();
+    let contended = peers[2].clone();
+    let release_holder = tokio::spawn(async move {
+        tokio::time::sleep_until(start + Duration::from_millis(1500)).await;
+        for holder in holders {
+            contended
+                .backend
+                .cleanup_grant(&holder.lock_id)
+                .await
+                .expect("release held key");
+        }
+    });
+    let results = lock_for(&peers, 3)
+        .acquire_guard_batch(
+            &[stalled.clone(), ready.clone()],
+            "owner",
+            Duration::from_secs(5),
+            Duration::from_secs(30),
+        )
+        .await;
+    release_holder.await.expect("holder release task");
+    assert!(results[0].is_err(), "unconfirmed cleanup keeps its own key blocked");
+    assert!(
+        results[1].is_ok(),
+        "a fully cleaned key retries without waiting for another key's rollback"
+    );
+    assert_eq!(start.elapsed(), Duration::from_secs(5));
+    for peer in &peers[..3] {
+        let calls = peer.calls.lock().expect("per-key retry attempts");
+        assert!(calls.len() >= 2);
+        assert!(calls[1..].iter().all(|batch| batch.len() == 1 && batch[0].resource == ready));
+    }
+    assert_eq!(peers[3].calls.lock().expect("ambiguous attempts").len(), 1);
+    assert!(
+        peers[0]
+            .backend
+            .list_lock_leases()
+            .await
+            .iter()
+            .any(|lease| lease.resource == stalled)
+    );
+    gate.add_permits(1);
+    peers[0].wait_releases(2).await;
+    assert!(
+        peers[0]
+            .backend
+            .list_lock_leases()
+            .await
+            .iter()
+            .all(|lease| lease.resource != stalled)
+    );
+    drop(results);
+    for (index, peer) in peers[..3].iter().enumerate() {
+        peer.wait_releases(if index == 2 { 1 } else { 3 }).await;
+        assert!(peer.backend.list_lock_leases().await.is_empty());
+    }
+}
+
+#[tokio::test(start_paused = true)]
+async fn successful_keys_are_renewed_while_hot_key_uses_the_total_budget() {
+    for refresh_allowed in [true, false] {
+        let peers: Vec<_> = (0..3).map(|_| TypedClient::new(Vec::new(), true, None)).collect();
+        let hot = ObjectKey::new("retained-guard", "hot");
+        let holder = lock_for(&peers, 3)
+            .lock_guard(hot.clone(), "holder", Duration::from_secs(5), Duration::from_secs(30))
+            .await
+            .expect("distributed holder")
+            .expect("q3 holder");
+        for peer in &peers {
+            peer.refresh_allowed.store(refresh_allowed, Ordering::SeqCst);
+        }
+        let start = Instant::now();
+        let results = lock_for(&peers, 3)
+            .acquire_guard_batch(
+                &[hot, ObjectKey::new("retained-guard", "free")],
+                "owner",
+                Duration::from_secs(5),
+                Duration::from_secs(2),
+            )
+            .await;
+        assert!(results[0].is_err(), "holder is never released inside the acquisition budget");
+        assert!(
+            peers.iter().any(|peer| peer.refreshes.load(Ordering::SeqCst) > 0),
+            "cold key must form an early guard before D_total: calls={:?}, grants={:?}, refreshes={:?}",
+            peers
+                .iter()
+                .map(|peer| peer.calls.lock().expect("call log").len())
+                .collect::<Vec<_>>(),
+            peers
+                .iter()
+                .map(|peer| peer.grants.load(Ordering::SeqCst))
+                .collect::<Vec<_>>(),
+            peers
+                .iter()
+                .map(|peer| peer.refreshes.load(Ordering::SeqCst))
+                .collect::<Vec<_>>(),
+        );
+        assert_eq!(
+            results[1].is_ok(),
+            refresh_allowed,
+            "the final result rejects a previously successful guard after quorum is lost"
+        );
+        assert_eq!(start.elapsed(), Duration::from_secs(5));
+        if refresh_allowed {
+            assert!(
+                peers.iter().all(|peer| peer.refreshes.load(Ordering::SeqCst) > 1),
+                "healthy early guards repeatedly renew every quorum peer during other keys' retries"
+            );
+        }
+        drop(results);
+        drop(holder);
+    }
+}
+
+#[tokio::test(start_paused = true)]
+async fn rollback_deadline_detaches_cleanup_without_reacquiring() {
+    let gate = Arc::new(Semaphore::new(0));
+    let peers = vec![
+        TypedClient::new(vec![Plan::Ambiguous], false, None),
+        TypedClient::new(Vec::new(), true, Some(gate.clone())),
+        TypedClient::new(Vec::new(), true, None),
+        TypedClient::new(vec![Plan::Retryable], true, None),
+    ];
+    let start = Instant::now();
+    let result = lock_for(&peers, 3)
+        .acquire_guard_batch(
+            &[ObjectKey::new("rollback-deadline", "key")],
+            "owner",
+            Duration::from_millis(500),
+            Duration::from_secs(30),
+        )
+        .await;
+    assert!(result[0].is_err());
+    assert_eq!(start.elapsed(), Duration::from_millis(500));
+    assert!(peers.iter().all(|peer| peer.calls.lock().expect("call log").len() == 1));
+    assert_eq!(peers[1].backend.list_lock_leases().await.len(), 1);
+    gate.add_permits(1);
+    peers[1].wait_releases(1).await;
+    assert!(peers[1].backend.list_lock_leases().await.is_empty());
+}
+
+#[test]
+fn batch_diagnostics_preserve_typed_no_grant_and_ambiguity() {
+    let outcomes = vec![
+        BatchAcquireOutcome::Granted {
+            lock_info: None,
+            wait_time: Duration::ZERO,
+        },
+        BatchAcquireOutcome::RetryableNoGrant {
+            error: LockError::internal("explicit"),
+        },
+        BatchAcquireOutcome::TerminalNoGrant {
+            error: LockError::internal("terminal"),
+        },
+        BatchAcquireOutcome::Ambiguous {
+            error: LockError::internal("unknown"),
+            cleanup_owned_by_client: false,
+        },
+    ];
+    let diagnostics = BatchLockPeerDiagnostics::from_outcomes(1, 5, &outcomes, Duration::ZERO);
+    assert_eq!(diagnostics.granted_keys, 1);
+    assert_eq!(diagnostics.retryable_no_grant_keys, 1);
+    assert_eq!(diagnostics.terminal_no_grant_keys, 1);
+    assert_eq!(diagnostics.ambiguous_keys, 1);
+    assert_eq!(diagnostics.missing_response_keys, 1);
+}
+
+#[derive(Debug)]
+struct LegacyHiddenClient {
+    backend: LocalClient,
+    requested: Mutex<Vec<LockId>>,
+    release_attempts: Mutex<std::collections::HashMap<LockId, usize>>,
+    cleanup_calls: AtomicUsize,
+    changed: Notify,
+}
+
+#[async_trait::async_trait]
+impl LockClient for LegacyHiddenClient {
+    async fn acquire_lock(&self, request: &LockRequest) -> crate::Result<LockResponse> {
+        self.requested.lock().expect("request IDs").push(request.lock_id.clone());
+        if request.resource.object.as_ref() == "error" {
+            return Err(LockError::internal("one legacy item failed"));
+        }
+        self.backend.acquire_lock(request).await
+    }
+    async fn release(&self, id: &LockId) -> crate::Result<bool> {
+        let attempt = {
+            let mut attempts = self.release_attempts.lock().expect("release attempts");
+            let attempt = attempts.entry(id.clone()).or_default();
+            *attempt += 1;
+            *attempt
+        };
+        self.changed.notify_one();
+        // An absence-looking first response cannot terminate uncertainty.
+        if attempt == 1 {
+            return Ok(false);
+        }
+        let result = self.backend.release(id).await;
+        self.changed.notify_one();
+        result
+    }
+    async fn cleanup_grant(&self, _id: &LockId) -> crate::Result<LockCleanupOutcome> {
+        self.cleanup_calls.fetch_add(1, Ordering::SeqCst);
+        Ok(LockCleanupOutcome::AlreadyAbsent)
+    }
+    async fn refresh(&self, id: &LockId) -> crate::Result<bool> {
+        self.backend.refresh(id).await
+    }
+    async fn force_release(&self, id: &LockId) -> crate::Result<bool> {
+        self.backend.force_release(id).await
+    }
+    async fn check_status(&self, id: &LockId) -> crate::Result<Option<LockInfo>> {
+        self.backend.check_status(id).await
+    }
+    async fn get_stats(&self) -> crate::Result<LockStats> {
+        self.backend.get_stats().await
+    }
+    async fn close(&self) -> crate::Result<()> {
+        self.backend.close().await
+    }
+    async fn is_online(&self) -> bool {
+        true
+    }
+    async fn is_local(&self) -> bool {
+        true
+    }
+}
+
+#[tokio::test(start_paused = true)]
+async fn legacy_collapsed_error_cleans_hidden_grant_without_accepting_absence() {
+    let peer = Arc::new(LegacyHiddenClient {
+        backend: LocalClient::with_manager(Arc::new(GlobalLockManager::new())),
+        requested: Mutex::new(Vec::new()),
+        release_attempts: Mutex::new(std::collections::HashMap::new()),
+        cleanup_calls: AtomicUsize::new(0),
+        changed: Notify::new(),
+    });
+    let lock = DistributedLock::new("legacy-hidden".to_owned(), vec![peer.clone()], 1);
+    let results = lock
+        .acquire_guard_batch(
+            &[
+                ObjectKey::new("legacy-hidden", "granted"),
+                ObjectKey::new("legacy-hidden", "error"),
+            ],
+            "owner",
+            Duration::from_secs(5),
+            Duration::from_secs(30),
+        )
+        .await;
+    assert!(
+        results.iter().all(Result::is_err),
+        "the default collect error obscures the successful sibling"
+    );
+    tokio::time::timeout(Duration::from_millis(50), async {
+        loop {
+            let notified = peer.changed.notified();
+            if peer.release_attempts.lock().expect("release attempts").len() == 2 {
+                break;
+            }
+            notified.await;
+        }
+    })
+    .await
+    .expect("both uncertain IDs receive the first release pass");
+    assert_eq!(
+        peer.backend.list_lock_leases().await.len(),
+        1,
+        "false release does not prove this hidden grant absent"
+    );
+    assert_eq!(
+        peer.cleanup_calls.load(Ordering::SeqCst),
+        0,
+        "uncertainty must never use authoritative-absence cleanup"
+    );
+    tokio::time::timeout(Duration::from_secs(1), async {
+        loop {
+            let notified = peer.changed.notified();
+            if peer.backend.list_lock_leases().await.is_empty() {
+                break;
+            }
+            notified.await;
+        }
+    })
+    .await
+    .expect("exact-ID release is retried and clears the hidden grant before its TTL");
+    let requests = peer.requested.lock().expect("request IDs");
+    let releases = peer.release_attempts.lock().expect("release attempts");
+    assert_eq!(requests.len(), 2, "ambiguous requests are never reacquired");
+    assert!(
+        releases.keys().all(|id| requests.contains(id)),
+        "cleanup only uses the original request IDs"
+    );
+    assert!(releases.values().all(|&attempts| attempts >= 2));
+}
+
+#[tokio::test(start_paused = true)]
+async fn local_wait_deadline_is_fixed_before_scheduling_and_single_peer_keeps_total() {
+    let gate = Arc::new(Semaphore::new(0));
+    let local = TypedClient::new(Vec::new(), true, None);
+    *local.local_gate.lock().expect("set lookup gate") = Some(gate.clone());
+    let peers = vec![
+        local.clone(),
+        TypedClient::new(vec![Plan::Retryable], true, None),
+        TypedClient::new(vec![Plan::Retryable], true, None),
+        TypedClient::new(vec![Plan::Ambiguous], false, None),
+    ];
+    let lock = lock_for(&peers, 3);
+    let operation = tokio::spawn(async move {
+        lock.acquire_guard_batch(
+            &[ObjectKey::new("late-local", "key")],
+            "owner",
+            Duration::from_secs(5),
+            Duration::from_secs(30),
+        )
+        .await
+    });
+    peers[1].wait_calls(1).await;
+    peers[2].wait_calls(1).await;
+    tokio::time::advance(Duration::from_millis(1250)).await;
+    gate.add_permits(1);
+    let result = operation.await.expect("retry after the delayed local lookup");
+    assert!(result[0].is_ok());
+    assert_eq!(
+        local.calls.lock().expect("local calls").len(),
+        1,
+        "the expired first local round never calls acquire again with a fresh budget"
+    );
+    assert_eq!(peers[1].calls.lock().expect("peer calls").len(), 2);
+    assert_eq!(peers[2].calls.lock().expect("peer calls").len(), 2);
+    drop(result);
+
+    let single = TypedClient::new(Vec::new(), true, None);
+    let start = Instant::now();
+    let results = lock_for(std::slice::from_ref(&single), 1)
+        .acquire_guard_batch(
+            &[ObjectKey::new("single-local", "key")],
+            "owner",
+            Duration::from_secs(5),
+            Duration::from_secs(30),
+        )
+        .await;
+    assert!(results[0].is_ok());
+    assert_eq!(single.deadlines.lock().expect("single deadline")[0], start + Duration::from_secs(5));
+    drop(results);
+}
+
+#[tokio::test(start_paused = true)]
+async fn total_deadline_never_dispatches_after_a_delayed_peer_lookup() {
+    let gate = Arc::new(Semaphore::new(0));
+    let peer = TypedClient::new(Vec::new(), false, None);
+    *peer.local_gate.lock().expect("set lookup gate") = Some(gate.clone());
+    let result = lock_for(std::slice::from_ref(&peer), 1)
+        .acquire_guard_batch(
+            &[ObjectKey::new("no-late-dispatch", "key")],
+            "owner",
+            Duration::from_secs(1),
+            Duration::from_secs(30),
+        )
+        .await;
+    assert!(result[0].is_err());
+    gate.add_permits(1);
+    // The lookup task is now runnable; yield until its gate permit was consumed.
+    tokio::time::timeout(Duration::from_secs(1), async {
+        while gate.available_permits() != 0 {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("late lookup task resumes under its retained owner");
+    assert!(peer.calls.lock().expect("peer calls").is_empty(), "no acquire call starts after D_total");
+    assert!(peer.backend.list_lock_leases().await.is_empty());
+}

crates/lock/src/lib.rs

@@ -38,7 +38,7 @@ pub mod types;
 // Re-export main types for easy access
 pub use crate::{
     // Client interfaces
-    client::{LockCleanupOutcome, LockClient, local::LocalClient},
+    client::{BatchAcquireOutcome, LockCleanupOutcome, LockClient, local::LocalClient},
     distributed_lock::DistributedLockGuard,
     // Error types
     error::{LockError, Result},

crates/notify/src/config_manager.rs

@@ -159,7 +159,7 @@ pub(crate) fn notify_configuration_hint() -> String {
     format!(
         "No notify targets configured. Check {}=true and instance-scoped target env vars (for example {webhook_enable_primary} + {webhook_endpoint_primary} for arn:rustfs:sqs::primary:webhook). If using default queue_dir, ensure {} is writable.",
         rustfs_config::ENV_NOTIFY_ENABLE,
-        rustfs_config::EVENT_DEFAULT_DIR,
+        rustfs_targets::store::resolve_queue_dir(rustfs_config::EVENT_DEFAULT_DIR),
     )
 }
 

crates/obs/README.md

@@ -145,7 +145,7 @@ With OTLP → Collector → Prometheus, the time from a failure to an alert also
 
 | Variable                        | Default      | Description                                                                                                                                                                                                                                                                                                                                                           |
 |---------------------------------|--------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `RUSTFS_OBS_LOGGER_LEVEL`       | `info`       | Log level; `RUST_LOG` syntax supported                                                                                                                                                                                                                                                                                                                                |
+| `RUSTFS_OBS_LOGGER_LEVEL`       | `error`      | Log level; `RUST_LOG` syntax supported                                                                                                                                                                                                                                                                                                                                |
 | `RUSTFS_OBS_LOG_STDOUT_ENABLED` | `false`      | When file logging is active, also mirror to stdout                                                                                                                                                                                                                                                                                                                    |
 | `RUSTFS_OBS_LOG_DIRECTORY`      | _(empty)_    | **Directory for rolling log files. When empty, logs go to stdout only**                                                                                                                                                                                                                                                                                               |
 | `RUSTFS_OBS_LOG_FILENAME`       | `rustfs.log` | Base filename for rolling logs. Rotated archives include a high-precision timestamp and counter. With the default `RUSTFS_OBS_LOG_MATCH_MODE=suffix`, names look like `<timestamp>-<counter>.rustfs.log` (e.g., `20231027103001.123456-0.rustfs.log`); with `prefix`, they look like `rustfs.log.<timestamp>-<counter>` (e.g., `rustfs.log.20231027103001.123456-0`). |
@@ -157,23 +157,31 @@ With OTLP → Collector → Prometheus, the time from a failure to an alert also
 
 | Variable                                        | Default      | Description                                                 |
 |-------------------------------------------------|--------------|-------------------------------------------------------------|
-| `RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES`           | `2147483648` | Hard cap on total log directory size (2 GiB)                |
-| `RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES`     | `0`          | Per-file size cap; `0` = unlimited                          |
+| `RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES`           | `2147483648` | Byte budget (2 GiB), applied to rolled logs and to compressed archives separately; `0` = no budget. See [Disk usage bound](#disk-usage-bound) |
+| `RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES`     | unset        | Unset or unparsable: the active file rolls at 256 MiB, or at a quarter of the byte budget when that is smaller (at least 1 MiB). A value: the active file rolls at that size and the cleaner also processes rolled files above it. `0` = no size roll |
 | `RUSTFS_OBS_LOG_COMPRESS_OLD_FILES`             | `true`       | Compress files before deleting                              |
 | `RUSTFS_OBS_LOG_GZIP_COMPRESSION_LEVEL`         | `6`          | Gzip level `1` (fastest) – `9` (best)                       |
 | `RUSTFS_OBS_LOG_COMPRESSION_ALGORITHM`          | `zstd`       | Compression codec: `zstd` or `gzip`                         |
 | `RUSTFS_OBS_LOG_PARALLEL_COMPRESS`              | `true`       | Enable work-stealing parallel compression                   |
 | `RUSTFS_OBS_LOG_PARALLEL_WORKERS`               | `6`          | Number of cleaner worker threads                            |
-| `RUSTFS_OBS_LOG_ZSTD_COMPRESSION_LEVEL`         | `8`          | Zstd level `1` (fastest) – `21` (best ratio)               |
+| `RUSTFS_OBS_LOG_ZSTD_COMPRESSION_LEVEL`         | `8`          | Zstd level `1` (fastest) – `22` (best ratio); `0` = codec default |
 | `RUSTFS_OBS_LOG_ZSTD_FALLBACK_TO_GZIP`          | `true`       | Fallback to gzip when zstd compression fails                |
 | `RUSTFS_OBS_LOG_ZSTD_WORKERS`                   | `1`          | zstdmt worker threads per compression task                  |
 | `RUSTFS_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS` | `30`         | Delete `.gz` / `.zst` archives older than N days; `0` = keep forever |
 | `RUSTFS_OBS_LOG_EXCLUDE_PATTERNS`               | _(empty)_    | Comma-separated glob patterns to never clean up             |
 | `RUSTFS_OBS_LOG_DELETE_EMPTY_FILES`             | `true`       | Remove zero-byte files                                      |
-| `RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS`           | `3600`       | Minimum file age (seconds) before cleanup                   |
-| `RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS`       | `1800`       | How often the cleanup task runs (0.5 hours)                 |
+| `RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS`           | `3600`       | Minimum file age (seconds) before cleanup; not applied while rolled logs exceed the byte budget |
+| `RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS`       | `1800`       | How often the cleanup task runs (0.5 hours); a pass also runs after every roll |
 | `RUSTFS_OBS_LOG_DRY_RUN`                        | `false`      | Report deletions without actually removing files            |
 
+### Disk usage bound
+
+With the defaults the log directory holds at most the byte budget of rolled logs, the same budget of compressed archives, and one active file of up to 256 MiB. The active file is rolled by size as well as by time, and the cleaner runs a pass after every roll in addition to its interval, so the budget is enforced when a file is rolled. While the rolled logs exceed the budget the cleaner processes the oldest ones, as many as the budget requires, even if they are younger than `RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS`; `RUSTFS_OBS_LOG_KEEP_FILES` still applies only to files past that age.
+
+The bound is not instantaneous: the directory can exceed it by what is written while one cleanup pass runs, archives created by a pass are trimmed by the next one, and a cleaner that compresses more slowly than logs are produced falls behind. Setting `RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES=0` removes the budget and `RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES=0` turns size rolling off, which leaves the active file bounded only by the rotation period. Neither setting limits how fast logs are produced.
+
+The cleaner recognizes only its own process's active file. Do not let two processes write log files that match the same pattern in one directory: a file another process is still writing looks like a rolled log, and the minimum file age no longer shields it once the budget is exceeded.
+
 ---
 
 ## Cleaner & Rotation Metrics

crates/obs/src/cleaner/core.rs

@@ -149,6 +149,7 @@ impl LogCleaner {
 
         let LogScanResult {
             mut logs,
+            recent_logs,
             mut compressed_archives,
         } = scan_log_directory(
             &self.log_dir,
@@ -161,8 +162,9 @@ impl LogCleaner {
             self.dry_run,
         )?;
 
+        logs.sort_by_key(|f| f.modified);
+        let mut to_delete = Vec::new();
         if !logs.is_empty() {
-            logs.sort_by_key(|f| f.modified);
             let total_size: u64 = logs.iter().map(|f| f.size).sum();
 
             info!(
@@ -178,16 +180,35 @@ impl LogCleaner {
 
             // Select the oldest files first, then additionally trim any files
             // that still violate configured size constraints.
-            let to_delete = self.select_files_to_process(&logs, total_size);
-            if !to_delete.is_empty() {
-                let (deleted, freed) = if self.parallel_compress && self.compress_old_files {
-                    self.parallel_stealing_compress(&to_delete)?
-                } else {
-                    self.serial_compress_and_delete(&to_delete)?
-                };
-                total_deleted += deleted;
-                total_freed += freed;
-            }
+            to_delete = self.select_files_to_process(&logs, total_size);
+        }
+
+        // `min_file_age_seconds` keeps freshly rolled files readable for a
+        // while, but those files fill the disk like any other. Once the rolled
+        // files that would remain exceed the byte budget, the budget wins:
+        // the oldest of them go, fresh or not, and only as many as it takes.
+        let over_budget = self.select_files_over_budget(&logs, &to_delete, recent_logs);
+        if !over_budget.is_empty() {
+            warn!(
+                event = EVENT_LOG_CLEANER_STATE,
+                component = LOG_COMPONENT_OBS,
+                subsystem = LOG_SUBSYSTEM_LOG_CLEANER,
+                result = "byte_budget_overrides_min_file_age",
+                log_files = over_budget.len(),
+                max_total_size_bytes = self.max_total_size_bytes,
+                "log cleaner state changed"
+            );
+            to_delete.extend(over_budget);
+        }
+
+        if !to_delete.is_empty() {
+            let (deleted, freed) = if self.parallel_compress && self.compress_old_files {
+                self.parallel_stealing_compress(&to_delete)?
+            } else {
+                self.serial_compress_and_delete(&to_delete)?
+            };
+            total_deleted += deleted;
+            total_freed += freed;
         }
 
         if !compressed_archives.is_empty() {
@@ -217,6 +238,44 @@ impl LogCleaner {
         Ok((total_deleted, total_freed))
     }
 
+    /// Byte-budget trim that also counts the rolled files still inside
+    /// `min_file_age_seconds`.
+    ///
+    /// `aged` is sorted oldest first and `selected` is what the regular
+    /// selection already took from it. When the files that would remain,
+    /// aged and recent together, exceed `max_total_size_bytes`, the oldest of
+    /// them are returned until the rest fits. `keep_files` and the per-file
+    /// limit are not applied to recent files: only the bytes matter here.
+    pub(super) fn select_files_over_budget(
+        &self,
+        aged: &[FileInfo],
+        selected: &[FileInfo],
+        mut recent: Vec<FileInfo>,
+    ) -> Vec<FileInfo> {
+        if self.max_total_size_bytes == 0 || recent.is_empty() {
+            return Vec::new();
+        }
+        recent.sort_by_key(|file| file.modified);
+        let selected: std::collections::HashSet<&std::path::Path> = selected.iter().map(|file| file.path.as_path()).collect();
+        // Aged files are older than every recent one, so chaining keeps the
+        // oldest-first order.
+        let remaining: Vec<&FileInfo> = aged
+            .iter()
+            .filter(|file| !selected.contains(file.path.as_path()))
+            .chain(recent.iter())
+            .collect();
+        let mut occupied = remaining.iter().fold(0u64, |total, file| total.saturating_add(file.size));
+        let mut over_budget = Vec::new();
+        for file in remaining {
+            if occupied <= self.max_total_size_bytes {
+                break;
+            }
+            occupied = occupied.saturating_sub(file.size);
+            over_budget.push(file.clone());
+        }
+        over_budget
+    }
+
     /// Choose regular log files that should be compressed and/or deleted.
     ///
     /// The `files` slice must already be sorted from oldest to newest. The

crates/obs/src/cleaner/mod.rs

@@ -444,6 +444,171 @@ mod tests {
         Ok(())
     }
 
+    /// Rolled files inside `min_file_age_seconds` occupy the disk too: once
+    /// they push the directory over the byte budget, the oldest are processed
+    /// regardless of age until the budget holds again.
+    #[test]
+    fn byte_budget_overrides_min_age_for_fresh_logs() -> std::io::Result<()> {
+        let tmp = TempDir::new()?;
+        let dir = tmp.path().to_path_buf();
+        for i in 0..6u64 {
+            let name = format!("app.log.2024-01-0{i}");
+            create_log_file(&dir, &name, 1024)?;
+            // All well inside the age gate, oldest first by name.
+            set_mtime_ago(&dir.join(&name), Duration::from_secs(60 - i * 5));
+        }
+        create_log_file(&dir, "app.log", 1024)?;
+
+        let cleaner = LogCleaner::builder(dir.clone(), "app.log.".to_string(), "app.log".to_string())
+            .match_mode(FileMatchMode::Prefix)
+            .keep_files(30)
+            .max_total_size_bytes(3 * 1024)
+            .min_file_age_seconds(3600)
+            .build();
+        let (deleted, freed) = cleaner.cleanup()?;
+
+        assert_eq!((deleted, freed), (3, 3 * 1024), "the three oldest fresh logs go; the budget then holds");
+        for i in 0..3 {
+            assert!(!dir.join(format!("app.log.2024-01-0{i}")).exists());
+        }
+        for i in 3..6 {
+            assert!(dir.join(format!("app.log.2024-01-0{i}")).exists(), "the newest fresh logs stay");
+        }
+        assert!(dir.join("app.log").exists(), "the active file is never touched");
+        Ok(())
+    }
+
+    /// Within the byte budget the age gate is unchanged: fresh logs are not
+    /// touched, even above `keep_files`.
+    #[test]
+    fn min_age_still_protects_fresh_logs_within_the_byte_budget() -> std::io::Result<()> {
+        let tmp = TempDir::new()?;
+        let dir = tmp.path().to_path_buf();
+        for i in 0..4 {
+            create_log_file(&dir, &format!("app.log.2024-01-0{i}"), 1024)?;
+        }
+
+        let cleaner = LogCleaner::builder(dir.clone(), "app.log.".to_string(), "app.log".to_string())
+            .match_mode(FileMatchMode::Prefix)
+            .keep_files(1)
+            .max_total_size_bytes(4 * 1024)
+            .min_file_age_seconds(3600)
+            .build();
+        assert_eq!(cleaner.cleanup()?, (0, 0), "exactly at the budget nothing is over it");
+        for i in 0..4 {
+            assert!(dir.join(format!("app.log.2024-01-0{i}")).exists());
+        }
+
+        // No byte budget configured: fresh files are never counted.
+        let unbounded = LogCleaner::builder(dir, "app.log.".to_string(), "app.log".to_string())
+            .match_mode(FileMatchMode::Prefix)
+            .keep_files(1)
+            .max_total_size_bytes(0)
+            .min_file_age_seconds(3600)
+            .build();
+        assert_eq!(unbounded.cleanup()?, (0, 0));
+        Ok(())
+    }
+
+    /// The budget override is a byte trim only: `keep_files` is not applied
+    /// to fresh files, so one byte over the budget costs one file, not all
+    /// but `keep_files` of them.
+    #[test]
+    fn byte_budget_override_removes_only_what_the_budget_needs() -> std::io::Result<()> {
+        let tmp = TempDir::new()?;
+        let dir = tmp.path().to_path_buf();
+        for i in 0..5u64 {
+            let name = format!("app.log.2024-01-0{i}");
+            create_log_file(&dir, &name, 1024)?;
+            set_mtime_ago(&dir.join(&name), Duration::from_secs(50 - i * 10));
+        }
+
+        let cleaner = LogCleaner::builder(dir.clone(), "app.log.".to_string(), "app.log".to_string())
+            .match_mode(FileMatchMode::Prefix)
+            .keep_files(1)
+            .max_single_file_size_bytes(512)
+            .max_total_size_bytes(4 * 1024)
+            .min_file_age_seconds(3600)
+            .build();
+        let (deleted, freed) = cleaner.cleanup()?;
+
+        assert_eq!(
+            (deleted, freed),
+            (1, 1024),
+            "neither keep_files nor the per-file limit applies to fresh files"
+        );
+        assert!(!dir.join("app.log.2024-01-00").exists(), "the oldest fresh file goes");
+        for i in 1..5 {
+            assert!(dir.join(format!("app.log.2024-01-0{i}")).exists());
+        }
+        Ok(())
+    }
+
+    /// Aged files keep their regular selection; the byte trim then counts the
+    /// fresh files and takes the oldest remaining files first.
+    #[test]
+    fn regular_selection_and_budget_override_compose() -> std::io::Result<()> {
+        let tmp = TempDir::new()?;
+        let dir = tmp.path().to_path_buf();
+        for i in 0..3u64 {
+            let name = format!("app.log.aged-{i}");
+            create_log_file(&dir, &name, 1024)?;
+            set_mtime_ago(&dir.join(&name), Duration::from_secs((5 - i) * 3600));
+        }
+        for i in 0..3u64 {
+            let name = format!("app.log.fresh-{i}");
+            create_log_file(&dir, &name, 1024)?;
+            set_mtime_ago(&dir.join(&name), Duration::from_secs(30 - i * 10));
+        }
+
+        // keep_files(2) takes aged-0. Five files remain against a budget of
+        // three, so the two oldest of them go as well: aged-1 and aged-2.
+        let cleaner = LogCleaner::builder(dir.clone(), "app.log.".to_string(), "app.log".to_string())
+            .match_mode(FileMatchMode::Prefix)
+            .keep_files(2)
+            .max_total_size_bytes(3 * 1024)
+            .min_file_age_seconds(3600)
+            .build();
+        let (deleted, freed) = cleaner.cleanup()?;
+
+        assert_eq!((deleted, freed), (3, 3 * 1024));
+        for i in 0..3 {
+            assert!(!dir.join(format!("app.log.aged-{i}")).exists());
+            assert!(dir.join(format!("app.log.fresh-{i}")).exists());
+        }
+        Ok(())
+    }
+
+    /// Aged and fresh logs share one budget: aged files alone fit, the fresh
+    /// ones put the directory over it, and the oldest files overall go first.
+    #[test]
+    fn aged_and_fresh_logs_share_the_byte_budget() -> std::io::Result<()> {
+        let tmp = TempDir::new()?;
+        let dir = tmp.path().to_path_buf();
+        create_log_file(&dir, "app.log.aged-0", 1024)?;
+        create_log_file(&dir, "app.log.aged-1", 1024)?;
+        set_mtime_ago(&dir.join("app.log.aged-0"), Duration::from_secs(3 * 3600));
+        set_mtime_ago(&dir.join("app.log.aged-1"), Duration::from_secs(2 * 3600));
+        create_log_file(&dir, "app.log.fresh-0", 1024)?;
+        create_log_file(&dir, "app.log.fresh-1", 1024)?;
+        set_mtime_ago(&dir.join("app.log.fresh-0"), Duration::from_secs(30));
+
+        let cleaner = LogCleaner::builder(dir.clone(), "app.log.".to_string(), "app.log".to_string())
+            .match_mode(FileMatchMode::Prefix)
+            .keep_files(30)
+            .max_total_size_bytes(2 * 1024)
+            .min_file_age_seconds(3600)
+            .build();
+        let (deleted, _) = cleaner.cleanup()?;
+
+        assert_eq!(deleted, 2);
+        assert!(!dir.join("app.log.aged-0").exists());
+        assert!(!dir.join("app.log.aged-1").exists());
+        assert!(dir.join("app.log.fresh-0").exists());
+        assert!(dir.join("app.log.fresh-1").exists());
+        Ok(())
+    }
+
     #[test]
     fn active_file_matching_pattern_is_protected() -> std::io::Result<()> {
         let tmp = TempDir::new()?;

crates/obs/src/cleaner/scanner.rs

@@ -48,6 +48,10 @@ const ORPHAN_TMP_GRACE_SECONDS: u64 = 60;
 pub(super) struct LogScanResult {
     /// Regular log files eligible for deletion/compression.
     pub logs: Vec<FileInfo>,
+    /// Regular log files younger than `min_file_age_seconds`. They occupy the
+    /// directory like any other rolled file, so the caller counts them against
+    /// the byte budget and may process them when that budget is exceeded.
+    pub recent_logs: Vec<FileInfo>,
     /// Already compressed files eligible for expiry deletion.
     pub compressed_archives: Vec<FileInfo>,
 }
@@ -83,11 +87,13 @@ pub(super) fn scan_log_directory(
     if file_pattern.is_empty() {
         return Ok(LogScanResult {
             logs: Vec::new(),
+            recent_logs: Vec::new(),
             compressed_archives: Vec::new(),
         });
     }
 
     let mut logs = Vec::new();
+    let mut recent_logs = Vec::new();
     let mut compressed_archives = Vec::new();
     let now = SystemTime::now();
 
@@ -99,6 +105,7 @@ pub(super) fn scan_log_directory(
             // as "no files found" instead of failing the whole cleanup pass.
             return Ok(LogScanResult {
                 logs,
+                recent_logs,
                 compressed_archives,
             });
         }
@@ -230,10 +237,7 @@ pub(super) fn scan_log_directory(
         // 6. Age gate regular logs only.
         // Compressed files deliberately bypass this check because archive
         // expiry is driven by a dedicated retention horizon in the caller.
-        if !is_compressed && !is_old_enough_for_cleanup(age, min_file_age_seconds) {
-            // Too young to be touched.
-            continue;
-        }
+        let recent = !is_compressed && !is_old_enough_for_cleanup(age, min_file_age_seconds);
 
         let info = FileInfo {
             path,
@@ -244,6 +248,9 @@ pub(super) fn scan_log_directory(
 
         if is_compressed {
             compressed_archives.push(info);
+        } else if recent {
+            // Too young to be touched while the directory is within budget.
+            recent_logs.push(info);
         } else {
             logs.push(info);
         }
@@ -251,6 +258,7 @@ pub(super) fn scan_log_directory(
 
     Ok(LogScanResult {
         logs,
+        recent_logs,
         compressed_archives,
     })
 }
@@ -311,6 +319,42 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn fresh_nonempty_log_is_reported_as_recent_not_as_a_candidate() -> std::io::Result<()> {
+        let tmp = TempDir::new()?;
+        let dir = tmp.path();
+        std::fs::write(dir.join("fresh.rustfs.log"), b"fresh")?;
+        std::fs::write(dir.join("aged.rustfs.log"), b"aged!!")?;
+        let old = SystemTime::now() - Duration::from_secs(7200);
+        File::options()
+            .write(true)
+            .open(dir.join("aged.rustfs.log"))?
+            .set_modified(old)?;
+        std::fs::write(dir.join("active.rustfs.log"), b"active")?;
+
+        let result = scan_log_directory(
+            dir,
+            ".rustfs.log",
+            Some("active.rustfs.log"),
+            FileMatchMode::Suffix,
+            &[],
+            3600,
+            true,
+            false,
+        )?;
+
+        let names = |files: &[FileInfo]| -> Vec<String> {
+            files
+                .iter()
+                .map(|file| file.path.file_name().expect("file name").to_string_lossy().into_owned())
+                .collect()
+        };
+        assert_eq!(names(&result.logs), vec!["aged.rustfs.log"]);
+        assert_eq!(names(&result.recent_logs), vec!["fresh.rustfs.log"], "the active file is in neither list");
+        assert_eq!(result.recent_logs[0].size, 5);
+        Ok(())
+    }
+
     #[test]
     fn orphan_tmp_archive_is_deleted() -> std::io::Result<()> {
         let tmp = TempDir::new()?;

crates/obs/src/config.rs

@@ -26,22 +26,21 @@ use rustfs_config::observability::{
     DEFAULT_OBS_ENVIRONMENT_PRODUCTION, DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS, DEFAULT_OBS_LOG_COMPRESS_OLD_FILES,
     DEFAULT_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS, DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM, DEFAULT_OBS_LOG_DELETE_EMPTY_FILES,
     DEFAULT_OBS_LOG_DRY_RUN, DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL, DEFAULT_OBS_LOG_MATCH_MODE,
-    DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS,
-    DEFAULT_OBS_LOG_PARALLEL_COMPRESS, DEFAULT_OBS_LOG_PARALLEL_WORKERS, DEFAULT_OBS_LOG_ZSTD_COMPRESSION_LEVEL,
-    DEFAULT_OBS_LOG_ZSTD_FALLBACK_TO_GZIP, DEFAULT_OBS_LOG_ZSTD_WORKERS, ENV_OBS_ENDPOINT, ENV_OBS_ENDPOINT_HEADERS,
-    ENV_OBS_ENDPOINT_LOGS_HEADERS, ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS, ENV_OBS_ENDPOINT_METRICS_HEADERS,
-    ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS, ENV_OBS_ENDPOINT_TIMEOUT_MILLIS, ENV_OBS_ENDPOINT_TRACES_HEADERS,
-    ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS, ENV_OBS_ENVIRONMENT, ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS,
-    ENV_OBS_LOG_COMPRESS_OLD_FILES, ENV_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS, ENV_OBS_LOG_COMPRESSION_ALGORITHM,
-    ENV_OBS_LOG_DELETE_EMPTY_FILES, ENV_OBS_LOG_DIRECTORY, ENV_OBS_LOG_DRY_RUN, ENV_OBS_LOG_ENDPOINT,
-    ENV_OBS_LOG_EXCLUDE_PATTERNS, ENV_OBS_LOG_FILENAME, ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL, ENV_OBS_LOG_KEEP_FILES,
-    ENV_OBS_LOG_MATCH_MODE, ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES,
-    ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, ENV_OBS_LOG_PARALLEL_COMPRESS, ENV_OBS_LOG_PARALLEL_WORKERS, ENV_OBS_LOG_ROTATION_TIME,
-    ENV_OBS_LOG_STDOUT_ENABLED, ENV_OBS_LOG_ZSTD_COMPRESSION_LEVEL, ENV_OBS_LOG_ZSTD_FALLBACK_TO_GZIP, ENV_OBS_LOG_ZSTD_WORKERS,
-    ENV_OBS_LOGGER_LEVEL, ENV_OBS_LOGS_EXPORT_ENABLED, ENV_OBS_METER_INTERVAL, ENV_OBS_METRIC_ENDPOINT,
-    ENV_OBS_METRICS_EXPORT_ENABLED, ENV_OBS_PROFILING_ENDPOINT, ENV_OBS_PROFILING_EXPORT_ENABLED, ENV_OBS_SAMPLE_RATIO,
-    ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_TLS_CA_FILE, ENV_OBS_TRACE_ENDPOINT, ENV_OBS_TRACES_EXPORT_ENABLED,
-    ENV_OBS_USE_STDOUT,
+    DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS, DEFAULT_OBS_LOG_PARALLEL_COMPRESS,
+    DEFAULT_OBS_LOG_PARALLEL_WORKERS, DEFAULT_OBS_LOG_ZSTD_COMPRESSION_LEVEL, DEFAULT_OBS_LOG_ZSTD_FALLBACK_TO_GZIP,
+    DEFAULT_OBS_LOG_ZSTD_WORKERS, ENV_OBS_ENDPOINT, ENV_OBS_ENDPOINT_HEADERS, ENV_OBS_ENDPOINT_LOGS_HEADERS,
+    ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS, ENV_OBS_ENDPOINT_METRICS_HEADERS, ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS,
+    ENV_OBS_ENDPOINT_TIMEOUT_MILLIS, ENV_OBS_ENDPOINT_TRACES_HEADERS, ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS,
+    ENV_OBS_ENVIRONMENT, ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS, ENV_OBS_LOG_COMPRESS_OLD_FILES,
+    ENV_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS, ENV_OBS_LOG_COMPRESSION_ALGORITHM, ENV_OBS_LOG_DELETE_EMPTY_FILES,
+    ENV_OBS_LOG_DIRECTORY, ENV_OBS_LOG_DRY_RUN, ENV_OBS_LOG_ENDPOINT, ENV_OBS_LOG_EXCLUDE_PATTERNS, ENV_OBS_LOG_FILENAME,
+    ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL, ENV_OBS_LOG_KEEP_FILES, ENV_OBS_LOG_MATCH_MODE, ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES,
+    ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, ENV_OBS_LOG_PARALLEL_COMPRESS,
+    ENV_OBS_LOG_PARALLEL_WORKERS, ENV_OBS_LOG_ROTATION_TIME, ENV_OBS_LOG_STDOUT_ENABLED, ENV_OBS_LOG_ZSTD_COMPRESSION_LEVEL,
+    ENV_OBS_LOG_ZSTD_FALLBACK_TO_GZIP, ENV_OBS_LOG_ZSTD_WORKERS, ENV_OBS_LOGGER_LEVEL, ENV_OBS_LOGS_EXPORT_ENABLED,
+    ENV_OBS_METER_INTERVAL, ENV_OBS_METRIC_ENDPOINT, ENV_OBS_METRICS_EXPORT_ENABLED, ENV_OBS_PROFILING_ENDPOINT,
+    ENV_OBS_PROFILING_EXPORT_ENABLED, ENV_OBS_SAMPLE_RATIO, ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_TLS_CA_FILE,
+    ENV_OBS_TRACE_ENDPOINT, ENV_OBS_TRACES_EXPORT_ENABLED, ENV_OBS_USE_STDOUT,
 };
 use rustfs_config::{
     APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_FILENAME,
@@ -189,9 +188,13 @@ pub struct OtelConfig {
     pub log_keep_files: Option<usize>,
 
     // ── Log cleanup ───────────────────────────────────────────────────────────
-    /// Hard ceiling on the total size (bytes) of all log files (default: 2 GiB).
+    /// Byte budget for rolled log files, and separately for compressed
+    /// archives (default: 2 GiB); `0` means no budget. The active file is not
+    /// counted.
     pub log_max_total_size_bytes: Option<u64>,
-    /// Per-file size ceiling (bytes); `0` means unlimited (default: `0`).
+    /// Per-file size ceiling (bytes). Unset or unparsable: the active file
+    /// rolls at 256 MiB (less under a small byte budget) and the cleaner
+    /// applies no per-file trim. `0`: no size roll and no trim.
     pub log_max_single_file_size_bytes: Option<u64>,
     /// Compress eligible files with gzip before deletion (default: `true`).
     pub log_compress_old_files: Option<bool>,
@@ -344,10 +347,8 @@ impl OtelConfig {
             log_keep_files,
             // Log cleanup
             log_max_total_size_bytes: Some(get_env_u64(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES)),
-            log_max_single_file_size_bytes: Some(get_env_u64(
-                ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES,
-                DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES,
-            )),
+            // Unset stays `None`: the appender and the cleaner default differently.
+            log_max_single_file_size_bytes: get_env_opt_u64(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES),
             log_compress_old_files: Some(get_env_bool(ENV_OBS_LOG_COMPRESS_OLD_FILES, DEFAULT_OBS_LOG_COMPRESS_OLD_FILES)),
             log_gzip_compression_level: Some(get_env_u64(
                 ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL,
@@ -480,6 +481,30 @@ mod tests {
         OtelConfig::extract_otel_config_from_env(None).profiling_export_enabled
     }
 
+    #[test]
+    fn single_file_size_limit_distinguishes_unset_from_an_explicit_zero() {
+        with_profiling_env_lock(|| {
+            temp_env::with_var_unset(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, || {
+                let config = OtelConfig::extract_otel_config_from_env(None);
+                assert_eq!(config.log_max_single_file_size_bytes, None, "unset leaves the default to each consumer");
+            });
+            temp_env::with_var(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, Some("0"), || {
+                let config = OtelConfig::extract_otel_config_from_env(None);
+                assert_eq!(config.log_max_single_file_size_bytes, Some(0), "an explicit 0 is kept as unlimited");
+            });
+            for unparsable in ["", "256MiB"] {
+                temp_env::with_var(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, Some(unparsable), || {
+                    let config = OtelConfig::extract_otel_config_from_env(None);
+                    assert_eq!(config.log_max_single_file_size_bytes, None, "{unparsable:?} is treated as unset");
+                });
+            }
+            temp_env::with_var(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, Some("1048576"), || {
+                let config = OtelConfig::extract_otel_config_from_env(None);
+                assert_eq!(config.log_max_single_file_size_bytes, Some(1_048_576));
+            });
+        });
+    }
+
     #[test]
     fn service_version_uses_application_version_when_environment_is_unset() {
         with_profiling_env_lock(|| {

crates/obs/src/telemetry/local.rs

@@ -40,12 +40,13 @@ use crate::telemetry::filter::{build_env_filter, effective_verbose_logging, pyro
 use crate::telemetry::rolling::{RollingAppender, Rotation};
 use metrics::counter;
 use rustfs_config::observability::{
-    DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS, DEFAULT_OBS_LOG_COMPRESS_OLD_FILES, DEFAULT_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS,
-    DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM, DEFAULT_OBS_LOG_DELETE_EMPTY_FILES, DEFAULT_OBS_LOG_DRY_RUN,
-    DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL, DEFAULT_OBS_LOG_MATCH_MODE, DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES,
-    DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS, DEFAULT_OBS_LOG_PARALLEL_COMPRESS,
-    DEFAULT_OBS_LOG_PARALLEL_WORKERS, DEFAULT_OBS_LOG_SPAN_EVENTS, DEFAULT_OBS_LOG_ZSTD_COMPRESSION_LEVEL,
-    DEFAULT_OBS_LOG_ZSTD_FALLBACK_TO_GZIP, DEFAULT_OBS_LOG_ZSTD_WORKERS, ENV_OBS_LOG_SPAN_EVENTS,
+    DEFAULT_OBS_LOG_ACTIVE_FILE_ROLL_SIZE_BYTES, DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS, DEFAULT_OBS_LOG_COMPRESS_OLD_FILES,
+    DEFAULT_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS, DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM, DEFAULT_OBS_LOG_DELETE_EMPTY_FILES,
+    DEFAULT_OBS_LOG_DRY_RUN, DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL, DEFAULT_OBS_LOG_MATCH_MODE,
+    DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS,
+    DEFAULT_OBS_LOG_PARALLEL_COMPRESS, DEFAULT_OBS_LOG_PARALLEL_WORKERS, DEFAULT_OBS_LOG_SPAN_EVENTS,
+    DEFAULT_OBS_LOG_ZSTD_COMPRESSION_LEVEL, DEFAULT_OBS_LOG_ZSTD_FALLBACK_TO_GZIP, DEFAULT_OBS_LOG_ZSTD_WORKERS,
+    ENV_OBS_LOG_SPAN_EVENTS,
 };
 use rustfs_config::{APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_ROTATION_TIME};
 use rustfs_utils::get_env_str;
@@ -159,6 +160,58 @@ fn validate_distinct_file_identities(
     Ok(())
 }
 
+/// Smallest roll size derived from a small byte budget.
+const MIN_DERIVED_ROLL_SIZE_BYTES: u64 = 1024 * 1024;
+
+/// Size at which the active log file is rolled. An explicit limit is used as
+/// is, and an explicit `0` keeps size rolling off. An unset limit rolls at the
+/// shipped default so the active file is bounded, lowered to a quarter of the
+/// byte budget when the budget is small: a rolled file larger than the whole
+/// budget would be removed by the pass its own roll starts.
+pub(crate) fn resolve_active_file_roll_size(config: &OtelConfig) -> u64 {
+    if let Some(configured) = config.log_max_single_file_size_bytes {
+        return configured;
+    }
+    match config
+        .log_max_total_size_bytes
+        .unwrap_or(DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES)
+    {
+        0 => DEFAULT_OBS_LOG_ACTIVE_FILE_ROLL_SIZE_BYTES,
+        budget => DEFAULT_OBS_LOG_ACTIVE_FILE_ROLL_SIZE_BYTES.min((budget / 4).max(MIN_DERIVED_ROLL_SIZE_BYTES)),
+    }
+}
+
+/// Per-file limit above which the cleaner processes a rolled file. Unset stays
+/// off: the appender's default roll size is not a reason to trim files that an
+/// earlier release rolled by time only.
+pub(crate) fn resolve_cleanup_single_file_limit(config: &OtelConfig) -> u64 {
+    config
+        .log_max_single_file_size_bytes
+        .unwrap_or(DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES)
+}
+
+/// Open the rolling appender for the configured log file together with the
+/// signal it raises after every roll. The caller hands that signal to
+/// [`spawn_cleanup_task`].
+pub(crate) fn open_rolling_appender(
+    config: &OtelConfig,
+    log_directory: &str,
+    log_filename: &str,
+    rotation: Rotation,
+    match_mode: FileMatchMode,
+) -> std::io::Result<(RollingAppender, Arc<tokio::sync::Notify>)> {
+    let roll_signal = Arc::new(tokio::sync::Notify::new());
+    let appender = RollingAppender::new(
+        log_directory,
+        log_filename.to_string(),
+        rotation,
+        resolve_active_file_roll_size(config),
+        match_mode,
+    )?
+    .with_roll_signal(roll_signal.clone());
+    Ok((appender, roll_signal))
+}
+
 fn resolve_log_cleanup_interval_seconds(config: &OtelConfig) -> u64 {
     match config.log_cleanup_interval_seconds {
         Some(0) => {
@@ -451,12 +504,7 @@ fn init_file_logging_internal(
         _ => Rotation::Daily,
     };
 
-    let max_single_file_size = config
-        .log_max_single_file_size_bytes
-        .unwrap_or(DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES);
-
-    let file_appender =
-        RollingAppender::new(log_directory, log_filename.to_string(), rotation, max_single_file_size, match_mode)?;
+    let (file_appender, roll_signal) = open_rolling_appender(config, log_directory, log_filename, rotation, match_mode)?;
     validate_stdout_sink(&file_appender)?;
 
     let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
@@ -497,7 +545,7 @@ fn init_file_logging_internal(
     set_observability_metric_enabled(false);
 
     // ── 5. Start background cleanup task ─────────────────────────────────────
-    let cleanup_handle = spawn_cleanup_task(config, log_directory, log_filename, keep_files);
+    let cleanup_handle = spawn_cleanup_task(config, log_directory, log_filename, keep_files, roll_signal);
 
     info!(
         event = EVENT_LOCAL_LOGGING_STATE,
@@ -606,6 +654,10 @@ pub(super) fn emit_file_logging_fallback_warning(log_directory: &str, error: &Te
 
 // ─── Cleanup task ─────────────────────────────────────────────────────────────
 
+/// Shortest pause between two cleanup passes started by rolls. A pass that the
+/// timer starts is never delayed by it.
+const ROLL_TRIGGERED_CLEANUP_SPACING: Duration = Duration::from_secs(1);
+
 /// Spawn a background task that periodically cleans up old log files.
 ///
 /// All cleanup parameters are derived from [`OtelConfig`] fields, with
@@ -623,6 +675,9 @@ pub(super) fn emit_file_logging_fallback_warning(log_directory: &str, error: &Te
 /// * `log_filename` - Base filename (used as the file prefix for matching).
 /// * `keep_files` - Legacy keep-files count; used as fallback when the new
 ///   `log_keep_count` field is absent.
+/// * `roll_signal` - Notified by the appender after every roll. The loop then
+///   runs a pass right away, so the byte budget is enforced when a file is
+///   rolled instead of up to one cleanup interval later.
 ///
 /// # Returns
 /// A [`tokio::task::JoinHandle`] for the spawned cleanup loop.
@@ -631,6 +686,7 @@ pub fn spawn_cleanup_task(
     log_directory: &str,
     log_filename: &str,
     keep_files: usize,
+    roll_signal: Arc<tokio::sync::Notify>,
 ) -> tokio::task::JoinHandle<()> {
     let log_dir = std::path::PathBuf::from(log_directory);
     // Use suffix matching for log files like `2026-03-01-06-21.rustfs.log`
@@ -645,9 +701,7 @@ pub fn spawn_cleanup_task(
     let max_total_size = config
         .log_max_total_size_bytes
         .unwrap_or(DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES);
-    let max_single_file_size = config
-        .log_max_single_file_size_bytes
-        .unwrap_or(DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES);
+    let max_single_file_size = resolve_cleanup_single_file_limit(config);
     let compress = config.log_compress_old_files.unwrap_or(DEFAULT_OBS_LOG_COMPRESS_OLD_FILES);
     let gzip_level = config
         .log_gzip_compression_level
@@ -740,9 +794,16 @@ pub fn spawn_cleanup_task(
     tokio::spawn(async move {
         let mut interval = tokio::time::interval(Duration::from_secs(cleanup_interval));
         loop {
-            // Wait for the next scheduled tick before dispatching another pass.
-            // The blocking filesystem work runs on a dedicated blocking thread.
-            interval.tick().await;
+            // Wait for the next scheduled tick, or for the appender to roll a
+            // file, before dispatching another pass. The blocking filesystem
+            // work runs on a dedicated blocking thread.
+            tokio::select! {
+                _ = interval.tick() => {}
+                _ = roll_signal.notified() => {
+                    // Rolls can come in bursts; one pass covers all of them.
+                    tokio::time::sleep(ROLL_TRIGGERED_CLEANUP_SPACING).await;
+                }
+            }
             let cleaner_clone = cleaner.clone();
             let result = tokio::task::spawn_blocking(move || cleaner_clone.cleanup()).await;
 
@@ -1253,6 +1314,153 @@ mod tests {
         assert_eq!(resolve_log_cleanup_interval_seconds(&config), DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS);
     }
 
+    #[test]
+    fn active_file_rolls_by_size_unless_explicitly_disabled() {
+        let unset = OtelConfig {
+            log_max_single_file_size_bytes: None,
+            log_max_total_size_bytes: None,
+            ..OtelConfig::default()
+        };
+        assert_eq!(resolve_active_file_roll_size(&unset), DEFAULT_OBS_LOG_ACTIVE_FILE_ROLL_SIZE_BYTES);
+        assert_eq!(DEFAULT_OBS_LOG_ACTIVE_FILE_ROLL_SIZE_BYTES, 256 * 1024 * 1024);
+        assert_eq!(
+            resolve_cleanup_single_file_limit(&unset),
+            0,
+            "the appender's default roll size is not a per-file trim for the cleaner"
+        );
+
+        let disabled = OtelConfig {
+            log_max_single_file_size_bytes: Some(0),
+            ..OtelConfig::default()
+        };
+        assert_eq!(resolve_active_file_roll_size(&disabled), 0, "an explicit 0 keeps size rolling off");
+        assert_eq!(resolve_cleanup_single_file_limit(&disabled), 0);
+
+        let configured = OtelConfig {
+            log_max_single_file_size_bytes: Some(64 * 1024 * 1024),
+            log_max_total_size_bytes: Some(1024),
+            ..OtelConfig::default()
+        };
+        assert_eq!(
+            resolve_active_file_roll_size(&configured),
+            64 * 1024 * 1024,
+            "an explicit limit is never derived"
+        );
+        assert_eq!(resolve_cleanup_single_file_limit(&configured), 64 * 1024 * 1024);
+    }
+
+    #[test]
+    fn default_roll_size_follows_a_small_byte_budget() {
+        let with_budget = |budget: u64| OtelConfig {
+            log_max_single_file_size_bytes: None,
+            log_max_total_size_bytes: Some(budget),
+            ..OtelConfig::default()
+        };
+        const MIB: u64 = 1024 * 1024;
+        assert_eq!(
+            resolve_active_file_roll_size(&with_budget(2048 * MIB)),
+            256 * MIB,
+            "the default budget keeps 256 MiB"
+        );
+        assert_eq!(resolve_active_file_roll_size(&with_budget(1024 * MIB)), 256 * MIB);
+        assert_eq!(
+            resolve_active_file_roll_size(&with_budget(256 * MIB)),
+            64 * MIB,
+            "a quarter of a small budget"
+        );
+        assert_eq!(resolve_active_file_roll_size(&with_budget(MIB)), MIB, "never below 1 MiB");
+        assert_eq!(resolve_active_file_roll_size(&with_budget(0)), 256 * MIB, "no budget, no derivation");
+    }
+
+    /// Both logging setups open the appender through this function, so the
+    /// cleanup task they start is always the one the appender wakes.
+    #[test]
+    fn opened_appender_reports_its_rolls_on_the_returned_signal() {
+        use std::io::Write as _;
+
+        let tmp = tempfile::tempdir().expect("log directory");
+        let config = OtelConfig {
+            log_max_single_file_size_bytes: Some(8),
+            ..OtelConfig::default()
+        };
+        let (mut appender, roll_signal) = open_rolling_appender(
+            &config,
+            tmp.path().to_str().expect("UTF-8 path"),
+            "rustfs.log",
+            Rotation::Daily,
+            FileMatchMode::Suffix,
+        )
+        .expect("appender");
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_time()
+            .build()
+            .expect("runtime");
+        let reported = || runtime.block_on(async { tokio::time::timeout(Duration::ZERO, roll_signal.notified()).await.is_ok() });
+
+        appender.write_all(b"12345678").expect("first write");
+        assert!(!reported());
+        appender.write_all(b"9").expect("write that rolls at the configured size");
+        assert!(reported(), "the configured roll size is in effect and the roll reaches the signal");
+    }
+
+    /// The cleanup loop runs a pass when the appender reports a roll, long
+    /// before its interval elapses, and that pass enforces the byte budget on
+    /// files younger than the age gate.
+    #[tokio::test]
+    async fn cleanup_loop_enforces_the_byte_budget_when_a_roll_is_reported() {
+        let tmp = tempfile::tempdir().expect("log directory");
+        let dir = tmp.path();
+        let config = OtelConfig {
+            log_max_total_size_bytes: Some(2 * 1024),
+            log_max_single_file_size_bytes: Some(0),
+            log_compress_old_files: Some(false),
+            log_min_file_age_seconds: Some(3600),
+            log_cleanup_interval_seconds: Some(3600),
+            log_match_mode: Some("suffix".to_string()),
+            log_filename: Some("rustfs.log".to_string()),
+            ..OtelConfig::default()
+        };
+        let roll_signal = Arc::new(tokio::sync::Notify::new());
+        let rolled_files = || {
+            fs::read_dir(dir)
+                .expect("log directory")
+                .filter_map(Result::ok)
+                .filter(|entry| entry.file_name() != "rustfs.log")
+                .count()
+        };
+        let write_rolled = |range: std::ops::Range<u32>| {
+            for index in range {
+                fs::write(dir.join(format!("2026-01-01-00-{index:02}.rustfs.log")), vec![b'x'; 1024]).expect("rolled file");
+            }
+        };
+        let wait_for_two_rolled_files = |what: &'static str| async move {
+            tokio::time::timeout(Duration::from_secs(10), async {
+                while rolled_files() > 2 {
+                    tokio::time::sleep(Duration::from_millis(50)).await;
+                }
+            })
+            .await
+            .unwrap_or_else(|_| panic!("{what}"));
+        };
+
+        // The pass the interval starts at once is observed through its effect,
+        // so what follows cannot be that pass arriving late.
+        fs::write(dir.join("rustfs.log"), vec![b'a'; 1024]).expect("active file");
+        write_rolled(0..6);
+        let cleanup = spawn_cleanup_task(&config, dir.to_str().expect("UTF-8 path"), "rustfs.log", 30, roll_signal.clone());
+        wait_for_two_rolled_files("the startup pass enforces the budget").await;
+
+        write_rolled(10..14);
+        tokio::time::sleep(Duration::from_millis(300)).await;
+        assert_eq!(rolled_files(), 6, "without a roll report nothing runs before the next interval");
+
+        roll_signal.notify_one();
+        wait_for_two_rolled_files("the reported roll starts a pass that brings the rolled files back under the budget").await;
+        assert_eq!(rolled_files(), 2);
+        assert!(dir.join("rustfs.log").exists(), "the active file is never removed");
+        cleanup.abort();
+    }
+
     #[test]
     fn empty_cleanup_pattern_falls_back_to_runtime_log_filename() {
         let config = OtelConfig {

crates/obs/src/telemetry/otel.rs

@@ -41,10 +41,10 @@ use crate::config::OtelConfig;
 use crate::global::set_observability_metric_enabled;
 use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter, s3s_log_filter};
 use crate::telemetry::guard::{OtelGuard, ProfilingAgent};
-use crate::telemetry::local::{build_json_log_layer, spawn_cleanup_task};
+use crate::telemetry::local::{build_json_log_layer, open_rolling_appender, spawn_cleanup_task};
 use crate::telemetry::recorder::{Recorder, install_process_global_recorder};
 use crate::telemetry::resource::build_resource;
-use crate::telemetry::rolling::{RollingAppender, Rotation};
+use crate::telemetry::rolling::Rotation;
 // Import helper functions from local.rs (sibling module)
 use crate::TelemetryError;
 use metrics::counter;
@@ -58,7 +58,7 @@ use opentelemetry_sdk::{
     trace::{RandomIdGenerator, Sampler, SdkTracerProvider},
 };
 use percent_encoding::percent_decode_str;
-use rustfs_config::observability::{DEFAULT_OBS_LOG_MATCH_MODE, DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES};
+use rustfs_config::observability::DEFAULT_OBS_LOG_MATCH_MODE;
 use rustfs_config::{
     APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOGS_EXPORT_ENABLED,
     DEFAULT_OBS_METRICS_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED, METER_INTERVAL, SAMPLE_RATIO,
@@ -244,17 +244,12 @@ pub(super) fn init_observability_http(
                 "daily" => Rotation::Daily,
                 _ => Rotation::Daily,
             };
-            let max_single_file_size = config
-                .log_max_single_file_size_bytes
-                .unwrap_or(DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES);
-
-            let file_appender =
-                RollingAppender::new(log_directory, log_filename.to_string(), rotation, max_single_file_size, match_mode)?;
+            let (file_appender, roll_signal) = open_rolling_appender(config, log_directory, log_filename, rotation, match_mode)?;
             crate::telemetry::local::validate_stdout_sink(&file_appender)?;
 
             let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
             let file_layer = build_json_log_layer(non_blocking, false, span_events.clone(), span_list);
-            let cleanup_handle = spawn_cleanup_task(config, log_directory, log_filename, keep_files);
+            let cleanup_handle = spawn_cleanup_task(config, log_directory, log_filename, keep_files, roll_signal);
             Ok((file_layer, guard, cleanup_handle, rotation_str))
         })();
 

crates/obs/src/telemetry/rolling.rs

@@ -29,6 +29,7 @@ use metrics::{counter, gauge, histogram};
 use std::fs::{self, File};
 use std::io::{self, Write};
 use std::path::{Path, PathBuf};
+use std::sync::Arc;
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::thread;
 use std::time::{Duration, Instant};
@@ -82,6 +83,9 @@ pub struct RollingAppender {
     // Store as seconds since Unix epoch
     last_roll_ts: i64,
     rotation_retry_cooldown_until: Option<Instant>,
+    /// Woken after every successful roll so the cleaner can enforce the
+    /// directory budget on the file that was just rolled.
+    roll_signal: Option<Arc<tokio::sync::Notify>>,
 }
 
 impl RollingAppender {
@@ -129,6 +133,7 @@ impl RollingAppender {
             size: 0,
             last_roll_ts: Zoned::now().timestamp().as_second(),
             rotation_retry_cooldown_until: None,
+            roll_signal: None,
         };
         // Eagerly open the file to validate the path and capture accurate
         // initial size / last-roll timestamp.
@@ -136,6 +141,12 @@ impl RollingAppender {
         Ok(appender)
     }
 
+    /// Notify `signal` after every successful roll.
+    pub(crate) fn with_roll_signal(mut self, signal: Arc<tokio::sync::Notify>) -> Self {
+        self.roll_signal = Some(signal);
+        self
+    }
+
     pub(crate) fn active_file_path(&self) -> PathBuf {
         self.dir.join(&self.filename)
     }
@@ -289,6 +300,9 @@ impl RollingAppender {
                     counter!(METRIC_LOG_CLEANER_ROTATION_TOTAL).increment(1);
                     histogram!(METRIC_LOG_CLEANER_ROTATION_DURATION_SECONDS).record(rotate_started.elapsed().as_secs_f64());
                     gauge!(METRIC_LOG_CLEANER_ACTIVE_FILE_SIZE_BYTES).set(self.size as f64);
+                    if let Some(signal) = &self.roll_signal {
+                        signal.notify_one();
+                    }
                     return Ok(());
                 }
                 Err(e) => {
@@ -452,6 +466,80 @@ mod tests {
 
     // ── Size-based rotation ────────────────────────────────────────────────────
 
+    /// A storm with the shipped roll behavior: the appender rolls by size and
+    /// wakes the cleaner, and the directory stays within the byte budget plus
+    /// the active file although every rolled file is younger than the age gate.
+    #[test]
+    fn log_directory_stays_bounded_under_a_storm() -> io::Result<()> {
+        use crate::cleaner::LogCleaner;
+
+        const ROLL_SIZE: u64 = 4 * 1024;
+        const BUDGET: u64 = 16 * 1024;
+        let tmp = TempDir::new()?;
+        let dir = tmp.path().to_path_buf();
+        let signal = Arc::new(tokio::sync::Notify::new());
+        let mut appender =
+            RollingAppender::new(&dir, "rustfs.log".to_string(), Rotation::Never, ROLL_SIZE, FileMatchMode::Suffix)?
+                .with_roll_signal(signal.clone());
+        let cleaner = LogCleaner::builder(dir.clone(), "rustfs.log".to_string(), "rustfs.log".to_string())
+            .match_mode(FileMatchMode::Suffix)
+            .keep_files(30)
+            .max_total_size_bytes(BUDGET)
+            .compress_old_files(false)
+            .min_file_age_seconds(3600)
+            .build();
+        let directory_bytes = |dir: &Path| -> io::Result<u64> {
+            let mut total = 0;
+            for entry in fs::read_dir(dir)? {
+                total += entry?.metadata()?.len();
+            }
+            Ok(total)
+        };
+        let runtime = tokio::runtime::Builder::new_current_thread().enable_time().build()?;
+
+        let line = [b'x'; 512];
+        let mut passes = 0;
+        let mut peak = 0;
+        // 512 KiB of log lines, 32 times the budget.
+        for _ in 0..1024 {
+            appender.write_all(&line)?;
+            // The cleanup loop runs one pass per reported roll.
+            let rolled = runtime.block_on(async { tokio::time::timeout(Duration::ZERO, signal.notified()).await.is_ok() });
+            if rolled {
+                cleaner.cleanup()?;
+                passes += 1;
+            }
+            peak = peak.max(directory_bytes(&dir)?);
+        }
+        appender.flush()?;
+
+        assert!(passes > 100, "size rolls woke the cleaner {passes} times");
+        // One freshly rolled file may sit above the budget until its pass runs.
+        assert!(
+            peak <= BUDGET + 2 * ROLL_SIZE,
+            "peak directory size {peak} exceeded the budget {BUDGET} plus the active and one rolled file"
+        );
+        assert!(directory_bytes(&dir)? <= BUDGET + ROLL_SIZE);
+        Ok(())
+    }
+
+    #[test]
+    fn every_successful_roll_wakes_the_roll_signal() {
+        let tmp = TempDir::new().unwrap();
+        let signal = Arc::new(tokio::sync::Notify::new());
+        let mut appender = RollingAppender::new(tmp.path(), "app.log".to_string(), Rotation::Never, 5, FileMatchMode::Suffix)
+            .unwrap()
+            .with_roll_signal(signal.clone());
+        let runtime = tokio::runtime::Builder::new_current_thread().enable_time().build().unwrap();
+        let woken = || runtime.block_on(async { tokio::time::timeout(Duration::ZERO, signal.notified()).await.is_ok() });
+
+        appender.write_all(b"12345").expect("write should succeed");
+        assert!(!woken(), "no roll yet");
+        appender.write_all(b"abcde").expect("write after rotation should succeed");
+        assert!(woken(), "the size roll is reported");
+        assert!(!woken(), "one roll is one wake-up");
+    }
+
     #[test]
     fn test_size_rotation_creates_archive() {
         let tmp = TempDir::new().unwrap();

crates/protocols/src/sftp/driver.rs

@@ -27,7 +27,7 @@ use super::attrs;
 use super::constants::limits::S3_COPY_OBJECT_MAX_SIZE;
 use super::errors::{SftpError, auth_err, auth_err_unreachable, is_no_such_upload_error, ok_status, s3_error_to_sftp};
 use super::lifecycle::SessionDiag;
-use super::paths::{format_copy_source, parse_s3_path, sanitise_control_bytes};
+use super::paths::{copy_source_value, parse_s3_path, sanitise_control_bytes};
 use super::state::{HandleState, WritePhase};
 use super::write::{build_write_tombstone, fstat_reported_size, rejects_excl_or_trunc_without_create, should_abort_on_drop};
 use crate::common::client::s3::StorageBackend;
@@ -937,8 +937,7 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
         // Multipart UploadPartCopy above that.
         if content_length <= S3_COPY_OBJECT_MAX_SIZE {
             self.authorize(&S3Action::CopyObject, &dst_bucket, Some(&dst_object)).await?;
-            let input = CopyObjectInput::builder()
-                .copy_source(format_copy_source(&src_bucket, &src_object))
+            let input = CopyObjectInput::builder(copy_source_value(&src_bucket, &src_object))
                 .bucket(BucketName::new(dst_bucket.clone()).map_err(|e| s3_error_to_sftp("build_copy_object", e))?)
                 .key(ObjectKey::new(dst_object.clone()).map_err(|e| s3_error_to_sftp("build_copy_object", e))?)
                 .build();

crates/protocols/src/sftp/paths.rs

@@ -101,11 +101,14 @@ pub(super) fn parse_s3_path(input: &str) -> Result<(String, Option<String>), Sft
     Ok((bucket, key))
 }
 
-/// Formats an S3 copy-source header while preserving the literal object key.
-/// The S3 copy-source parser URL-decodes the path and treats a raw `?versionId=`
-/// as query data, so encode the complete key before building the header.
-pub(super) fn format_copy_source(bucket: &str, key: &str) -> String {
-    format!("/{bucket}/{}", urlencoding::encode(key))
+/// Describes an S3 copy source by its literal bucket and object key, with no version.
+/// The key is carried verbatim; the storage client renders the copy-source header from it.
+pub(super) fn copy_source_value(bucket: &str, key: &str) -> rustfs_gateway_types::CopySourceValue {
+    rustfs_gateway_types::CopySourceValue::Bucket {
+        bucket: bucket.to_owned(),
+        key: key.to_owned(),
+        version_id: None,
+    }
 }
 
 /// Replace C0 control bytes (other than tab) with the literal byte 0x3F
@@ -137,10 +140,17 @@ mod tests {
     use russh_sftp::protocol::StatusCode;
 
     #[test]
-    fn format_copy_source_preserves_encoded_and_query_like_keys() {
-        assert_eq!(format_copy_source("bucket", "%2F"), "/bucket/%252F");
-        assert_eq!(format_copy_source("bucket", "foo?versionId=1"), "/bucket/foo%3FversionId%3D1");
-        assert_eq!(format_copy_source("bucket", "a/b"), "/bucket/a%2Fb");
+    fn copy_source_value_keeps_encoded_and_query_like_keys_literal() {
+        for key in ["%2F", "foo?versionId=1", "a/b", "plain"] {
+            assert_eq!(
+                copy_source_value("bucket", key),
+                rustfs_gateway_types::CopySourceValue::Bucket {
+                    bucket: "bucket".to_owned(),
+                    key: key.to_owned(),
+                    version_id: None,
+                }
+            );
+        }
     }
 
     #[test]

crates/protocols/src/sftp/write.rs

@@ -25,7 +25,7 @@ use super::constants::limits::{
 };
 use super::driver::SftpDriver;
 use super::errors::{SftpError, is_not_found_error, s3_error_to_sftp};
-use super::paths::{format_copy_source, parse_s3_path, sanitise_control_bytes};
+use super::paths::{copy_source_value, parse_s3_path, sanitise_control_bytes};
 use super::state::{CompletedPart, HandleState, MultipartUpload, WritePhase};
 use crate::common::client::s3::StorageBackend;
 use crate::common::gateway::S3Action;
@@ -1051,12 +1051,11 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
 
                 self.authorize(&S3Action::UploadPart, dst_bucket, Some(dst_key)).await?;
 
-                let input = UploadPartCopyInput::builder()
+                let input = UploadPartCopyInput::builder(copy_source_value(src_bucket, src_key))
                     .bucket(bucket_name(dst_bucket, "build_upload_part_copy")?)
                     .key(object_key(dst_key, "build_upload_part_copy")?)
                     .upload_id(upload_claim(&mp.upload_id))
                     .part_number(part_number)
-                    .copy_source(format_copy_source(src_bucket, src_key))
                     .copy_source_range(range)
                     .build();
 

crates/protocols/src/webdav/driver.rs

@@ -949,7 +949,7 @@ where
                             modified,
                             created: modified,
                             is_dir: false,
-                            etag: Some(etag_to_string(&obj.e_tag)),
+                            etag: obj.e_tag.as_ref().map(etag_to_string),
                             content_type: None,
                         },
                     });

crates/protos/src/generated/proto_gen/node_service.rs

@@ -945,6 +945,9 @@ pub struct GenerallyLockResult {
     /// Per-ID result for unlock batches.
     #[prost(enumeration = "LockReleaseStatus", tag = "4")]
     pub release_status: i32,
+    /// Only populated by lock batches.
+    #[prost(enumeration = "LockAcquireStatus", tag = "5")]
+    pub acquire_status: i32,
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct BatchGenerallyLockResponse {
@@ -1912,6 +1915,36 @@ impl LockReleaseStatus {
         }
     }
 }
+/// Only explicit no-grant results permit a caller to retry without grant cleanup.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
+#[repr(i32)]
+pub enum LockAcquireStatus {
+    Unspecified = 0,
+    RetryableNoGrant = 1,
+    RejectedNoGrant = 2,
+}
+impl LockAcquireStatus {
+    /// String value of the enum field names used in the ProtoBuf definition.
+    ///
+    /// The values are not transformed in any way and thus are considered stable
+    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
+    pub fn as_str_name(&self) -> &'static str {
+        match self {
+            Self::Unspecified => "LOCK_ACQUIRE_STATUS_UNSPECIFIED",
+            Self::RetryableNoGrant => "LOCK_ACQUIRE_STATUS_RETRYABLE_NO_GRANT",
+            Self::RejectedNoGrant => "LOCK_ACQUIRE_STATUS_REJECTED_NO_GRANT",
+        }
+    }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "LOCK_ACQUIRE_STATUS_UNSPECIFIED" => Some(Self::Unspecified),
+            "LOCK_ACQUIRE_STATUS_RETRYABLE_NO_GRANT" => Some(Self::RetryableNoGrant),
+            "LOCK_ACQUIRE_STATUS_REJECTED_NO_GRANT" => Some(Self::RejectedNoGrant),
+            _ => None,
+        }
+    }
+}
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
 #[repr(i32)]
 pub enum TierMutationPeerState {

crates/protos/src/lib.rs

@@ -4121,7 +4121,7 @@ mod tests {
 
 #[cfg(test)]
 mod lock_release_tests {
-    use super::proto_gen::node_service::{GenerallyLockResponse, GenerallyLockResult, LockReleaseStatus};
+    use super::proto_gen::node_service::{GenerallyLockResponse, GenerallyLockResult, LockAcquireStatus, LockReleaseStatus};
     use prost::Message;
 
     /// The release response schema before structured outcomes were added.
@@ -4168,6 +4168,7 @@ mod lock_release_tests {
                 error_info: error.clone(),
                 lock_info: None,
                 release_status: status.into(),
+                ..Default::default()
             };
             for encoded in [response.encode_to_vec(), result.encode_to_vec()] {
                 let legacy = LegacyReleaseResponse::decode(encoded.as_slice()).expect("old peer ignores additive outcome");
@@ -4177,4 +4178,43 @@ mod lock_release_tests {
             }
         }
     }
+
+    #[test]
+    fn batch_acquire_outcomes_are_additive_and_preserve_legacy_grants() {
+        for success in [false, true] {
+            let legacy = LegacyReleaseResponse {
+                success,
+                error_info: (!success).then(|| "legacy failure".to_string()),
+                lock_info: None,
+            };
+            let result = GenerallyLockResult::decode(legacy.encode_to_vec().as_slice()).expect("decode legacy acquire");
+            assert_eq!(result.success, success);
+            assert_eq!(result.acquire_status(), LockAcquireStatus::Unspecified);
+        }
+        for status in [LockAcquireStatus::RetryableNoGrant, LockAcquireStatus::RejectedNoGrant] {
+            let result = GenerallyLockResult {
+                success: false,
+                error_info: Some("not granted".to_string()),
+                acquire_status: status.into(),
+                ..Default::default()
+            };
+            let legacy = LegacyReleaseResponse::decode(result.encode_to_vec().as_slice()).expect("decode additive acquire");
+            assert!(!legacy.success);
+            assert_eq!(legacy.error_info, result.error_info);
+            let single =
+                GenerallyLockResponse::decode(result.encode_to_vec().as_slice()).expect("single schema ignores batch field");
+            assert_eq!(single.release_status(), LockReleaseStatus::Unspecified);
+        }
+    }
+
+    #[test]
+    fn batch_acquire_unknown_status_is_preserved_for_conservative_classification() {
+        let result = GenerallyLockResult {
+            acquire_status: 99,
+            ..Default::default()
+        };
+        let decoded = GenerallyLockResult::decode(result.encode_to_vec().as_slice()).expect("decode unknown enum");
+        assert_eq!(decoded.acquire_status, 99);
+        assert!(LockAcquireStatus::try_from(decoded.acquire_status).is_err());
+    }
 }

crates/protos/src/node.proto

@@ -662,11 +662,19 @@ message BatchGenerallyLockRequest {
   repeated string args = 1;
 }
 
+// Only explicit no-grant results permit a caller to retry without grant cleanup.
+enum LockAcquireStatus {
+  LOCK_ACQUIRE_STATUS_UNSPECIFIED = 0;
+  LOCK_ACQUIRE_STATUS_RETRYABLE_NO_GRANT = 1;
+  LOCK_ACQUIRE_STATUS_REJECTED_NO_GRANT = 2;
+}
+
 message GenerallyLockResult {
   bool success = 1;
   optional string error_info = 2;
   optional string lock_info = 3; // JSON serialized LockInfo
   LockReleaseStatus release_status = 4; // Per-ID result for unlock batches.
+  LockAcquireStatus acquire_status = 5; // Only populated by lock batches.
 }
 
 message BatchGenerallyLockResponse {

crates/scanner/src/scanner_folder.rs

@@ -425,7 +425,10 @@ enum FolderScanSource {
 #[derive(Clone, Copy, PartialEq, Eq)]
 enum FolderScanMode {
     Namespace,
-    ErasureCandidate { reuse_namespace: bool },
+    ErasureCandidate {
+        reuse_namespace: bool,
+        parent_has_metadata: bool,
+    },
 }
 
 #[derive(Debug, PartialEq, Eq)]
@@ -2065,7 +2068,9 @@ impl FolderScanner {
                     if candidate_unclassified {
                         // Apply namespace reuse only after the current listing
                         // proves there is no direct erasure part.
-                        if let FolderScanMode::ErasureCandidate { reuse_namespace: true } = mode
+                        if let FolderScanMode::ErasureCandidate {
+                            reuse_namespace: true, ..
+                        } = mode
                             && !into.compacted
                             && self.old_cache.find(&folder.name).is_some()
                         {
@@ -2079,10 +2084,23 @@ impl FolderScanner {
                             return Ok(FolderScanOutcome::Namespace);
                         }
                         candidate_unclassified = false;
-                        if !self.budget.try_start_directory() {
-                            return Err(ScannerError::Other("Operation cancelled".to_string()));
+                        // An empty UUID directory beside object metadata has
+                        // no namespace work. Its listing was already charged;
+                        // nonempty namespaces still need directory admission.
+                        let empty_object_child = matches!(
+                            mode,
+                            FolderScanMode::ErasureCandidate {
+                                parent_has_metadata: true,
+                                ..
+                            }
+                        ) && deferred_metadata.is_none()
+                            && queued_folders.is_empty();
+                        if !empty_object_child {
+                            if !self.budget.try_start_directory() {
+                                return Err(ScannerError::Other("Operation cancelled".to_string()));
+                            }
+                            wait_for_scanner_io(&ctx, self.sleeper.sleep_folder_preset()).await?;
                         }
-                        wait_for_scanner_io(&ctx, self.sleeper.sleep_folder_preset()).await?;
                     }
                     let Some(entry) = deferred_metadata.take() else { break };
                     (entry, true)
@@ -2552,20 +2570,6 @@ impl FolderScanner {
                 return Err(ScannerError::Other("Operation cancelled".to_string()));
             }
 
-            if found_object_metadata && self.is_erasure_mode {
-                // If we found an object in erasure mode, we skip subdirs (only datadirs)...
-                debug!(
-                    target: "rustfs::scanner::folder",
-                    event = EVENT_SCANNER_FOLDER_STATE,
-                    component = LOG_COMPONENT_SCANNER,
-                    subsystem = LOG_SUBSYSTEM_FOLDER,
-                    folder = %folder.name,
-                    state = "erasure_object_found",
-                    "Scanner folder descent stopped after erasure object"
-                );
-                break;
-            }
-
             // Checkpointed sweeps need child entries to validate their resume
             // frontier (#8292). Outside that, also compact a wide bucket root:
             // excluding it left #8961's ~170k top-level prefixes unbounded until
@@ -2654,6 +2658,7 @@ impl FolderScanner {
                 let mut child_mode = if self.is_erasure_mode && is_erasure_candidate(&folder_item.name) {
                     FolderScanMode::ErasureCandidate {
                         reuse_namespace: reuse_covered,
+                        parent_has_metadata: found_object_metadata,
                     }
                 } else {
                     FolderScanMode::Namespace
@@ -2721,7 +2726,10 @@ impl FolderScanner {
                                 }
                                 // Sampling may skip namespace work, never the
                                 // classification that protects its parent object.
-                                child_mode = FolderScanMode::ErasureCandidate { reuse_namespace: true };
+                                child_mode = FolderScanMode::ErasureCandidate {
+                                    reuse_namespace: true,
+                                    parent_has_metadata: found_object_metadata,
+                                };
                             } else {
                                 folder_item.object_heal_prob_div = data_usage_update_dir_cycles();
                             }
@@ -2816,6 +2824,11 @@ impl FolderScanner {
                         updates.cache.delete_recursive(&h);
                     }
                     into.children.remove(&clone_cache_value(&h.0));
+                    // Object keys can also be prefixes. A classified payload
+                    // directory must not hide their other namespace children.
+                    if found_object_metadata {
+                        continue;
+                    }
                     // Earlier namespace siblings may have completed, but none
                     // can certify coverage of this unresolved parent object.
                     self.coverage_gap |= forward_sweep;

crates/scanner/src/scanner_folder/tests/erasure_probe.rs

@@ -17,9 +17,18 @@ static ORDERED_DIRECTORIES: LazyLock<Mutex<HashMap<String, DirectoryOrder>>> = L
 
 pub(in crate::scanner_folder) struct OrderedDirectory(String);
 
+/// The failed-object key the scanner records for `relative` under `root`.
+/// Production builds it with `path_join_buf`, which uses `/` separators on
+/// every platform, so a native `root.join(..)` never matches on Windows.
+fn failed_object_key(root: &std::path::Path, relative: &str) -> String {
+    rustfs_utils::path::path_join_buf(&[&root.join(relative).to_string_lossy()])
+}
+
 impl OrderedDirectory {
     fn new(path: &std::path::Path) -> Self {
-        let path = path.to_string_lossy().into_owned();
+        // Key the probe by the cleaned form the scanner opens; on Windows a joined
+        // `C:\root\bucket/object` would otherwise never match `C:/root/bucket/object`.
+        let path = rustfs_utils::path::path_join_buf(&[&path.to_string_lossy()]);
         ORDERED_DIRECTORIES
             .lock()
             .expect("ordered directory registry")
@@ -136,7 +145,7 @@ async fn erasure_candidate_finds_late_part_without_scanning_data_children() {
             .new_cache
             .info
             .failed_objects
-            .contains_key(root.join("bucket/object/xl.meta").to_string_lossy().as_ref())
+            .contains_key(&failed_object_key(&root, "bucket/object/xl.meta"))
     );
     assert_eq!(budget.reason(), None, "payload directories must not consume namespace directory quota");
     assert!(scanner.new_cache.info.pending_heals.iter().any(|pending| {
@@ -183,6 +192,117 @@ fn charged_ops(root: &std::path::Path) -> u64 {
     status.charged_ops
 }
 
+#[tokio::test]
+#[serial]
+async fn erasure_object_metadata_preserves_child_object_namespaces() {
+    for (corrupt_parent, cached_parent) in [(false, false), (true, false), (false, true)] {
+        let (mut scanner, root) = build_test_scanner().await;
+        let _guard = TestGuard::new(60, 100, &mut scanner, root.clone());
+        scanner.old_cache.info.name = "bucket".into();
+        scanner.new_cache.info.name = "bucket".into();
+        scanner.is_erasure_mode = true;
+        bind_budget(&mut scanner, &root, 1_000_000);
+
+        let namespace = format!("parent/{}/nested", Uuid::new_v4());
+        for object in ["parent", "parent/child", namespace.as_str()] {
+            write_test_object_metadata_bytes(&root, "bucket", object, &metadata_for_object_version("bucket", object, None)).await;
+        }
+        let payload = root.join("bucket/parent").join(Uuid::new_v4().to_string());
+        tokio::fs::create_dir_all(&payload)
+            .await
+            .expect("create parent payload directory");
+        tokio::fs::write(payload.join("part.1"), b"shard")
+            .await
+            .expect("create actual payload evidence");
+        if corrupt_parent {
+            tokio::fs::write(root.join("bucket/parent/xl.meta"), b"")
+                .await
+                .expect("corrupt only the parent metadata");
+        }
+        if cached_parent {
+            scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
+            scanner.old_cache.replace(
+                "bucket/parent",
+                "bucket",
+                DataUsageEntry {
+                    objects: 1,
+                    size: 1,
+                    compacted: true,
+                    ..Default::default()
+                },
+            );
+            let interval = data_usage_update_dir_cycles();
+            scanner.old_cache.info.next_cycle = u64::from(
+                (0..interval)
+                    .find(|cycle| hash_path("bucket/parent").mod_(*cycle, interval))
+                    .expect("one normal sampling cycle revisits the old parent-only cache"),
+            );
+        }
+
+        let mut into = DataUsageEntry::default();
+        scanner
+            .scan_folder(
+                CancellationToken::new(),
+                CachedFolder {
+                    name: "bucket".into(),
+                    parent: None,
+                    object_heal_prob_div: 1,
+                },
+                &mut into,
+            )
+            .await
+            .expect("scan object and child namespaces independently");
+        let total = scanner.new_cache.size_recursive("bucket").expect("retain bucket accounting");
+        assert_eq!(total.objects, if corrupt_parent { 2 } else { 3 });
+        assert_eq!(total.size, if corrupt_parent { 2 } else { 3 });
+        assert_eq!(total.failed_objects, usize::from(corrupt_parent));
+        assert_eq!(
+            scanner.new_cache.info.failed_objects.len(),
+            usize::from(corrupt_parent),
+            "payload evidence must not invent missing parent metadata"
+        );
+    }
+}
+
+#[tokio::test]
+#[serial]
+async fn erasure_object_prefix_children_keep_the_object_budget() {
+    let (mut scanner, root) = build_test_scanner().await;
+    let _guard = TestGuard::new(60, 100, &mut scanner, root.clone());
+    scanner.old_cache.info.name = "bucket".into();
+    scanner.new_cache.info.name = "bucket".into();
+    scanner.is_erasure_mode = true;
+    bind_budget(&mut scanner, &root, 1_000_000);
+    for object in ["parent", "parent/child", "parent/other"] {
+        write_test_object_metadata(&root, "bucket", object).await;
+    }
+    let visits = metadata_visits::observe(&mut scanner);
+    let parent = CancellationToken::new();
+    let budget = ScannerCycleBudget::new_with_progress_tracking(
+        &parent,
+        crate::scanner_budget::ScannerCycleBudgetConfig {
+            max_objects: Some(2),
+            ..Default::default()
+        },
+    );
+    scanner.budget = budget.clone();
+    let result = scanner
+        .scan_folder(
+            budget.token(),
+            CachedFolder {
+                name: "bucket".into(),
+                parent: None,
+                object_heal_prob_div: 1,
+            },
+            &mut DataUsageEntry::default(),
+        )
+        .await;
+    assert!(result.is_err(), "parent metadata cannot hide children or bypass their budget");
+    assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects));
+    assert_eq!(budget.progress().0, 2);
+    assert_eq!(visits.load(Ordering::Relaxed), 2);
+}
+
 #[tokio::test]
 #[serial]
 async fn erasure_directory_probe_reaches_eof_across_budgeted_batches() {
@@ -204,7 +324,10 @@ async fn erasure_directory_probe_reaches_eof_across_budgeted_batches() {
                 ctx,
                 folder,
                 &mut DataUsageEntry::default(),
-                FolderScanMode::ErasureCandidate { reuse_namespace: false },
+                FolderScanMode::ErasureCandidate {
+                    reuse_namespace: false,
+                    parent_has_metadata: false,
+                },
                 false,
             )
             .await
@@ -271,7 +394,10 @@ async fn scanner_directory_probe_budget_wait_cancels_without_rate_delay() {
             ctx.clone(),
             folder,
             &mut DataUsageEntry::default(),
-            FolderScanMode::ErasureCandidate { reuse_namespace: false },
+            FolderScanMode::ErasureCandidate {
+                reuse_namespace: false,
+                parent_has_metadata: false,
+            },
             false,
         )
         .await;
@@ -311,7 +437,10 @@ async fn scanner_directory_probe_cancels_during_entry_budget_wait() {
             ctx.clone(),
             folder,
             &mut DataUsageEntry::default(),
-            FolderScanMode::ErasureCandidate { reuse_namespace: false },
+            FolderScanMode::ErasureCandidate {
+                reuse_namespace: false,
+                parent_has_metadata: false,
+            },
             false,
         )
         .await;
@@ -382,7 +511,16 @@ async fn erasure_candidate_defers_metadata_until_part_classification_finishes()
     scanner.budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, Default::default());
     let mut into = DataUsageEntry::default();
     let outcome = scanner
-        .scan_folder_inner(ctx, folder, &mut into, FolderScanMode::ErasureCandidate { reuse_namespace: false }, false)
+        .scan_folder_inner(
+            ctx,
+            folder,
+            &mut into,
+            FolderScanMode::ErasureCandidate {
+                reuse_namespace: false,
+                parent_has_metadata: false,
+            },
+            false,
+        )
         .await
         .expect("classify candidate");
     assert_eq!(outcome, FolderScanOutcome::ErasureData);
@@ -473,7 +611,7 @@ async fn erasure_candidate_cancelled_batch_keeps_partial_checkpoint_and_reclassi
             .new_cache
             .info
             .failed_objects
-            .contains_key(root.join("bucket/xl.meta").to_string_lossy().as_ref())
+            .contains_key(&failed_object_key(&root, "bucket/xl.meta"))
     );
     assert_eq!(
         charged_ops(&root),
@@ -532,7 +670,7 @@ async fn erasure_late_data_sibling_prevents_parent_coverage_certification() {
             .new_cache
             .info
             .failed_objects
-            .contains_key(root.join("bucket/object/xl.meta").to_string_lossy().as_ref())
+            .contains_key(&failed_object_key(&root, "bucket/object/xl.meta"))
     );
 }
 
@@ -594,7 +732,10 @@ async fn erasure_candidate_part_symlink_is_not_data_evidence() {
             CancellationToken::new(),
             folder,
             &mut DataUsageEntry::default(),
-            FolderScanMode::ErasureCandidate { reuse_namespace: false },
+            FolderScanMode::ErasureCandidate {
+                reuse_namespace: false,
+                parent_has_metadata: false,
+            },
             false,
         )
         .await

crates/targets/src/config/common.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use crate::TargetError;
+use crate::store::resolve_queue_dir;
 use crate::target::nats::validate_jetstream_settings;
 use crate::target::pulsar::validate_pulsar_broker;
 use async_nats::ServerAddr;
@@ -29,6 +30,12 @@ use std::path::Path;
 use std::str::FromStr;
 use url::Url;
 
+/// Return the queue directory this node opens for a target, with logical defaults resolved.
+pub(super) fn queue_dir_or_default(config: &KVS, key: &str, default_queue_dir: &str) -> String {
+    let queue_dir = config.lookup(key).unwrap_or_else(|| default_queue_dir.to_string());
+    resolve_queue_dir(&queue_dir).into_owned()
+}
+
 pub(super) fn split_env_field_and_instance(rest: &str, valid_fields: &HashSet<String>) -> Option<(String, String)> {
     let normalized = rest.to_lowercase();
     if valid_fields.contains(&normalized) {
@@ -124,7 +131,7 @@ pub(super) fn validate_nats_server_config(server: &ServerAddr, config: &KVS, def
         ));
     }
 
-    let queue_dir = config.lookup(NATS_QUEUE_DIR).unwrap_or_else(|| default_queue_dir.to_string());
+    let queue_dir = queue_dir_or_default(config, NATS_QUEUE_DIR, default_queue_dir);
     if !queue_dir.is_empty() && !Path::new(&queue_dir).is_absolute() {
         return Err(TargetError::Configuration("NATS queue directory must be an absolute path".to_string()));
     }
@@ -204,9 +211,7 @@ pub(super) fn validate_pulsar_broker_config(broker: &str, config: &KVS, default_
         ));
     }
 
-    let queue_dir = config
-        .lookup(PULSAR_QUEUE_DIR)
-        .unwrap_or_else(|| default_queue_dir.to_string());
+    let queue_dir = queue_dir_or_default(config, PULSAR_QUEUE_DIR, default_queue_dir);
     if !queue_dir.is_empty() && !Path::new(&queue_dir).is_absolute() {
         return Err(TargetError::Configuration("Pulsar queue directory must be an absolute path".to_string()));
     }

crates/targets/src/config/target_args.rs

@@ -13,8 +13,8 @@
 // limitations under the License.
 
 use super::common::{
-    parse_jetstream_ack_timeout_secs, parse_jetstream_enable, parse_target_bool, parse_url, validate_nats_server_config,
-    validate_outbound_http_url, validate_pulsar_broker_config,
+    parse_jetstream_ack_timeout_secs, parse_jetstream_enable, parse_target_bool, parse_url, queue_dir_or_default,
+    validate_nats_server_config, validate_outbound_http_url, validate_pulsar_broker_config,
 };
 use crate::error::TargetError;
 use crate::target::{
@@ -118,7 +118,7 @@ pub fn build_amqp_args(config: &KVS, default_queue_dir: &str, target_type: Targe
         tls_ca: config.lookup(AMQP_TLS_CA).unwrap_or_default(),
         tls_client_cert: config.lookup(AMQP_TLS_CLIENT_CERT).unwrap_or_default(),
         tls_client_key: config.lookup(AMQP_TLS_CLIENT_KEY).unwrap_or_default(),
-        queue_dir: config.lookup(AMQP_QUEUE_DIR).unwrap_or_else(|| default_queue_dir.to_string()),
+        queue_dir: queue_dir_or_default(config, AMQP_QUEUE_DIR, default_queue_dir),
         queue_limit: config
             .lookup(AMQP_QUEUE_LIMIT)
             .and_then(|v| v.parse::<u64>().ok())
@@ -146,9 +146,7 @@ pub fn build_webhook_args(config: &KVS, default_queue_dir: &str, target_type: Ta
         enable: true,
         endpoint: endpoint_url,
         auth_token: config.lookup(WEBHOOK_AUTH_TOKEN).unwrap_or_default(),
-        queue_dir: config
-            .lookup(WEBHOOK_QUEUE_DIR)
-            .unwrap_or_else(|| default_queue_dir.to_string()),
+        queue_dir: queue_dir_or_default(config, WEBHOOK_QUEUE_DIR, default_queue_dir),
         queue_limit: config
             .lookup(WEBHOOK_QUEUE_LIMIT)
             .and_then(|v| v.parse::<u64>().ok())
@@ -180,9 +178,7 @@ pub fn validate_webhook_config(config: &KVS, default_queue_dir: &str) -> Result<
         ));
     }
 
-    let queue_dir = config
-        .lookup(WEBHOOK_QUEUE_DIR)
-        .unwrap_or_else(|| default_queue_dir.to_string());
+    let queue_dir = queue_dir_or_default(config, WEBHOOK_QUEUE_DIR, default_queue_dir);
     if !queue_dir.is_empty() && !Path::new(&queue_dir).is_absolute() {
         return Err(TargetError::Configuration("Webhook queue directory must be an absolute path".to_string()));
     }
@@ -234,7 +230,7 @@ pub fn build_mqtt_args(config: &KVS, default_queue_dir: &str, target_type: Targe
             config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
             config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
         )?,
-        queue_dir: config.lookup(MQTT_QUEUE_DIR).unwrap_or_else(|| default_queue_dir.to_string()),
+        queue_dir: queue_dir_or_default(config, MQTT_QUEUE_DIR, default_queue_dir),
         queue_limit: config
             .lookup(MQTT_QUEUE_LIMIT)
             .and_then(|v| v.parse::<u64>().ok())
@@ -311,7 +307,7 @@ pub fn build_nats_args(config: &KVS, default_queue_dir: &str, target_type: Targe
         tls_client_cert: config.lookup(NATS_TLS_CLIENT_CERT).unwrap_or_default(),
         tls_client_key: config.lookup(NATS_TLS_CLIENT_KEY).unwrap_or_default(),
         tls_required: parse_target_bool(config.lookup(NATS_TLS_REQUIRED).as_deref()).unwrap_or(false),
-        queue_dir: config.lookup(NATS_QUEUE_DIR).unwrap_or_else(|| default_queue_dir.to_string()),
+        queue_dir: queue_dir_or_default(config, NATS_QUEUE_DIR, default_queue_dir),
         queue_limit: config
             .lookup(NATS_QUEUE_LIMIT)
             .and_then(|v| v.parse::<u64>().ok())
@@ -353,9 +349,7 @@ pub fn build_pulsar_args(config: &KVS, default_queue_dir: &str, target_type: Tar
         tls_ca: config.lookup(PULSAR_TLS_CA).unwrap_or_default(),
         tls_allow_insecure: parse_target_bool(config.lookup(PULSAR_TLS_ALLOW_INSECURE).as_deref()).unwrap_or(false),
         tls_hostname_verification: parse_target_bool(config.lookup(PULSAR_TLS_HOSTNAME_VERIFICATION).as_deref()).unwrap_or(true),
-        queue_dir: config
-            .lookup(PULSAR_QUEUE_DIR)
-            .unwrap_or_else(|| default_queue_dir.to_string()),
+        queue_dir: queue_dir_or_default(config, PULSAR_QUEUE_DIR, default_queue_dir),
         queue_limit: config
             .lookup(PULSAR_QUEUE_LIMIT)
             .and_then(|v| v.parse::<u64>().ok())
@@ -405,9 +399,7 @@ pub fn build_redis_args(
             .and_then(|v| v.parse::<u64>().ok())
             .map(Duration::from_secs)
             .unwrap_or_else(|| Duration::from_secs(15)),
-        queue_dir: config
-            .lookup(REDIS_QUEUE_DIR)
-            .unwrap_or_else(|| default_queue_dir.to_string()),
+        queue_dir: queue_dir_or_default(config, REDIS_QUEUE_DIR, default_queue_dir),
         queue_limit: config
             .lookup(REDIS_QUEUE_LIMIT)
             .and_then(|v| v.parse::<u64>().ok())
@@ -463,9 +455,7 @@ pub fn build_postgres_args(config: &KVS, default_queue_dir: &str, target_type: T
         tls_ca: config.lookup(POSTGRES_TLS_CA).unwrap_or_default(),
         tls_client_cert: config.lookup(POSTGRES_TLS_CLIENT_CERT).unwrap_or_default(),
         tls_client_key: config.lookup(POSTGRES_TLS_CLIENT_KEY).unwrap_or_default(),
-        queue_dir: config
-            .lookup(POSTGRES_QUEUE_DIR)
-            .unwrap_or_else(|| default_queue_dir.to_string()),
+        queue_dir: queue_dir_or_default(config, POSTGRES_QUEUE_DIR, default_queue_dir),
         queue_limit: config
             .lookup(POSTGRES_QUEUE_LIMIT)
             .and_then(|v| v.parse::<u64>().ok())
@@ -529,9 +519,7 @@ pub fn build_kafka_args(config: &KVS, default_queue_dir: &str, target_type: Targ
         },
         sasl_username,
         sasl_password,
-        queue_dir: config
-            .lookup(KAFKA_QUEUE_DIR)
-            .unwrap_or_else(|| default_queue_dir.to_string()),
+        queue_dir: queue_dir_or_default(config, KAFKA_QUEUE_DIR, default_queue_dir),
         queue_limit: config
             .lookup(KAFKA_QUEUE_LIMIT)
             .and_then(|v| v.parse::<u64>().ok())
@@ -569,9 +557,7 @@ pub fn build_mysql_args(config: &KVS, default_queue_dir: &str, target_type: Targ
         tls_ca: config.lookup(MYSQL_TLS_CA).unwrap_or_default(),
         tls_client_cert: config.lookup(MYSQL_TLS_CLIENT_CERT).unwrap_or_default(),
         tls_client_key: config.lookup(MYSQL_TLS_CLIENT_KEY).unwrap_or_default(),
-        queue_dir: config
-            .lookup(MYSQL_QUEUE_DIR)
-            .unwrap_or_else(|| default_queue_dir.to_string()),
+        queue_dir: queue_dir_or_default(config, MYSQL_QUEUE_DIR, default_queue_dir),
         queue_limit: config
             .lookup(MYSQL_QUEUE_LIMIT)
             .and_then(|v| v.parse::<u64>().ok())

crates/targets/src/store.rs

@@ -15,12 +15,13 @@
 use crate::error::StoreError;
 use rustfs_config::notify::{COMPRESS_EXT, DEFAULT_EXT};
 use rustfs_config::{
-    DEFAULT_LIMIT, DEFAULT_TARGET_STORE_COMPRESS, DEFAULT_TARGET_STORE_MAX_BYTES, ENV_TARGET_STORE_COMPRESS,
-    ENV_TARGET_STORE_MAX_BYTES, EnableState,
+    AUDIT_DEFAULT_DIR, DEFAULT_LIMIT, DEFAULT_TARGET_STORE_COMPRESS, DEFAULT_TARGET_STORE_MAX_BYTES, ENV_TARGET_STORE_COMPRESS,
+    ENV_TARGET_STORE_DIR, ENV_TARGET_STORE_MAX_BYTES, EVENT_DEFAULT_DIR, EnableState,
 };
 use serde::{Serialize, de::DeserializeOwned};
 use snap::raw::{Decoder, Encoder};
 use std::{
+    borrow::Cow,
     collections::HashMap,
     fs::File,
     io::Write,
@@ -155,6 +156,46 @@ fn queue_store_compression_enabled() -> bool {
     resolve_queue_store_compression_from_env_value(value.as_deref())
 }
 
+/// Map a configured target `queue_dir` to the directory this node opens.
+///
+/// `EVENT_DEFAULT_DIR` and `AUDIT_DEFAULT_DIR` are persisted cluster-wide as logical
+/// defaults, so every node resolves them locally. `RUSTFS_TARGET_STORE_DIR` relocates
+/// their root on any platform. Without it, Windows uses `%ProgramData%\RustFS`, because
+/// the Unix literal has no drive prefix and is not an absolute path there. Every other
+/// value, including an explicit operator path, is returned unchanged.
+pub fn resolve_queue_dir(queue_dir: &str) -> Cow<'_, str> {
+    let store_root = std::env::var(ENV_TARGET_STORE_DIR).ok();
+    resolve_queue_dir_with(queue_dir, store_root.as_deref(), platform_queue_root().as_deref())
+}
+
+#[cfg(windows)]
+fn platform_queue_root() -> Option<PathBuf> {
+    std::env::var_os("ProgramData")
+        .filter(|root| !root.is_empty())
+        .map(|root| PathBuf::from(root).join("RustFS"))
+}
+
+#[cfg(not(windows))]
+fn platform_queue_root() -> Option<PathBuf> {
+    None
+}
+
+fn resolve_queue_dir_with<'a>(queue_dir: &'a str, store_root: Option<&str>, platform_root: Option<&Path>) -> Cow<'a, str> {
+    let leaf = match queue_dir {
+        EVENT_DEFAULT_DIR => "events",
+        AUDIT_DEFAULT_DIR => "audit",
+        _ => return Cow::Borrowed(queue_dir),
+    };
+    let root = match store_root.map(str::trim).filter(|root| !root.is_empty()) {
+        Some(root) => Path::new(root),
+        None => match platform_root {
+            Some(root) => root,
+            None => return Cow::Borrowed(queue_dir),
+        },
+    };
+    Cow::Owned(root.join(leaf).to_string_lossy().into_owned())
+}
+
 /// Represents a key for an entry in the store
 #[derive(Debug, Clone)]
 pub struct Key {
@@ -1365,6 +1406,56 @@ mod tests {
         thread,
     };
 
+    #[test]
+    fn logical_default_queue_dirs_resolve_under_the_configured_root() {
+        let root = Path::new("/srv/rustfs-queues");
+        let platform = Path::new("/platform/RustFS");
+        assert_eq!(
+            resolve_queue_dir_with(EVENT_DEFAULT_DIR, Some("/srv/rustfs-queues"), Some(platform)),
+            root.join("events").to_string_lossy()
+        );
+        assert_eq!(
+            resolve_queue_dir_with(AUDIT_DEFAULT_DIR, Some(" /srv/rustfs-queues "), None),
+            root.join("audit").to_string_lossy()
+        );
+        assert_eq!(
+            resolve_queue_dir_with(EVENT_DEFAULT_DIR, Some("  "), Some(platform)),
+            platform.join("events").to_string_lossy(),
+            "a blank override must fall back to the platform root"
+        );
+    }
+
+    #[test]
+    fn explicit_and_unmapped_queue_dirs_are_preserved() {
+        let platform = Path::new("/platform/RustFS");
+        for explicit in [
+            "/var/lib/rustfs/events",
+            "/opt/rustfs/events/",
+            "/opt/rustfs",
+            "",
+            "relative/queue",
+        ] {
+            assert_eq!(
+                resolve_queue_dir_with(explicit, Some("/srv/rustfs-queues"), Some(platform)),
+                explicit,
+                "only the exact logical defaults may be relocated"
+            );
+        }
+        assert_eq!(resolve_queue_dir_with(EVENT_DEFAULT_DIR, None, None), EVENT_DEFAULT_DIR);
+        assert_eq!(resolve_queue_dir_with(AUDIT_DEFAULT_DIR, None, None), AUDIT_DEFAULT_DIR);
+    }
+
+    #[cfg(windows)]
+    #[test]
+    fn windows_logical_default_queue_dirs_are_absolute_without_an_override() {
+        let platform = platform_queue_root().expect("Windows defines ProgramData");
+        for default in [EVENT_DEFAULT_DIR, AUDIT_DEFAULT_DIR] {
+            let resolved = resolve_queue_dir_with(default, None, Some(&platform));
+            assert!(Path::new(resolved.as_ref()).is_absolute(), "{default} resolved to {resolved}");
+            assert!(resolved.contains("RustFS"), "{default} resolved to {resolved}");
+        }
+    }
+
     fn temp_store_dir(name: &str) -> PathBuf {
         std::env::temp_dir().join(format!("rustfs-targets-{name}-{}", Uuid::new_v4()))
     }
@@ -1456,7 +1547,13 @@ mod tests {
         store.put_failed_raw("failed-event", &[4; 16]).unwrap();
         assert_eq!(store.stored_bytes.load(Ordering::SeqCst), 48);
         let failed = store.failed_dir().join("failed-event");
-        File::open(&failed).unwrap().set_modified(UNIX_EPOCH).unwrap();
+        // Windows needs write-attribute access on the handle to change mtime.
+        std::fs::OpenOptions::new()
+            .write(true)
+            .open(&failed)
+            .unwrap()
+            .set_modified(UNIX_EPOCH)
+            .unwrap();
         assert_eq!(store.prune_failed_store().unwrap(), 1);
         assert_eq!(store.stored_bytes.load(Ordering::SeqCst), 32);
         std::fs::create_dir(store.failed_dir().join("cannot-replace-directory")).unwrap();

crates/utils/src/logging.rs

@@ -18,6 +18,7 @@ use std::sync::atomic::{AtomicU64, Ordering};
 
 /// Bounds a repetitive diagnostic without changing its underlying counters.
 /// Each emitted event includes the number suppressed since the previous one.
+#[derive(Debug)]
 pub struct LogThrottle {
     interval_ms: u64,
     last_ms: AtomicU64,

docs/architecture/s3-compatibility-matrix.md

@@ -130,7 +130,7 @@ These behaviors are client-visible and differ from AWS S3 or MinIO. They bound h
 | Bucket tag key or value outside the AWS tag character class (`[\p{L}\p{N}\p{Z}+\-=._:/@]`) | `400 InvalidTag` on `PutBucketTagging` | AWS enforces the same class; MinIO is more permissive | AWS parity. Legacy stored tags still load. |
 | `CopyObject` with the default `x-amz-tagging-directive: COPY` from a source that carries tags | `403 AccessDenied` unless the caller also holds `s3:GetObjectTagging` (or `s3:GetObjectVersionTagging`) on the source; the destination's `s3:RequestObjectTag/*` conditions are evaluated against the copied tag set | AWS and MinIO copy the tags with `s3:GetObject` alone | A copy would otherwise disclose source tags to a principal that may not read them, and source tags could satisfy destination conditions the caller cannot assert (rustfs/backlog#3034). Use `REPLACE` with an explicit `x-amz-tagging` to copy without the tagging grant. |
 | `ListObjectVersions` / `ListObjectsV2` with the include-deleted extension | Requires `s3:ListBucketVersions` | Vendor extension | Delete markers and non-current versions are version-listing data. |
-| `DeleteObject` of a missing key that names a prefix | `204 NoContent` when the recovery probe of the prefix fits the request budget (`DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES`, 4096 entries per drive, one entry per 64 plain children); `503 SlowDown` beyond that, deterministically, and for the duration of a rolling upgrade while a drive in the set still runs a release without the bounded recovery RPC | `204 NoContent` | The probe looks for interrupted delete transactions under the prefix and is bounded so a hostile prefix cannot pin the request. Prefixes with more than 262,144 direct children per drive, or roughly 1,400 to 4,096 UUID-named children, exceed the bound (rustfs/backlog#3038 residual). |
+| `DeleteObject` of a missing key that names a prefix | `204 NoContent` when the recovery probe of the prefix fits its per-drive allowance (`DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES`, 262,144 entries: one per UUID-named direct child, one per 64 other names), its per-call time limit (a third of the drive timeout, at most 10 seconds) and the request budget (`DeleteRecoveryWorkBudget`, shared by every key and drive of the request); `503 SlowDown` beyond that, and for the duration of a rolling upgrade while a drive in the set still runs a release without the bounded recovery RPC | `204 NoContent` | The probe looks for interrupted delete transactions of the key and is bounded so a hostile prefix cannot pin the request. A folder completes with up to 262,144 UUID-named direct children or about 16.7 million other direct children per drive when the drive can walk them within the time limit (a warm cache walks roughly 25,000 UUID-named children per second; a cold HDD far fewer); inside a UUID-named child only journal files and data-directory markers are examined, so a child key's own `xl.meta`, data directories or children cost 1/64 each. |
 | `PutObject` / `UploadPart` body delivering less than 64 KiB per 60 seconds of server wait | `400 RequestTimeout` (the minimum-progress floor, logged as `upload_body_progress_stalled`); independent of `RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT`, including when that timeout is 0 | No minimum upload rate | A trickle upload otherwise pins upload admission and its buffers indefinitely; roughly 1.1 KiB/s is below any practical client. |
 | First write after `SetBucketQuota`, or after the quota changes, on a bucket without an authoritative usage ledger | `503 SlowDown` (retryable) until a background scan of the bucket publishes the exact ledger; a failed scan retries after 30 seconds | AWS has no bucket quota; MinIO admits with unknown usage | Quota admission uses only the authoritative ledger (scanner reporting is informational, see `reporting_retirement` tests), and the request must not hold its bucket metadata guard across a whole-bucket listing. The window is one bucket listing long. |
 

docs/operations/s3-tables-catalog-lifecycle.md

@@ -13,7 +13,7 @@ The server persists an inactive catalog entry before clearing the bucket marker.
 
 ## Clean Up Deleted Empty Tables
 
-`DELETE /iceberg/v1/buckets/{warehouse}?cleanup=true` removes recognized metadata and internal records left by deleted empty tables, then disables the catalog. Ordinary table deletion continues to preserve files. Cleanup preserves the existing table-record formats and accepts eligible empty tables deleted before the server upgrade.
+`DELETE /iceberg/v1/buckets/{warehouse}?cleanup=true` removes recognized metadata and internal records left by deleted empty tables, then disables the catalog. Ordinary table deletion removes the table-specific maintenance configuration after removing the table entry. It preserves Iceberg data and metadata files. Bucket cleanup also accepts valid table-specific configurations left by earlier releases or interrupted drops. Remaining table entries and maintenance job records block bucket cleanup.
 
 Complete the upgrade across all serving nodes and set `RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED=true` before cleanup. Delete the empty tables and namespaces through catalog APIs. The cleanup principal needs `admin:SetTableBucket` and `s3:GetObject`/`s3:DeleteObject` permissions for the target metadata objects. Omitting `cleanup` or setting it to `false` keeps ordinary disablement; repeated or invalid values are rejected.
 

docs/operations/target-queue-storage.md

@@ -1,4 +1,22 @@
-# Target queue storage limits
+# Target queue storage
+
+## Queue directory location
+
+A target's `queue_dir` is stored in the cluster-wide target configuration. The built-in defaults `/opt/rustfs/events` and `/opt/rustfs/audit` are logical defaults: each node resolves them to a local directory when it validates or opens a queue. The stock notify and audit default tables both persist `/opt/rustfs/events`; `/opt/rustfs/audit` applies only to an audit target whose configuration has no `queue_dir` key. Any other `queue_dir`, including an explicit operator path, is used as written and must be absolute.
+
+| Node setting | `/opt/rustfs/events` resolves to | `/opt/rustfs/audit` resolves to |
+|---|---|---|
+| `RUSTFS_TARGET_STORE_DIR=<root>` (any platform) | `<root>/events` | `<root>/audit` |
+| Windows, variable unset | `%ProgramData%\RustFS\events` | `%ProgramData%\RustFS\audit` |
+| Linux/macOS, variable unset | `/opt/rustfs/events` | `/opt/rustfs/audit` |
+
+`RUSTFS_TARGET_STORE_DIR` must be an absolute path; a relative value makes target validation fail with the queue-directory error. The queue store creates missing directories when it opens, so the resolved directory only needs to be creatable by the RustFS process. An admin request that names a default path explicitly is checked at its resolved location and, as for any explicit `queue_dir`, that directory must already exist.
+
+The official container image links `/opt/rustfs/events` and `/opt/rustfs/audit` into `/data/.rustfs`, so its queues survive container recreation without this variable. Windows previously rejected targets that kept the default because the Unix literal is not an absolute Windows path; those targets now validate and queue under `%ProgramData%\RustFS`.
+
+Setting or changing `RUSTFS_TARGET_STORE_DIR` on an existing node moves where its default queues are opened. Entries still pending in the previous directory are not migrated: let them drain first, or move the directory contents while RustFS is stopped. Rolling back to a version without this variable returns default queues to the previous location.
+
+## Queue size limits
 
 Disk queues for notification and audit targets enforce both their configured entry limit and a shared byte limit for each queue directory. `RUSTFS_TARGET_STORE_MAX_BYTES` sets that byte limit; the default is 1 GiB. The value must be a positive integer in bytes. Missing, zero, invalid, and overflowing values use the default.
 

docs/testing/ci-gates.md

@@ -54,6 +54,7 @@ The four site-replication state-writer concurrency proofs reserve the available
 | PR, non-doc change | `Typos` | `ci.yml` `typos` | Via aggregate | `typos` |
 | Full selection | `Workspace tests` (serial ILM step) | `ci.yml` `workspace-tests` | Via aggregate | exact `Run ignored ILM integration tests serially` command in the job |
 | Nightly schedule, manual dispatch | `Test and Lint (rio-v2)`, `Test and Lint (swift)`, `Test and Lint (sftp)`, `Test and Lint (http3)` | `ci.yml` `test-and-lint-rio-v2`, `test-and-lint-protocols` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo nextest run` with the job's `--features` |
+| Full selection, every event | `Test and Lint (gateway-stack)` | `ci.yml` `test-and-lint-gateway-stack` | Via aggregate | `cargo clippy -p rustfs --all-targets --features gateway-stack -- -D warnings`; `cargo nextest run -p rustfs --features gateway-stack` with the job's filter |
 | Nightly schedule, manual dispatch, full selection | `Connect Feature Boundaries` | `ci.yml` `connect-feature-boundaries` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo test -p rustfs --test connect_registration --features connect-e2e-short-credentials`; `cargo check -p rustfs --release --features connect-e2e-short-credentials` must fail; `scripts/check_offline_enrollment_e2e.sh` |
 | PR, non-doc change | `Build RustFS Debug Binary` | `ci.yml` `build-rustfs-debug-binary` | Via aggregate; prerequisite for black-box jobs | `python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks` (binary plus its `rustfs.e2e.json` sidecar) |
 | Nightly schedule, manual dispatch, full selection | `io_uring Integration (real)` | `ci.yml` `uring-integration` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture` |
@@ -101,6 +102,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run. The v
 | `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes (disabled) | `make coverage` |
 | `runner-hygiene.yml` (monthly) | `check-ephemerality` | runner ephemerality | yes | dispatch |
 | `e2e-upgrade.yml` (weekly, and on version tags) | `upgrade` (serial cases sharing one harness build) | upgrade and rollback gate; server logs | yes | `python3 scripts/e2e_binary.py build`, then the job's `python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test` commands with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
+| `reference-parity.yml` (nightly, and on PRs that change the lane's own files) | `plan`, `build-candidate`, `s3tests` (one job per target in `.config/reference-parity-targets.toml`), `compare` | report-only parity gate: fails on any per-node-id s3-tests difference between a branch candidate and the release pinned in `.config/reference-binary.toml` that `.config/reference-parity-tolerated.txt` does not list, and on a stale tolerated entry; per-target JUnit, logs and `diff.md`/`diff.json` | yes | `DEPLOY_MODE=binary RUSTFS_BINARY=<binary> MAXFAIL=0 XDIST=4 ./scripts/s3-tests/run.sh` once per target, then `python3 scripts/s3-tests/reference_parity.py compare` as in the `compare` job |
 | `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | yes | see the PR row |
 | `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | yes (disabled) | see the PR row |
 | `gateway-integration-sync.yml` (weekdays) | `sync` | merges `main` into `gateway/integration` as a `--no-ff` merge commit; a conflicting merge is aborted and opens a `sync/<source>-<date>` pull request listing the conflicting files, which keeps the run green; a rejected push or missing pull request fails it | yes | dispatch, optionally with `source_ref` |

helm/README.md

@@ -157,6 +157,7 @@ behavior.
 | config.rustfs.obs_environment | string | `"development"` |  |
 | config.rustfs.obs_log_directory | string | `"/logs"` | Log directory inside the RustFS container. Set to `""` to disable log PVCs and mounts. |
 | config.rustfs.region | string | `"us-east-1"` |  |
+| config.rustfs.target_store_dir | string | `""` | Absolute root for notify and audit target queues that keep the default `queue_dir`, set as `RUSTFS_TARGET_STORE_DIR`; queues move to `<root>/events` and `<root>/audit`. Empty keeps the image layout, where `/opt/rustfs/events` and `/opt/rustfs/audit` link into `/data/.rustfs`. |
 | config.rustfs.volumes | string | `""` | Explicit distributed volume topology. When empty, the chart generates the topology and normally injects `RUSTFS_LOCAL_ENDPOINT_HOST`; custom topologies must configure local endpoint identity explicitly when needed. |
 | config.rustfs.log_rotation.size | int | `"100"` | Default log rotation size mb for rustfs. |
 | config.rustfs.log_rotation.time | string | `"hour"` | Default log rotation time for rustfs. |

helm/rustfs/templates/configmap.yaml

@@ -21,6 +21,9 @@ data:
   {{- if .region }}
   RUSTFS_REGION: {{ .region | quote }}
   {{- end }}
+  {{- if .target_store_dir }}
+  RUSTFS_TARGET_STORE_DIR: {{ .target_store_dir | quote }}
+  {{- end }}
   {{- if .domains }}
   RUSTFS_SERVER_DOMAINS: {{ include "rustfs.serverDomains" $ | quote }}
   {{- end }}

helm/rustfs/values.yaml

@@ -143,6 +143,9 @@ config:
     console_enable: "true"
     console_address: ":9001"
     region: "us-east-1"
+    # Root for notify/audit target queues that keep the default queue_dir; queues use
+    # <root>/events and <root>/audit. Empty keeps the image layout (/opt/rustfs -> /data/.rustfs).
+    target_store_dir: ""
     # Optionally enable support for virtual-hosted-style requests.
     # See more information: https://docs.rustfs.com/integration/virtual.html
     domains: "" # e.g. "example.com"

rustfs/Cargo.toml

@@ -56,6 +56,12 @@ required-features = ["swift"]
 name = "swift_object_integration_test"
 required-features = ["swift"]
 
+# Starts an embedded server on the gateway stack, which only a build with the
+# feature can select.
+[[test]]
+name = "embedded_gateway_stack_test"
+required-features = ["gateway-stack"]
+
 [features]
 default = ["ftps", "webdav", "gcs"]
 fips = ["rustls/fips", "tokio-rustls/fips", "rustfs-tls-runtime/fips", "rustfs-ecstore/fips", "rustfs-protocols/fips", "rustfs-s3-client/fips", "rustfs-targets/fips", "rustfs-utils/fips"]
@@ -69,12 +75,19 @@ license = []
 io-scheduler-debug = []  # Enable debug information in I/O scheduler
 tracing-chunk-debug = []  # Enable per-chunk tracing in data plane (high noise, for debugging only)
 full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope", "gcs"]
-e2e-test-hooks = ["rustfs-ecstore/e2e-test-hooks", "rustfs-scanner/e2e-test-hooks"]
+# The debug-only E2E binary also carries the gateway S3 stack, so the gateway smoke
+# lane runs the same artifact as the legacy one with RUSTFS_S3_STACK as its only change.
+e2e-test-hooks = ["rustfs-ecstore/e2e-test-hooks", "rustfs-scanner/e2e-test-hooks", "gateway-stack"]
 # Shortens Connect credentials only in debug E2E builds.
 connect-e2e-short-credentials = []
 # Builds the dedicated rustfs-cli-e2e target with a build-time public enrollment root.
 offline-enrollment-e2e-root = []
 rio-v2 = ["rustfs-ecstore/rio-v2"]
+# The rustfs-gateway S3 stack, selected at startup with RUSTFS_S3_STACK=gateway
+# (rustfs/backlog#2734). Under construction: off by default and never enabled in a
+# release build; the stack refuses to compile without debug assertions, and
+# `scripts/check_gateway_stack_release_boundary.sh` keeps it out of release inputs.
+gateway-stack = ["dep:rustfs-gateway", "dep:rustfs-gateway-adapter"]
 pyroscope = ["rustfs-obs/pyroscope", "dep:pyroscope", "dep:object", "dep:symbolic-demangle"]
 # Tokio runtime telemetry. Requires `--cfg tokio_unstable`; use `make build-profiling`.
 dial9 = ["rustfs-obs/dial9"]
@@ -255,6 +268,8 @@ rustfs-notify = { workspace = true }
 rustfs-obs = { workspace = true }
 rustfs-policy = { workspace = true }
 rustfs-protocols = { workspace = true }
+rustfs-gateway = { workspace = true, optional = true }
+rustfs-gateway-adapter = { workspace = true, optional = true }
 rustfs-gateway-types = { workspace = true, features = ["compat-s3s-rustfs"] }
 rustfs-protos = { workspace = true }
 rustfs-rio = { workspace = true }

rustfs/src/admin/handlers/account.rs

@@ -165,7 +165,8 @@ impl Operation for ChangeOwnPasswordHandler {
         let signing_secret = req
             .credentials()
             .ok_or_else(|| s3::error(S3ErrorCode::AccessDenied, "authentication required"))?
-            .secret_key()
+            .require_secret_key()
+            .map_err(|err| s3::error(S3ErrorCode::InternalError, err.to_string()))?
             .to_string();
         let caller = CallerIdentity::resolve_envelope(req.envelope()).await?;
         let audit = AccountAuditContext::from_envelope(req.envelope());

rustfs/src/admin/handlers/bucket_recovery.rs

@@ -63,8 +63,10 @@ impl Operation for RecoverOrphanedBucketHandler {
         authorize_admin_envelope(req.envelope(), vec![Action::AdminAction(AdminAction::RecoverOrphanedBucketAction)]).await?;
         let secret_key = req
             .credentials()
-            .map(|credentials| Zeroizing::new(credentials.secret_key().to_owned()))
-            .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "authentication required"))?;
+            .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "authentication required"))?
+            .require_secret_key()
+            .map(|secret| Zeroizing::new(secret.to_owned()))
+            .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, err.to_string()))?;
         let bucket = params
             .get("bucket")
             .filter(|bucket| !bucket.is_empty())

rustfs/src/admin/handlers/object_zip_download.rs

@@ -15,11 +15,12 @@
 use crate::admin::operation::{AdminRequest, AdminResponse, Operation};
 use crate::admin::router::{ADMIN_OBJECT_ZIP_DOWNLOADS_PATH, AdminOperation, S3Router};
 use crate::admin::runtime_sources::{ServerContextSlot, current_action_credentials, current_object_store_handle, current_region};
-use crate::admin::storage_api::access::{ReqInfo, authorize_request};
+use crate::admin::storage_api::access::{ReqInfo, authorize_owned_request, authorize_request};
 use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions};
 use crate::admin::storage_api::contract::list::ListOperations as _;
 use crate::admin::storage_api::contract::object::{ObjectIO as _, ObjectOperations as _};
 use crate::admin::storage_api::object::StorageObjectOptions as ObjectOptions;
+use crate::app::{RequestEnvelope, RequestHead};
 use crate::auth::{check_key_valid, get_session_token};
 use crate::error::ApiError;
 use crate::license::license_check;
@@ -495,25 +496,32 @@ async fn authorize_object_zip_download_s3_action(
         req_info.version_id = None;
     }
 
-    let mut authorization = s3_authorization_request(
-        req.method().clone(),
-        req.uri().clone(),
-        &authenticated_zip_download_auth_context(req),
-        req.envelope().server_context().cloned(),
-        req.req_info().cloned(),
+    let mut extensions = http::Extensions::new();
+    extensions.insert(req.remote_addr().map(RemoteAddr));
+    if let Some(client_info) = req.client_info() {
+        extensions.insert(client_info.clone());
+    }
+    if let Some(server_context) = req.envelope().server_context() {
+        extensions.insert(server_context.clone());
+    }
+    if let Some(req_info) = req.req_info() {
+        extensions.insert(req_info.clone());
+    }
+    let mut authorization = RequestEnvelope::from_parts(
+        RequestHead {
+            method: req.method().clone(),
+            uri: req.uri().clone(),
+            headers: req.headers().clone(),
+            credentials: None,
+            region: None,
+            service: None,
+        },
+        extensions,
     );
-    authorize_request(&mut authorization, Action::S3Action(action))
-        .await
-        .map_err(Into::into)
+    authorize_owned_request(&mut authorization, Action::S3Action(action), None).await
 }
 
-/// The S3 gate (`storage::access::authorize_request`) still evaluates the s3s
-/// request shape, so the facts it reads are copied into one for the duration
-/// of the call: the authenticated `ReqInfo`, the peer and proxy-reported client
-/// addresses, the headers and query the policy conditions come from, and the
-/// server slot that selects the store. The gate reads neither the presented
-/// credentials nor the region or service, so those stay unset, and nothing it
-/// attaches to the request is read afterwards.
+/// Builds the legacy authorization request used when a download token is redeemed.
 fn s3_authorization_request(
     method: Method,
     uri: Uri,
@@ -943,6 +951,103 @@ mod tests {
     use super::*;
     use rustfs_s3_types::S3ErrorCode;
 
+    #[tokio::test]
+    async fn zip_scope_authorizes_real_iam_and_restores_prefix_query() {
+        use crate::admin::operation::test_support::TestRequest;
+        use crate::admin::runtime_sources::{AppContext, IamInterface, KmsInterface};
+        use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
+        use rustfs_iam::store::{Store as _, UserType};
+        use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq};
+        use rustfs_policy::policy::Policy;
+
+        struct ReadyIam(Arc<rustfs_iam::sys::IamSys<rustfs_iam::store::object::ObjectStore>>);
+        impl IamInterface for ReadyIam {
+            fn handle(&self) -> Arc<rustfs_iam::sys::IamSys<rustfs_iam::store::object::ObjectStore>> {
+                Arc::clone(&self.0)
+            }
+            fn is_ready(&self) -> bool {
+                true
+            }
+        }
+        struct TestKms;
+        impl KmsInterface for TestKms {
+            fn handle(&self) -> Arc<rustfs_kms::KmsServiceManager> {
+                Arc::new(rustfs_kms::KmsServiceManager::new())
+            }
+        }
+
+        let (_temp_dir, _disk_paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
+        store
+            .make_bucket("photos", &MakeBucketOptions::default())
+            .await
+            .expect("create bucket");
+        rustfs_iam::store::object::ObjectStore::new(Arc::clone(&store))
+            .save_iam_config(
+                serde_json::json!({"version": 1}),
+                format!("{}/format.json", *rustfs_iam::store::object::IAM_CONFIG_PREFIX),
+            )
+            .await
+            .expect("seed IAM format");
+        let iam = rustfs_iam::build_iam_sys(Arc::clone(&store)).await.expect("initialize IAM");
+        let access_key = "zip-scope-user";
+        iam.create_user(
+            access_key,
+            &AddOrUpdateUserReq {
+                secret_key: "zip-scope-secret-key".to_owned(),
+                policy: None,
+                status: AccountStatus::Enabled,
+            },
+        )
+        .await
+        .expect("create user");
+        iam.set_policy(
+            "zip-scope-policy",
+            Policy::parse_config(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:ListBucket","Resource":"arn:aws:s3:::photos","Condition":{"StringEquals":{"s3:prefix":"allowed/"}}},{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/allowed/*"}]}"#)
+                .expect("parse policy"),
+        )
+        .await
+        .expect("store policy");
+        iam.policy_db_set(access_key, UserType::Reg, false, "zip-scope-policy")
+            .await
+            .expect("attach policy");
+        let credentials = iam.get_user(access_key).await.expect("load user").credentials;
+        let context = Arc::new(AppContext::new(store, Arc::new(ReadyIam(iam)), Arc::new(TestKms)));
+        let slot = ServerContextSlot::new();
+        assert!(slot.install(context));
+        let original_uri = "/rustfs/admin/v3/object-zip-downloads?prefix=old&marker=1";
+        let mut request = TestRequest::new(Method::POST, original_uri)
+            .extension(slot)
+            .extension(ReqInfo {
+                cred: Some(credentials),
+                ..Default::default()
+            })
+            .build();
+
+        let mut selection = valid_request();
+        selection.prefix = None;
+        selection.prefixes = vec!["allowed/".to_owned()];
+        selection.objects = vec!["allowed/file".to_owned()];
+        authorize_object_zip_download_scope(&mut request, &selection)
+            .await
+            .expect("allowed prefix and object");
+        assert_eq!(request.uri().to_string(), original_uri);
+
+        selection.prefixes = vec!["denied/".to_owned()];
+        let err = authorize_object_zip_download_scope(&mut request, &selection)
+            .await
+            .expect_err("another prefix is denied");
+        assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
+        assert_eq!(request.uri().to_string(), original_uri);
+
+        selection.prefixes = vec!["allowed/".to_owned()];
+        selection.objects = vec!["denied/file".to_owned()];
+        let err = authorize_object_zip_download_scope(&mut request, &selection)
+            .await
+            .expect_err("another object is denied");
+        assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
+        assert_eq!(request.uri().to_string(), original_uri);
+    }
+
     fn valid_request() -> CreateObjectZipDownloadRequest {
         CreateObjectZipDownloadRequest {
             bucket: "photos".to_string(),

rustfs/src/admin/handlers/service_account.rs

@@ -343,7 +343,7 @@ impl Operation for AddServiceAccount {
         let (cred, owner) =
             check_key_valid(get_session_token(req.uri(), req.headers()).unwrap_or_default(), req_cred.access_key()).await?;
 
-        if !constant_time_eq(&cred.secret_key, req_cred.secret_key()) {
+        if !req_cred.secret_key_matches(&cred.secret_key) {
             return Err(S3Error::with_message(S3ErrorCode::AccessDenied, "access denied"));
         }
 

rustfs/src/admin/handlers/sts.rs

@@ -222,7 +222,7 @@ async fn handle_assume_role(
 
     // The signature was verified before the request body was consumed. A fresh
     // IAM lookup must not silently replace the secret that authenticated it.
-    if !crate::auth::constant_time_eq(&cred.secret_key, user.secret_key()) {
+    if !user.secret_key_matches(&cred.secret_key) {
         return Err(S3Error::with_message(S3ErrorCode::AccessDenied, "Access Denied"));
     }
 

rustfs/src/admin/handlers/table_catalog/mod.rs

@@ -15,7 +15,7 @@
 use crate::admin::operation::{AdminRequest, AdminResponse};
 use crate::admin::runtime_sources;
 use crate::admin::runtime_sources::default_admin_usecase;
-use crate::admin::storage_api::access::{ReqInfo, authorize_internal_object_request};
+use crate::admin::storage_api::access::{ReqInfo, authorize_owned_internal_object_request};
 use crate::admin::storage_api::bucket::metadata::table_catalog_path_hash;
 use crate::admin::storage_api::runtime::ECStore;
 use crate::admin::utils::owned_empty_response;
@@ -1088,12 +1088,8 @@ fn build_sensitive_json_response<T: Serialize>(status: StatusCode, body: &T) ->
     Ok(AdminResponse::with_headers(status, body, headers))
 }
 
-// The commit-path handlers (`TableCommitObjectBackend::for_request`) authorize
-// exact object keys through `storage::access::authorize_internal_object_request`,
-// which still takes the s3s request. Until that path has an owned form they keep
-// the s3s request and read every other request fact through this owned view.
-// TODO(T1.12-codemod): remove once the generic object authorization path moves
-// to the request envelope.
+// These handlers still use the legacy router while their request bodies migrate.
+// TODO(T1.12-codemod): move them to the owned operation boundary.
 type LegacyRequest = s3s::S3Request<s3s::Body>;
 type LegacyResponse = s3s::S3Response<(StatusCode, s3s::Body)>;
 
@@ -1318,15 +1314,14 @@ fn install_table_catalog_s3_request_info(req: &mut LegacyRequest, principal: &Ta
 }
 
 async fn authorize_table_catalog_s3_actions(
-    req: &mut LegacyRequest,
+    req: &mut AdminRequest,
     bucket: &str,
     object: &str,
     actions: &[S3Action],
 ) -> S3Result<()> {
     let original = {
         let req_info = req
-            .extensions
-            .get_mut::<ReqInfo>()
+            .req_info_mut()
             .ok_or_else(|| s3_error!(AccessDenied, "authentication required"))?;
         (
             req_info.bucket.replace(bucket.to_string()),
@@ -1337,15 +1332,15 @@ async fn authorize_table_catalog_s3_actions(
 
     let mut result = Ok(());
     for action in actions {
-        if let Err(err) = authorize_internal_object_request(req, Action::S3Action(*action)).await {
+        if let Err(err) = authorize_owned_internal_object_request(req.envelope_mut(), Action::S3Action(*action)).await {
             result = Err(err);
             break;
         }
     }
-    if let Some(req_info) = req.extensions.get_mut::<ReqInfo>() {
+    if let Some(req_info) = req.req_info_mut() {
         (req_info.bucket, req_info.object, req_info.version_id) = original;
     }
-    result.map_err(Into::into)
+    result
 }
 
 async fn table_catalog_request_principal(req: &AdminRequest) -> S3Result<TableCatalogRequestPrincipal> {
@@ -1383,7 +1378,7 @@ async fn table_catalog_request_principal(req: &AdminRequest) -> S3Result<TableCa
 
 #[derive(Clone)]
 enum TableCommitObjectAuthorization {
-    Request(Arc<tokio::sync::Mutex<LegacyRequest>>),
+    Request(Arc<tokio::sync::Mutex<AdminRequest>>),
     Preauthorized,
     #[cfg(test)]
     Test {
@@ -1459,7 +1454,7 @@ where
     fn for_request(backend: B, req: LegacyRequest) -> Self {
         Self::new(
             backend,
-            TableCommitObjectAuthorization::Request(Arc::new(tokio::sync::Mutex::new(req))),
+            TableCommitObjectAuthorization::Request(Arc::new(tokio::sync::Mutex::new(legacy_request_view(&req)))),
             rustfs_utils::get_env_bool(crate::table_catalog::ENV_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED, false),
         )
     }
@@ -2093,7 +2088,7 @@ where
         let result = match &self.authorization {
             TableCommitObjectAuthorization::Request(req) => {
                 let req = req.lock().await;
-                authorize_table_warehouse_location_request(&legacy_request_view(&req), table_bucket, location).await
+                authorize_table_warehouse_location_request(&req, table_bucket, location).await
             }
             TableCommitObjectAuthorization::Preauthorized => Ok(()),
             #[cfg(test)]