Merge remote-tracking branch 'origin/main' into gw/T2.4-observer-audit

# Conflicts:
#	crates/gateway-adapter/src/lib.rs

.config/make/lint-fmt.mak

@@ -75,6 +75,11 @@ test-count-ratchet-check: ## Check no workspace crate fell below its test floor
 	@echo "🧪 Checking per-crate test-count ratchet..."
 	./scripts/check_test_count_ratchet.sh
 
+.PHONY: test-seams-check
+test-seams-check: ## Check tests outside an operation's owner call its use-case only through rustfs/src/app/test_seams
+	@echo "🧪 Checking cross-owner test seams..."
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_seams.py
+
 .PHONY: fips-wording-check
 fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities
 	@echo "📣 Checking cryptographic capability wording guard..."

.config/make/pre-commit.mak

@@ -20,11 +20,11 @@ planning-docs-check: ## Check that no planning-type documents are committed
 	./scripts/check_no_planning_docs.sh
 
 .PHONY: pre-commit
-pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
+pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check test-seams-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
 	@echo "✅ All pre-commit checks passed!"
 
 .PHONY: pre-pr
-pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
+pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check test-seams-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
 	@echo "✅ All pre-PR checks passed!"
 
 .PHONY: dev-check

.config/make/tests.mak

@@ -64,8 +64,11 @@ script-tests: ## Run shell script tests
 	./scripts/test_check_test_count_ratchet.sh
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/codemods/split_lifecycle_transition_api_test.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_seams.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_download_release_asset.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_reference_binary.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
 	$(RUSTFS_PYTHON_BIN) -m unittest discover -s scripts/release -p "test_*.py"
@@ -92,6 +95,7 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_junit_diff.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_reference_parity.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
 	./scripts/validate_object_data_cache_cold_stampede.sh --self-test

.config/reference-binary.toml

@@ -0,0 +1,20 @@
+# Pinned RustFS release used as the "reference" target of parity lanes
+# (.github/workflows/reference-parity.yml). This is the single place that names
+# the reference release; .github/actions/reference-binary resolves, downloads and
+# verifies it, and fails on a missing pin, a missing asset or a digest mismatch.
+#
+# Bumping the reference is a reviewed event: every per-test difference between
+# the candidate and the reference is measured against this exact build.
+#
+# Each sha256 is the value published in the release's SHA256SUMS asset, which
+# matches the digest GitHub reports for the same asset. The action checks both.
+
+schema = 1
+repository = "rustfs/rustfs"
+version = "1.0.5"
+tag = "1.0.5"
+
+[platforms.linux-x86_64-gnu]
+asset = "rustfs-linux-x86_64-gnu-v1.0.5.zip"
+sha256 = "5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0"
+executable = "rustfs"

.config/reference-parity-targets.toml

@@ -0,0 +1,24 @@
+# Targets of the s3-tests reference parity lane (.github/workflows/reference-parity.yml).
+#
+# Exactly one target uses `binary = "reference"` (the release pinned in
+# .config/reference-binary.toml). Every other target uses `binary = "candidate"`
+# (the binary built from the branch under test) and is compared per node id
+# against the reference. All targets run the same harness, test list and server
+# configuration; `env` adds only the target's own RUSTFS_* settings.
+#
+# Adding a target is one more [[targets]] table, for example:
+#
+#   [[targets]]
+#   name = "candidate-gateway"
+#   binary = "candidate"
+#   env = { RUSTFS_S3_STACK = "gateway" }
+
+schema = 1
+
+[[targets]]
+name = "reference"
+binary = "reference"
+
+[[targets]]
+name = "candidate-legacy"
+binary = "candidate"

.config/reference-parity-tolerated.txt

@@ -0,0 +1,12 @@
+# Tolerated per-test differences of the s3-tests reference parity lane.
+#
+# One entry per line, four fields separated by " | ":
+#
+#   <target> | <pytest node id> | <tracking issue URL> | <reason>
+#
+# <target> is a candidate name from .config/reference-parity-targets.toml, the
+# node id is exactly as the lane prints it (s3tests/functional/test_s3.py::...),
+# and the issue URL is https://github.com/rustfs/<repo>/issues/<number>.
+#
+# Same semantics as scripts/junit_diff.py --allow: an entry whose difference no
+# longer occurs is stale and fails the lane, so remove it when it is fixed.

.config/test-count-baseline.txt

@@ -25,6 +25,7 @@ rustfs-data-usage	58	0
 rustfs-ecstore	7858	2
 rustfs-extension-schema	23	0
 rustfs-filemeta	361	0
+rustfs-gateway-adapter	74	0
 rustfs-heal	1128	3
 rustfs-heal-contracts	8	0
 rustfs-iam	415	0

.github/actions/quick-checks/action.yml

@@ -101,6 +101,10 @@ runs:
       shell: bash
       run: ./scripts/check_test_count_ratchet.sh
 
+    - name: Check cross-owner tests use test seams
+      shell: bash
+      run: python3 ./scripts/check_test_seams.py
+
     - name: Check cryptographic capability wording
       shell: bash
       run: ./scripts/check_fips_wording.sh

.github/actions/reference-binary/action.yml

@@ -0,0 +1,96 @@
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "Reference RustFS binary"
+description: "Download the pinned reference RustFS release, verify its sha256 and expose the executable path. A missing pin, missing asset or digest mismatch fails the step."
+
+inputs:
+  pin-file:
+    description: "Pin file naming the reference release (repository-relative)"
+    required: false
+    default: ".config/reference-binary.toml"
+  platform:
+    description: "Platform table in the pin file"
+    required: false
+    default: "linux-x86_64-gnu"
+  destination:
+    description: "Directory that receives the executable; defaults to $RUNNER_TEMP/reference-binary"
+    required: false
+    default: ""
+
+outputs:
+  path:
+    description: "Absolute path of the verified executable"
+    value: ${{ steps.unpack.outputs.path }}
+  version:
+    description: "Pinned reference version"
+    value: ${{ steps.pin.outputs.version }}
+  sha256:
+    description: "Pinned and verified archive sha256"
+    value: ${{ steps.pin.outputs.sha256 }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Resolve reference pin
+      id: pin
+      shell: bash
+      env:
+        PIN_TOOL: ${{ github.action_path }}/reference_binary.py
+        PIN_FILE: ${{ inputs.pin-file }}
+        PIN_PLATFORM: ${{ inputs.platform }}
+      run: |
+        set -euo pipefail
+        resolved="$(python3 "$PIN_TOOL" resolve --pin "$PIN_FILE" --platform "$PIN_PLATFORM")"
+        printf '%s\n' "$resolved"
+        printf '%s\n' "$resolved" >> "$GITHUB_OUTPUT"
+
+    - name: Download and verify reference archive
+      id: download
+      shell: bash
+      env:
+        GH_TOKEN: ${{ github.token }}
+        ASSET_DOWNLOADER: ${{ github.action_path }}/../setup/download_release_asset.py
+        REF_REPOSITORY: ${{ steps.pin.outputs.repository }}
+        REF_TAG: ${{ steps.pin.outputs.tag }}
+        REF_ASSET: ${{ steps.pin.outputs.asset }}
+        REF_SHA256: ${{ steps.pin.outputs.sha256 }}
+      run: |
+        set -euo pipefail
+        archive="$RUNNER_TEMP/reference-archive/$REF_ASSET"
+        # Fails unless the release lists exactly one asset of this name whose
+        # published digest equals the pin and whose downloaded bytes match it.
+        python3 "$ASSET_DOWNLOADER" --repository "$REF_REPOSITORY" --tag "$REF_TAG" \
+          --asset "$REF_ASSET" --output "$archive" --sha256 "$REF_SHA256"
+        echo "archive=$archive" >> "$GITHUB_OUTPUT"
+
+    - name: Unpack reference executable
+      id: unpack
+      shell: bash
+      env:
+        PIN_TOOL: ${{ github.action_path }}/reference_binary.py
+        REF_ARCHIVE: ${{ steps.download.outputs.archive }}
+        REF_SHA256: ${{ steps.pin.outputs.sha256 }}
+        REF_EXECUTABLE: ${{ steps.pin.outputs.executable }}
+        REF_DESTINATION: ${{ inputs.destination }}
+      run: |
+        set -euo pipefail
+        destination="${REF_DESTINATION:-$RUNNER_TEMP/reference-binary}"
+        unpacked="$(python3 "$PIN_TOOL" unpack --archive "$REF_ARCHIVE" --sha256 "$REF_SHA256" \
+          --executable "$REF_EXECUTABLE" --destination "$destination")"
+        path="${unpacked#path=}"
+        test -x "$path"
+        echo "Reference executable: $path"
+        "$path" --version
+        echo "path=$path" >> "$GITHUB_OUTPUT"

.github/actions/reference-binary/reference_binary.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Resolve and unpack the pinned reference RustFS release binary.
+
+`resolve` reads the pin file and prints the selected platform's identity as
+`key=value` lines (GitHub step-output format). `unpack` re-verifies the
+downloaded archive digest against the pin and extracts exactly one executable.
+Every missing or malformed input is a failure; nothing is skipped.
+"""
+
+import argparse
+import hashlib
+from pathlib import Path
+import re
+import sys
+import tomllib
+import zipfile
+
+REPOSITORY = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*")
+TAG = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+-]*")
+PLATFORM = re.compile(r"[a-z0-9][a-z0-9_-]*")
+ASSET = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+-]*\.zip")
+SHA256 = re.compile(r"[0-9a-f]{64}")
+EXECUTABLE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*")
+MAX_EXECUTABLE_BYTES = 1024 * 1024 * 1024
+
+
+class PinError(Exception):
+    """A pin, archive or argument that cannot be trusted."""
+
+
+def field(table, key, pattern, where):
+    value = table.get(key)
+    if not isinstance(value, str) or not pattern.fullmatch(value):
+        raise PinError(f"{where}: missing or invalid '{key}'")
+    return value
+
+
+def resolve(pin_path, platform):
+    if not PLATFORM.fullmatch(platform or ""):
+        raise PinError("invalid platform name")
+    try:
+        data = tomllib.loads(Path(pin_path).read_text(encoding="utf-8"))
+    except FileNotFoundError as exc:
+        raise PinError(f"{pin_path}: reference pin file is missing") from exc
+    except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError) as exc:
+        raise PinError(f"{pin_path}: cannot read reference pin: {exc}") from exc
+    if data.get("schema") != 1 or type(data.get("schema")) is not int:
+        raise PinError(f"{pin_path}: unsupported pin schema")
+    identity = {
+        "repository": field(data, "repository", REPOSITORY, pin_path),
+        "version": field(data, "version", TAG, pin_path),
+        "tag": field(data, "tag", TAG, pin_path),
+    }
+    platforms = data.get("platforms")
+    if not isinstance(platforms, dict) or not platforms:
+        raise PinError(f"{pin_path}: no pinned platforms")
+    entry = platforms.get(platform)
+    if not isinstance(entry, dict):
+        raise PinError(f"{pin_path}: no pin for platform '{platform}'")
+    where = f"{pin_path} [platforms.{platform}]"
+    identity["asset"] = field(entry, "asset", ASSET, where)
+    identity["sha256"] = field(entry, "sha256", SHA256, where)
+    identity["executable"] = field(entry, "executable", EXECUTABLE, where)
+    if identity["version"] not in identity["asset"]:
+        raise PinError(f"{where}: asset name does not carry version {identity['version']}")
+    unknown = set(entry) - {"asset", "sha256", "executable"}
+    if unknown:
+        raise PinError(f"{where}: unknown keys {sorted(unknown)}")
+    return identity
+
+
+def file_sha256(path):
+    with open(path, "rb") as stream:
+        return hashlib.file_digest(stream, "sha256").hexdigest()
+
+
+def unpack(archive, sha256, executable, destination):
+    if not SHA256.fullmatch(sha256 or "") or not EXECUTABLE.fullmatch(executable or ""):
+        raise PinError("invalid expected digest or executable name")
+    try:
+        actual = file_sha256(archive)
+    except OSError as exc:
+        raise PinError(f"{archive}: reference archive is missing or unreadable") from exc
+    if actual != sha256:
+        raise PinError(f"{archive}: sha256 {actual} does not match the pinned digest")
+    try:
+        with zipfile.ZipFile(archive) as bundle:
+            matches = [info for info in bundle.infolist() if info.filename == executable]
+            if len(matches) != 1:
+                raise PinError(f"{archive}: must contain exactly one top-level '{executable}'")
+            info = matches[0]
+            if info.is_dir() or not 0 < info.file_size <= MAX_EXECUTABLE_BYTES:
+                raise PinError(f"{archive}: '{executable}' is not a plausible executable")
+            destination = Path(destination)
+            destination.mkdir(parents=True, exist_ok=True)
+            target = destination / executable
+            with bundle.open(info) as source, open(target, "wb") as sink:
+                while chunk := source.read(1024 * 1024):
+                    sink.write(chunk)
+    except zipfile.BadZipFile as exc:
+        raise PinError(f"{archive}: not a valid zip archive") from exc
+    target.chmod(0o755)
+    return target
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    sub = parser.add_subparsers(dest="mode", required=True)
+    res = sub.add_parser("resolve")
+    res.add_argument("--pin", required=True)
+    res.add_argument("--platform", required=True)
+    unp = sub.add_parser("unpack")
+    unp.add_argument("--archive", required=True)
+    unp.add_argument("--sha256", required=True)
+    unp.add_argument("--executable", required=True)
+    unp.add_argument("--destination", required=True)
+    args = parser.parse_args(argv)
+    try:
+        if args.mode == "resolve":
+            for key, value in resolve(args.pin, args.platform).items():
+                print(f"{key}={value}")
+        else:
+            print(f"path={unpack(args.archive, args.sha256, args.executable, args.destination)}")
+    except PinError as exc:
+        print(f"reference binary: {exc}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scheduled-validations.json

@@ -869,6 +869,33 @@
       "failure_owner": "watchdog",
       "origin": "baseline"
     },
+    {
+      "workflow": ".github/workflows/reference-parity.yml",
+      "name": "Reference Parity",
+      "roles": [
+        "pr",
+        "periodic",
+        "manual"
+      ],
+      "events": [
+        "pull_request",
+        "schedule",
+        "workflow_dispatch"
+      ],
+      "cadence": "nightly",
+      "timezone": "UTC",
+      "schedule": [
+        "37 1 * * *"
+      ],
+      "owner": "@overtrue",
+      "policy": "active",
+      "monitor": "workflow",
+      "max_age_hours": 36,
+      "result_owner": "reference-parity.yml",
+      "disposition": "s3-tests per-node-id parity of branch candidates against the pinned reference release",
+      "failure_owner": "watchdog",
+      "origin": "added"
+    },
     {
       "workflow": ".github/workflows/functional-chain-health.yml",
       "name": "Functional chain health",
@@ -994,6 +1021,26 @@
       "policy_reason": "Preserve operator-disabled state; enabling requires a separate policy decision.",
       "origin": "baseline"
     },
+    {
+      "workflow": ".github/workflows/manual-main-packages.yml",
+      "name": "Manual main packages",
+      "roles": [
+        "manual"
+      ],
+      "events": [
+        "workflow_dispatch"
+      ],
+      "cadence": "event",
+      "timezone": "UTC",
+      "schedule": [],
+      "owner": "@overtrue",
+      "policy": "active",
+      "monitor": "none",
+      "result_owner": "manual-main-packages.yml",
+      "disposition": "Packages the origin/main tip as workflow artifacts only. No GitHub Release or download-channel writes.",
+      "monitor_reason": "Operator-requested packaging has no periodic freshness obligation.",
+      "origin": "manual-main-packages"
+    },
     {
       "workflow": ".github/workflows/minio-interop.yml",
       "name": "minio-interop",

.github/workflows/manual-main-packages.yml

@@ -0,0 +1,415 @@
+# Copyright 2026 RustFS Team
+# SPDX-License-Identifier: Apache-2.0
+
+# Manual packaging of the current origin/main tip. Workflow artifacts only.
+# Dispatch uses this file from the selected branch. The build fetches
+# refs/heads/main and checks out that commit, not the dispatch ref or a tag.
+
+name: Manual main packages
+
+on:
+  workflow_dispatch:
+    inputs:
+      architecture:
+        description: "Linux targets: x86 (x86_64 musl+gnu), arm (aarch64 musl+gnu), or all"
+        required: true
+        type: choice
+        options:
+          - x86
+          - arm
+          - all
+        default: all
+      version:
+        description: "Binary and package version: MAJOR.MINOR.PATCH, or MAJOR.MINOR.PATCH-(alpha|beta|rc).N. No leading v."
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+  CARGO_INCREMENTAL: 0
+
+jobs:
+  prepare:
+    name: Resolve main tip and version
+    if: github.repository == 'rustfs/rustfs'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    outputs:
+      source_sha: ${{ steps.pin.outputs.source_sha }}
+      version: ${{ steps.version.outputs.version }}
+      toolchain: ${{ steps.inputs.outputs.toolchain }}
+      console: ${{ steps.inputs.outputs.console }}
+      matrix: ${{ steps.matrix.outputs.matrix }}
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          ref: main
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Pin the tip of origin/main
+        id: pin
+        shell: bash
+        run: |
+          set -euo pipefail
+          GIT_TERMINAL_PROMPT=0 git fetch --no-tags origin refs/heads/main
+          sha=$(git rev-parse FETCH_HEAD)
+          git checkout --detach "$sha"
+          test "$(git rev-parse HEAD)" = "$sha"
+          printf 'source_sha=%s\n' "$sha" >> "$GITHUB_OUTPUT"
+          printf 'Pinned origin/main %s\n' "$sha"
+
+      - name: Validate package version
+        id: version
+        shell: bash
+        env:
+          PACKAGE_VERSION: ${{ inputs.version }}
+        run: |
+          set -euo pipefail
+          bash scripts/release/package_manual_artifacts.sh validate \
+            --version "$PACKAGE_VERSION" \
+            --arch x86_64 \
+            --libc gnu | tee "$RUNNER_TEMP/package-version.txt"
+          printf 'version=%s\n' "$PACKAGE_VERSION" >> "$GITHUB_OUTPUT"
+
+      - name: Install stable Rust
+        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+        with:
+          toolchain: stable
+
+      - name: Resolve console assets and toolchain
+        id: inputs
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          toolchain=$(rustc --version | awk '{print $2}')
+          [[ "$toolchain" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
+          gh api repos/rustfs/console/releases/latest > console-release.json
+          TOOLCHAIN="$toolchain" python3 - <<'PY'
+          import json, os, sys
+          sys.path.insert(0, "scripts/release")
+          import candidate
+          console = candidate.console_asset(candidate.read_json("console-release.json"))
+          candidate.emit_output("console", json.dumps(console, separators=(",", ":")))
+          candidate.emit_output("toolchain", os.environ["TOOLCHAIN"])
+          print(f"toolchain={os.environ['TOOLCHAIN']}")
+          print(f"console={console['tag']} {console['name']}")
+          PY
+
+      - name: Select Linux targets
+        id: matrix
+        shell: bash
+        env:
+          ARCHITECTURE: ${{ inputs.architecture }}
+        run: |
+          set -euo pipefail
+          matrix=$(bash scripts/release/package_manual_artifacts.sh matrix "$ARCHITECTURE")
+          count=$(python3 -c 'import json,sys; print(len(json.loads(sys.argv[1])["include"]))' "$matrix")
+          [[ "$count" -gt 0 ]]
+          printf 'matrix=%s\n' "$matrix" >> "$GITHUB_OUTPUT"
+          printf 'Selected %s target(s) for %s\n' "$count" "$ARCHITECTURE"
+
+  package:
+    name: Package RustFS (${{ matrix.target_id }})
+    needs: prepare
+    if: github.repository == 'rustfs/rustfs'
+    # x86_64 uses sm-standard-4, the official release compile pool: the workspace
+    # release build needs that disk (placement rule: disk) and reads build.yml's
+    # cache without writing it. aarch64 uses ubicloud-standard-16-arm so the
+    # packaged binary and console smoke execute natively (placement rule: hardware).
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 210
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      RUSTUP_TOOLCHAIN: ${{ needs.prepare.outputs.toolchain }}
+      RUSTFLAGS: "${{ matrix.rustflags }}"
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.prepare.outputs.matrix) }}
+    steps:
+      - name: Checkout the pinned origin/main commit
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          ref: ${{ needs.prepare.outputs.source_sha }}
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Confirm the checkout is that main commit
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
+        run: |
+          set -euo pipefail
+          test "$(git rev-parse HEAD)" = "$SOURCE_SHA"
+          GIT_TERMINAL_PROMPT=0 git fetch --no-tags origin refs/heads/main
+          git merge-base --is-ancestor "$SOURCE_SHA" FETCH_HEAD
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: ${{ needs.prepare.outputs.toolchain }}
+          target: ${{ matrix.target }}
+          cache-shared-key: build-${{ matrix.target }}${{ matrix.glibc && format('-glibc-{0}', matrix.glibc) || '' }}${{ matrix.os == 'ubicloud-standard-16-arm' && '-ubicloud-arm' || '' }}
+          # build.yml owns this key. This job only restores it.
+          cache-save-if: 'false'
+          install-cross-tools: ${{ matrix.cross }}
+          install-build-packaging-tools: 'true'
+          install-test-tools: 'false'
+
+      - name: Download static console assets
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          CONSOLE_IDENTITY: ${{ needs.prepare.outputs.console }}
+        run: |
+          set -euo pipefail
+          python3 - <<'PYCONSOLE'
+          import json, os, pathlib, subprocess, zipfile
+          value = json.loads(os.environ["CONSOLE_IDENTITY"])
+          subprocess.run(["python3", ".github/actions/setup/download_release_asset.py",
+              "--repository", "rustfs/console", "--tag", value["tag"], "--asset", value["name"],
+              "--sha256", value["sha256"], "--output", "console.zip"], check=True)
+          destination = pathlib.Path("rustfs/static")
+          destination.mkdir(parents=True, exist_ok=True)
+          with zipfile.ZipFile("console.zip") as archive:
+              for member in archive.infolist():
+                  path = pathlib.PurePosixPath(member.filename)
+                  if path.is_absolute() or ".." in path.parts or "\\" in member.filename:
+                      raise SystemExit("unsafe Console archive member")
+              archive.extractall(destination)
+          if not (destination / "index.html").is_file():
+              raise SystemExit("Console archive has no index.html")
+          PYCONSOLE
+          rm -f console.zip
+
+      - name: Build RustFS
+        shell: bash
+        env:
+          RUSTFS_BUILD_VERSION: ${{ needs.prepare.outputs.version }}
+          BUILD_TARGET: ${{ matrix.target }}
+          BUILD_OS: ${{ matrix.build_os }}
+          BUILD_FEATURES: ${{ matrix.features }}
+          BUILD_GLIBC: ${{ matrix.glibc }}
+          BUILD_CROSS: ${{ matrix.cross }}
+        run: |
+          set -euo pipefail
+          touch rustfs/build.rs
+
+          feature_args=()
+          if [[ -n "$BUILD_FEATURES" ]]; then
+            feature_args=(--features "$BUILD_FEATURES")
+          fi
+
+          # gnu targets pin glibc 2.31 through cargo-zigbuild (<target>.<glibc>).
+          target_arg="$BUILD_TARGET"
+          if [[ -n "$BUILD_GLIBC" ]]; then
+            target_arg="${target_arg}.${BUILD_GLIBC}"
+          fi
+
+          profile_args=()
+          if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
+            profile_args=(--config 'profile.release.strip="none"')
+          fi
+
+          case "$BUILD_CROSS" in
+            true)
+              cargo zigbuild --release --locked --target "$target_arg" "${feature_args[@]}" "${profile_args[@]}" -p rustfs --bin rustfs
+              ;;
+            false)
+              cargo build --release --locked --target "$target_arg" "${feature_args[@]}" "${profile_args[@]}" -p rustfs --bin rustfs
+              ;;
+            *)
+              printf 'unexpected cross flag: %q\n' "$BUILD_CROSS" >&2
+              exit 1
+              ;;
+          esac
+
+          bin="target/${BUILD_TARGET}/release/rustfs"
+          version_output=$("$bin" --version)
+          first=${version_output%%$'\n'*}
+          if [[ "$first" != "rustfs ${RUSTFS_BUILD_VERSION}" ]]; then
+            printf 'binary version line is %q, expected %q\n' "$first" "rustfs ${RUSTFS_BUILD_VERSION}" >&2
+            printf '%s\n' "$version_output" >&2
+            exit 1
+          fi
+          if ! grep -Fxq "build target : ${BUILD_TARGET}" <<< "$version_output" || \
+             ! grep -Fxq "build os     : ${BUILD_OS}" <<< "$version_output" || \
+             ! grep -Fxq "rust channel : stable-${BUILD_TARGET}" <<< "$version_output"; then
+            printf 'release binary reports an unexpected compile target\n' >&2
+            printf '%s\n' "$version_output" >&2
+            exit 1
+          fi
+          if [[ -n "$BUILD_GLIBC" ]]; then
+            python3 scripts/release/check_glibc.py "$BUILD_GLIBC" "$bin"
+          fi
+
+      - name: Generate the CPU symbol catalog
+        if: endsWith(matrix.target, '-linux-gnu') && matrix.features == 'pyroscope'
+        shell: bash
+        env:
+          BUILD_TARGET: ${{ matrix.target }}
+        run: |
+          set -euo pipefail
+          cargo metadata --locked --format-version 1 --filter-platform "$BUILD_TARGET" \
+            --features rustfs/pyroscope > "$RUNNER_TEMP/cpu-catalog-metadata.json"
+          features=$(jq -ce '
+            (.packages[] | select(.name == "rustfs") | .id) as $id |
+            .resolve.nodes[] | select(.id == $id) | .features | map(select(. != "default")) | sort
+          ' "$RUNNER_TEMP/cpu-catalog-metadata.json")
+          release_dir="target/${BUILD_TARGET}/release"
+          rm -f "$release_dir/rustfs.cpu-symbol-catalog.json" "$release_dir/rustfs.cpu-symbol-catalog.sha256"
+          cargo run --locked -p rustfs-connect-symbol-catalog -- \
+            "$release_dir/rustfs" "$(git rev-parse HEAD)" "$BUILD_TARGET" "$features" \
+            "$release_dir/rustfs.cpu-symbol-catalog.json" > "$release_dir/rustfs.cpu-symbol-catalog.sha256"
+
+      - name: Install DEB and RPM tools
+        shell: bash
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y ruby ruby-dev build-essential rpm fakeroot cpio
+          sudo gem install fpm --version 1.17.0 --no-document
+
+      - name: Package zip, deb, and rpm
+        shell: bash
+        env:
+          PACKAGE_VERSION: ${{ needs.prepare.outputs.version }}
+          ARCH: ${{ matrix.arch }}
+          LIBC: ${{ matrix.libc }}
+          BUILD_TARGET: ${{ matrix.target }}
+          BUILD_FEATURES: ${{ matrix.features }}
+          SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
+        run: |
+          set -euo pipefail
+          binary="target/${BUILD_TARGET}/release/rustfs"
+          extra=()
+          if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
+            extra=(
+              --zip-extra "target/${BUILD_TARGET}/release/rustfs.cpu-symbol-catalog.json"
+              --zip-extra "target/${BUILD_TARGET}/release/rustfs.cpu-symbol-catalog.sha256"
+            )
+          fi
+          bash scripts/release/package_manual_artifacts.sh package \
+            --version "$PACKAGE_VERSION" \
+            --arch "$ARCH" \
+            --libc "$LIBC" \
+            --binary "$binary" \
+            --output dist \
+            "${extra[@]}"
+          printf 'Packaged %s from origin/main %s\n' "$PACKAGE_VERSION" "$SOURCE_SHA"
+
+      - name: Verify the packaged binary
+        shell: bash
+        env:
+          PACKAGE_VERSION: ${{ needs.prepare.outputs.version }}
+          ARCH: ${{ matrix.arch }}
+          LIBC: ${{ matrix.libc }}
+          BUILD_TARGET: ${{ matrix.target }}
+          BUILD_OS: ${{ matrix.build_os }}
+          BUILD_FEATURES: ${{ matrix.features }}
+        run: |
+          set -euo pipefail
+          stem="rustfs-linux-${ARCH}-${LIBC}-v${PACKAGE_VERSION}"
+          package_file="dist/${stem}.zip"
+          [[ -f "$package_file" && -f "dist/${stem}.deb" && -f "dist/${stem}.rpm" ]]
+          [[ "$(find dist -maxdepth 1 -type f | wc -l)" -eq 3 ]]
+          archive_members=$(unzip -Z1 "$package_file" | sort)
+          if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
+            expected_members=$(printf '%s\n' rustfs rustfs.cpu-symbol-catalog.json rustfs.cpu-symbol-catalog.sha256 | sort)
+          else
+            expected_members=$(printf '%s\n' rustfs)
+          fi
+          if [[ "$archive_members" != "$expected_members" ]]; then
+            printf 'Linux package does not contain the expected files\n' >&2
+            printf '%s\n' "$archive_members" >&2
+            exit 1
+          fi
+          binary_dir=$(mktemp -d)
+          trap 'rm -rf "$binary_dir"' EXIT
+          unzip -q "$package_file" rustfs -d "$binary_dir"
+          version_output=$("$binary_dir/rustfs" --version)
+          first=${version_output%%$'\n'*}
+          if [[ "$first" != "rustfs ${PACKAGE_VERSION}" ]]; then
+            printf 'packaged binary version line is %q, expected %q\n' "$first" "rustfs ${PACKAGE_VERSION}" >&2
+            exit 1
+          fi
+          if ! grep -Fxq "build target : ${BUILD_TARGET}" <<< "$version_output" || \
+             ! grep -Fxq "build os     : ${BUILD_OS}" <<< "$version_output"; then
+            printf 'packaged binary reports an unexpected compile target\n' >&2
+            exit 1
+          fi
+
+      - name: Verify packaged console
+        if: matrix.target == 'x86_64-unknown-linux-gnu' || matrix.target == 'aarch64-unknown-linux-gnu'
+        shell: bash
+        env:
+          PACKAGE_VERSION: ${{ needs.prepare.outputs.version }}
+          ARCH: ${{ matrix.arch }}
+          LIBC: ${{ matrix.libc }}
+        run: |
+          set -euo pipefail
+          package_file="dist/rustfs-linux-${ARCH}-${LIBC}-v${PACKAGE_VERSION}.zip"
+
+          find_free_port() {
+            python3 - <<'PY'
+          import socket
+
+          with socket.socket() as sock:
+              sock.bind(("127.0.0.1", 0))
+              print(sock.getsockname()[1])
+          PY
+          }
+
+          package_dir=$(mktemp -d)
+          data_dir=$(mktemp -d)
+          log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
+          server_pid=""
+          trap '
+            if [[ -n "${server_pid:-}" ]]; then
+              kill "$server_pid" 2>/dev/null || true
+              wait "$server_pid" 2>/dev/null || true
+            fi
+            rm -rf "$package_dir" "$data_dir"
+          ' EXIT
+
+          unzip -q "$package_file" -d "$package_dir"
+          api_port=$(find_free_port)
+          console_port=$(find_free_port)
+          console_url="http://127.0.0.1:${console_port}/rustfs/console/"
+
+          "$package_dir/rustfs" server \
+            --address "127.0.0.1:${api_port}" \
+            --console-enable \
+            --console-address "127.0.0.1:${console_port}" \
+            --access-key console-smoke \
+            --secret-key console-smoke-secret \
+            "$data_dir" >"$log_file" 2>&1 &
+          server_pid=$!
+
+          for _ in {1..40}; do
+            response=$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)
+            if [[ "$response" == 200\ text/html* ]]; then
+              exit 0
+            fi
+            sleep 0.25
+          done
+
+          printf 'Console endpoint did not return 200 text/html: %s\n' "$response" >&2
+          cat "$log_file"
+          exit 1
+
+      - name: Upload packages
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: packages-${{ matrix.target_id }}-v${{ needs.prepare.outputs.version }}
+          path: dist/*
+          if-no-files-found: error
+          compression-level: 0
+          retention-days: 30

.github/workflows/mint.yml

@@ -112,9 +112,11 @@ jobs:
     # GitHub-hosted: reliably provides a running Docker daemon + buildx +
     # python3. See the header note (ci-2) for why the self-hosted sm-standard-4
     # label was abandoned (no docker.sock on the pod its only run landed on).
+    # sm-standard-2 has no docker.sock either (weekly runs failed at "Enable
+    # buildx" from 2026-09-27 to 2026-11-08), so keep this job on ubuntu-latest.
     # TODO(ci-8): scheduled-failure alerting (auto-open issue on a red weekly
     # run) is added by the ci-8 composite action; do not implement it here.
-    runs-on: sm-standard-2
+    runs-on: ubuntu-latest
     timeout-minutes: 120
     steps:
       - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7

.github/workflows/reference-parity.yml

@@ -0,0 +1,308 @@
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# s3-tests reference parity lane (rustfs/backlog#2773).
+#
+# Runs the ceph/s3-tests implemented list (scripts/s3-tests/implemented_tests.txt)
+# against every target in .config/reference-parity-targets.toml and fails on
+# any per-node-id difference between a candidate and the reference:
+#
+#   - reference: the release pinned in .config/reference-binary.toml, downloaded
+#     and sha256-verified by .github/actions/reference-binary;
+#   - candidate targets: the debug binary built from this checkout exactly as
+#     ci.yml's build-rustfs-debug-binary job builds it for S3 Implemented Tests.
+#
+# Every target runs the same harness (scripts/s3-tests/run.sh from this
+# checkout), test list and server configuration on its own hosted runner. A
+# target's failures get one rerun against a fresh server; a case that passes it
+# is flaky, not a difference. scripts/s3-tests/reference_parity.py judges the
+# result and honours .config/reference-parity-tolerated.txt, whose entries must
+# each match a real difference.
+#
+# Report-only: not part of the Test and Lint aggregate. Scheduled failures are
+# owned by the scheduled-validation watchdog (.github/scheduled-validations.json).
+
+name: Reference Parity
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/reference-parity.yml"
+      - ".github/actions/reference-binary/**"
+      - ".config/reference-binary.toml"
+      - ".config/reference-parity-targets.toml"
+      - ".config/reference-parity-tolerated.txt"
+      - "scripts/s3-tests/reference_parity.py"
+      - "scripts/junit_diff.py"
+  schedule:
+    - cron: "37 1 * * *"
+  workflow_dispatch:
+
+# Matches ci.yml: these variables are hashed into the rust-cache key, so the
+# candidate build reads the same ci-dev cache as build-rustfs-debug-binary.
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  plan:
+    name: Plan parity targets
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      targets: ${{ steps.targets.outputs.targets }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Validate target list
+        id: targets
+        run: |
+          set -euo pipefail
+          targets="$(python3 scripts/s3-tests/reference_parity.py targets --config .config/reference-parity-targets.toml)"
+          echo "targets=${targets}" >> "$GITHUB_OUTPUT"
+          echo "Parity targets: ${targets}"
+
+  build-candidate:
+    name: Build candidate binary
+    # Critical-path CPU and cache ownership: the same debug build as ci.yml's
+    # build-rustfs-debug-binary, reading the ci-dev cache written on sm-standard-4.
+    runs-on: sm-standard-4
+    timeout-minutes: 60
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-dev
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
+
+      - name: Build debug binary
+        run: python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
+
+      - name: Upload candidate binary
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: reference-parity-candidate-binary
+          overwrite: true
+          path: |
+            target/debug/rustfs
+            target/debug/rustfs.e2e.json
+          if-no-files-found: error
+          retention-days: 1
+
+  s3tests:
+    name: s3-tests (${{ matrix.target.name }})
+    needs: [ plan, build-candidate ]
+    runs-on: ubuntu-latest
+    timeout-minutes: 75
+    strategy:
+      fail-fast: false
+      matrix:
+        target: ${{ fromJSON(needs.plan.outputs.targets) }}
+    env:
+      TARGET_NAME: ${{ matrix.target.name }}
+      TARGET_KIND: ${{ matrix.target.binary }}
+      # Above this many first-run failures the target is systemically broken;
+      # a rerun would only repeat it, so the first run is judged as is.
+      RERUN_CAP: "50"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Fetch pinned reference binary
+        id: reference
+        if: matrix.target.binary == 'reference'
+        uses: ./.github/actions/reference-binary
+
+      - name: Download candidate binary
+        if: matrix.target.binary == 'candidate'
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: reference-parity-candidate-binary
+          path: target/debug
+
+      - name: Select target binary
+        env:
+          REFERENCE_PATH: ${{ steps.reference.outputs.path }}
+        run: |
+          set -euo pipefail
+          case "$TARGET_KIND" in
+            reference) binary="$REFERENCE_PATH" ;;
+            candidate) chmod +x target/debug/rustfs; binary="$PWD/target/debug/rustfs" ;;
+            *) echo "unknown target binary kind: $TARGET_KIND" >&2; exit 1 ;;
+          esac
+          test -x "$binary"
+          echo "TARGET_BINARY=$binary" >> "$GITHUB_ENV"
+          mkdir -p "parity/$TARGET_NAME"
+          {
+            echo "target=$TARGET_NAME"
+            echo "binary=$TARGET_KIND"
+            echo "sha256=$(sha256sum "$binary" | cut -d' ' -f1)"
+            echo "version=$("$binary" --version | tr '\n' ' ')"
+          } | tee "parity/$TARGET_NAME/target.txt"
+
+      - name: Apply target environment
+        env:
+          TARGET_JSON: ${{ toJSON(matrix.target) }}
+        run: |
+          set -euo pipefail
+          python3 scripts/s3-tests/reference_parity.py export-env --target-json "$TARGET_JSON" \
+            | tee -a "$GITHUB_ENV" "parity/$TARGET_NAME/target-env.txt"
+
+      - name: Run implemented s3-tests
+        run: |
+          set -uo pipefail
+          report="parity/$TARGET_NAME"
+          harness="artifacts/s3tests-single"
+
+          # Same invocation as ci.yml's S3 Implemented Tests, with a fresh data
+          # root and port per run. $1 optionally replaces the test list.
+          run_suite() {
+            local list="$1" label="$2" root port
+            root="${RUNNER_TEMP}/rustfs-s3tests-${label}"
+            rm -rf "$harness" "$root"
+            mkdir -p "$root"
+            port="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
+            env DEPLOY_MODE=binary \
+              RUSTFS_BINARY="$TARGET_BINARY" \
+              TEST_MODE=single \
+              MAXFAIL=0 \
+              XDIST=4 \
+              S3_PORT="$port" \
+              DATA_ROOT="$root" \
+              S3TESTS_CONF="$harness/s3tests.conf" \
+              ${list:+IMPLEMENTED_TESTS_FILE="$list"} \
+              ./scripts/s3-tests/run.sh
+          }
+
+          # A run is usable only if it produced a JUnit report; test failures
+          # are data for the comparison, a missing report is a lane failure.
+          keep() {
+            local prefix="$1"
+            for file in pytest.log compat-report.md; do
+              if [[ -f "$harness/$file" ]]; then cp "$harness/$file" "$report/${prefix}${file}"; fi
+            done
+            if [[ -f "$harness/rustfs-single/rustfs.log" ]]; then cp "$harness/rustfs-single/rustfs.log" "$report/${prefix}rustfs.log"; fi
+            test -s "$harness/junit.xml" || { echo "::error::$TARGET_NAME produced no JUnit report (${prefix:-first} run)"; return 1; }
+            cp "$harness/junit.xml" "$report/${prefix}junit.xml"
+          }
+
+          run_suite "" first
+          echo "first_run_exit=$?" >> "$report/target.txt"
+          keep "" || exit 1
+          cp "$harness/selected-nodeids.txt" "$report/selected-nodeids.txt" || exit 1
+
+          failures="$(python3 scripts/s3-tests/reference_parity.py failed --report "$report")" || exit 1
+          count="$(printf '%s' "$failures" | grep -c . || true)"
+          echo "first_run_failures=$count" >> "$report/target.txt"
+          if (( count == 0 )); then
+            exit 0
+          fi
+          if (( count > RERUN_CAP )); then
+            echo "::warning::$TARGET_NAME has $count failures (cap $RERUN_CAP); judging the first run without a rerun"
+            echo "rerun=skipped-over-cap" >> "$report/target.txt"
+            exit 0
+          fi
+          printf '%s\n' "$failures" > "$RUNNER_TEMP/rerun-tests.txt"
+          run_suite "$RUNNER_TEMP/rerun-tests.txt" rerun
+          echo "rerun_exit=$?" >> "$report/target.txt"
+          keep rerun- || exit 1
+
+      - name: Upload target report
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: reference-parity-report-${{ matrix.target.name }}
+          overwrite: true
+          path: parity/${{ matrix.target.name }}/
+          if-no-files-found: error
+          retention-days: 14
+
+  compare:
+    name: Compare targets
+    needs: [ plan, s3tests ]
+    if: always() && needs.plan.result == 'success'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download target reports
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: reference-parity-report-*
+          path: parity-download
+
+      - name: Compare per node id
+        env:
+          TARGETS: ${{ needs.plan.outputs.targets }}
+          S3TESTS_RESULT: ${{ needs.s3tests.result }}
+        run: |
+          set -uo pipefail
+          mkdir -p parity
+          for dir in parity-download/reference-parity-report-*; do
+            if [[ -d "$dir" ]]; then mv "$dir" "parity/${dir#parity-download/reference-parity-report-}"; fi
+          done
+          python3 scripts/s3-tests/reference_parity.py compare \
+            --targets-json "$TARGETS" \
+            --reports parity \
+            --tolerated .config/reference-parity-tolerated.txt \
+            --markdown "$GITHUB_STEP_SUMMARY" \
+            --json parity/diff.json > parity/diff.md
+          status=$?
+          cat parity/diff.md
+          if [[ "$S3TESTS_RESULT" != "success" ]]; then
+            echo "::error::s3-tests target jobs finished with '$S3TESTS_RESULT'"
+            status=1
+          fi
+          exit "$status"
+
+      - name: Upload parity diff
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: reference-parity-diff
+          overwrite: true
+          path: |
+            parity/diff.md
+            parity/diff.json
+          if-no-files-found: warn
+          retention-days: 14

Cargo.lock

@@ -4698,6 +4698,12 @@ version = "0.3.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
 
+[[package]]
+name = "futures-timer"
+version = "3.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968"
+
 [[package]]
 name = "futures-util"
 version = "0.3.34"
@@ -7914,10 +7920,34 @@ version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6"
 dependencies = [
+ "phf_macros",
  "phf_shared 0.14.0",
  "serde",
 ]
 
+[[package]]
+name = "phf_generator"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100"
+dependencies = [
+ "fastrand",
+ "phf_shared 0.14.0",
+]
+
+[[package]]
+name = "phf_macros"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085"
+dependencies = [
+ "phf_generator",
+ "phf_shared 0.14.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
 [[package]]
 name = "phf_shared"
 version = "0.12.1"
@@ -9999,6 +10029,114 @@ dependencies = [
  "xxhash-rust",
 ]
 
+[[package]]
+name = "rustfs-gateway"
+version = "0.57.0"
+source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
+dependencies = [
+ "arc-swap",
+ "bytes",
+ "form_urlencoded",
+ "futures-timer",
+ "futures-util",
+ "http 1.5.0",
+ "http-body",
+ "http-body-util",
+ "httparse",
+ "hyper",
+ "pin-project-lite",
+ "rustfs-gateway-core",
+ "rustfs-gateway-http",
+ "rustfs-gateway-macros",
+ "rustfs-gateway-sig",
+ "rustfs-gateway-stream",
+ "rustfs-gateway-types",
+ "rustfs-gateway-xml",
+ "sha2 0.11.1",
+ "tokio",
+ "tower",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "rustfs-gateway-adapter"
+version = "1.0.5"
+dependencies = [
+ "http 1.5.0",
+ "rustfs-credentials",
+ "rustfs-gateway",
+ "rustfs-gateway-http",
+ "rustfs-iam",
+ "rustfs-policy",
+ "serde_json",
+ "time",
+ "tokio",
+]
+
+[[package]]
+name = "rustfs-gateway-core"
+version = "0.54.0"
+source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
+dependencies = [
+ "bytes",
+ "http 1.5.0",
+ "md-5 0.11.0",
+ "percent-encoding",
+ "rustfs-gateway-http",
+ "rustfs-gateway-sig",
+ "rustfs-gateway-stream",
+ "rustfs-gateway-types",
+ "rustfs-gateway-xml",
+ "serde",
+ "serde_json",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "rustfs-gateway-http"
+version = "0.1.4"
+source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
+dependencies = [
+ "hmac 0.13.0",
+ "http 1.5.0",
+ "httparse",
+ "memchr",
+ "phf 0.14.0",
+ "rustfs-gateway-stream",
+ "rustfs-gateway-types",
+ "sha2 0.11.1",
+ "smallvec",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "rustfs-gateway-macros"
+version = "0.1.4"
+source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.9",
+]
+
+[[package]]
+name = "rustfs-gateway-sig"
+version = "0.13.0"
+source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
+dependencies = [
+ "hmac 0.13.0",
+ "http 1.5.0",
+ "rustfs-gateway-http",
+ "sha1 0.11.0",
+ "sha2 0.11.1",
+ "smallvec",
+ "subtle",
+ "zeroize",
+]
+
 [[package]]
 name = "rustfs-gateway-stream"
 version = "0.2.0"
@@ -12603,10 +12741,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.4.3",
+ "getrandom 0.3.4",
  "once_cell",
  "rustix",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]

Cargo.toml

@@ -26,6 +26,7 @@ members = [
     "crates/ecstore", # Erasure coding storage implementation
     "crates/e2e_test", # End-to-end test suite
     "crates/filemeta", # File metadata management
+    "crates/gateway-adapter", # RustFS implementations of the gateway extension traits
     "crates/heal", # Erasure set and object healing
     "crates/heal-contracts", # Heal request/response channel contracts
     "crates/iam", # Identity and Access Management
@@ -93,6 +94,7 @@ redundant_clone = "warn"
 [workspace.dependencies]
 # RustFS Internal Crates
 rustfs = { path = "./rustfs", version = "1.0.5" }
+rustfs-gateway-adapter = { path = "crates/gateway-adapter", version = "1.0.5" }
 rustfs-heal = { path = "crates/heal", version = "1.0.5" }
 rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.5" }
 rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.5" }
@@ -149,6 +151,12 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
 # the s3s-facing edge of the rustfs binary.
 rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.32.0", default-features = false }
 rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.2.0", default-features = false }
+# The gateway framework itself, consumed for its extension traits by the ring-2
+# adapter. `default-features = false` keeps the `server` feature, and with it the
+# server crate, out of the build.
+rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.57.0", default-features = false }
+# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
+rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", default-features = false }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"

Dockerfile.source

@@ -24,9 +24,10 @@
 #
 # Supports cross-compilation for amd64 and arm64 via TARGETPLATFORM.
 
-ARG TARGETPLATFORM
-ARG BUILDPLATFORM
-ARG TARGETARCH
+# Do not declare TARGETPLATFORM, BUILDPLATFORM or TARGETARCH here: a global
+# declaration without a value shadows BuildKit's automatic values, so the
+# stage-level re-declarations below see empty strings. They are only
+# declared inside the stage that uses them.
 ARG RUSTFS_BUILD_FEATURES=""
 ARG RC_RELEASE=latest
 
@@ -169,7 +170,24 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
     ;; \
     esac
 
-RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
+# TARGETARCH is only provided by BuildKit. Fall back to TARGETPLATFORM (for
+# example set by docker-compose) and then to the build host, like the cargo
+# step above, so every path that builds this file passes a real architecture.
+RUN set -eu; \
+    rc_arch="${TARGETARCH:-}"; \
+    if [ -z "${rc_arch}" ]; then \
+    case "${TARGETPLATFORM:-}" in \
+    linux/amd64) rc_arch=amd64 ;; \
+    linux/arm64) rc_arch=arm64 ;; \
+    *) \
+    case "$(uname -m)" in \
+    x86_64) rc_arch=amd64 ;; \
+    aarch64|arm64) rc_arch=arm64 ;; \
+    *) echo "TARGETARCH is empty and TARGETPLATFORM='${TARGETPLATFORM:-}' is not linux/amd64 or linux/arm64; cannot choose an rc architecture on $(uname -m)" >&2; exit 1 ;; \
+    esac ;; \
+    esac; \
+    fi; \
+    sh /usr/src/rustfs/scripts/download_rc.sh "${rc_arch}" musl "${RC_RELEASE}" /usr/local/bin/rc
 
 # -----------------------------
 # Development stage (keeps toolchain)

crates/e2e_test/src/distributed/multipart_fence_acceptance_test.rs

@@ -1285,7 +1285,9 @@ async fn list_parts_disk_loss_after_metadata_read(
     let error = result.expect_err("one missing directory and three unavailable participants cannot prove an empty upload");
     let response = error.raw_response().expect("ListParts must return an S3 service response");
     assert_eq!(response.status().as_u16(), 503);
-    assert_eq!(error.as_service_error().and_then(|err| err.meta().code()), Some("SlowDownRead"));
+    // Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
+    // ListParts read-quorum failure is reported as ServiceUnavailable.
+    assert_eq!(error.as_service_error().and_then(|err| err.meta().code()), Some("ServiceUnavailable"));
     assert!(response.headers().get("x-rustfs-error-cause").is_none());
     assert!(response.headers().get("retry-after").is_none());
     assert!(

crates/e2e_test/src/distributed/upgrade_test.rs

@@ -493,7 +493,12 @@ async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &Upgra
     assert!(legacy.parts().is_empty());
     let current = current.expect_err("current coordinator must not infer empty from missing directories and offline disks");
     assert_eq!(current.raw_response().expect("current S3 response").status().as_u16(), 503);
-    assert_eq!(current.as_service_error().and_then(|error| error.meta().code()), Some("SlowDownRead"));
+    // Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
+    // ListParts read-quorum failure is reported as ServiceUnavailable.
+    assert_eq!(
+        current.as_service_error().and_then(|error| error.meta().code()),
+        Some("ServiceUnavailable")
+    );
     dist.cluster.start_node_from_binary(3, previous).await?;
     wait_for_upgrade_write_readiness(&clients, &seed.history_bucket, "mixed-list-restored", CREDENTIAL_TIMEOUT).await?;
     for (path, before) in original_files {

crates/ecstore/src/disk/local.rs

@@ -11541,11 +11541,14 @@ impl LocalDisk {
     ) -> Result<()> {
         multipart_staging::reject_reserved_anchor(path)?;
         let fuse_with_admission = admission.is_some() && !crash_inject::is_armed(CrashPoint::MetaWriteAfterTmpBeforeRename, path);
+        // Held until the staging root is published, by the filesystem work
+        // itself, so a concurrent listing cannot prune the marker in between.
+        let mut index_pin = None;
         let prepared_index = if volume == RUSTFS_META_MULTIPART_BUCKET {
             if fuse_with_admission {
                 self.multipart_index.prepare_record_head(path, &buf).await
             } else {
-                self.multipart_index.record_head(path, &buf).await?;
+                index_pin = self.multipart_index.record_head(path, &buf).await?;
                 None
             }
         } else {
@@ -11571,8 +11574,10 @@ impl LocalDisk {
             let root = self.publication_root.clone();
             let lease = os::acquire_namespace_mutation_lease_with_owner(&file_path, namespace_owner).await;
             os::run_blocking_namespace_operation_with_sync(lease, do_sync, move || {
-                if let Some(record) = prepared_index {
-                    record.run_std()?;
+                let _index_pin = prepared_index.map(|record| record.run_std()).transpose()?;
+                #[cfg(test)]
+                if let Some(pin) = &_index_pin {
+                    pin.pause_for_test();
                 }
                 super::fs::create_dir_all_std(&tmp_volume_dir)?;
                 #[cfg(test)]
@@ -11616,12 +11621,16 @@ impl LocalDisk {
             return Err(DiskError::Unexpected);
         }
 
+        let publication_owner: Option<Arc<dyn Send + Sync>> = match index_pin {
+            Some(pin) => Some(Arc::new((namespace_owner.clone(), pin))),
+            None => namespace_owner.clone(),
+        };
         os::rename_all_with_precondition_and_parent_sync(
             &tmp_file_path,
             &file_path,
             &volume_dir,
             &self.publication_root,
-            namespace_owner.clone(),
+            publication_owner,
             admission.as_ref().map(|admission| admission.precondition()),
             admission.is_some() && sync && durability.syncs_commit_metadata(),
         )

crates/ecstore/src/disk/local/commit.rs

@@ -521,8 +521,12 @@ impl LocalDisk {
             state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
         })?;
         crate::hp_guard!("LocalDisk::rename_data");
-        if dst_volume == super::super::RUSTFS_META_MULTIPART_BUCKET {
-            self.multipart_index.record_info(dst_path, &fi).await?;
+        if dst_volume == super::super::RUSTFS_META_MULTIPART_BUCKET
+            && let Some(pin) = self.multipart_index.record_info(dst_path, &fi).await?
+        {
+            // The owner chain reaches every lease this rename publishes under,
+            // so the index marker stays pinned until that work has returned.
+            state.namespace_owner = Some(Arc::new((state.namespace_owner.take(), pin)));
         }
         #[cfg(feature = "e2e-test-hooks")]
         if fi.is_healing() {

crates/ecstore/src/disk/local/delete_recovery.rs

@@ -41,19 +41,53 @@ struct DiscoverySession {
     touched: Instant,
 }
 
-/// Plain (non-transaction) children of an object-scoped discovery root cost one
-/// budget entry per this many names: enough to bound a hostile prefix without
-/// making a real folder prefix undeletable.
+/// Names an object-scoped scan skips without a stat (plain children of the
+/// key's directory, and entries of a UUID child that cannot be delete evidence)
+/// cost one budget entry per this many names: enough to bound a hostile prefix
+/// without making a real folder prefix undeletable.
 const PLAIN_CHILDREN_PER_BUDGET_ENTRY: u32 = 64;
 
+/// Entries one step of the resumable (background) discovery may visit. The
+/// foreground bounded protocol has its own, larger per-call allowance.
+const DISCOVERY_STEP_ENTRIES: u32 = 4096;
+
+/// Wall-clock allowance of one bounded discovery call: a third of the drive
+/// timeout that wraps the call (including its wait for a lane worker), capped at
+/// ten seconds, so an incomplete walk returns before the call can time out.
+fn bounded_discovery_time_limit() -> std::time::Duration {
+    (crate::disk::disk_store::get_max_timeout_duration() / 3).min(std::time::Duration::from_secs(10))
+}
+
+/// Whether an entry inside a UUID child of an object-scoped root can name one
+/// of the key's delete transactions: a journal file of a rollback directory or
+/// a data-directory marker. Every other name is inert to discovery.
+fn may_be_delete_evidence(name: &str) -> bool {
+    [PREPARED, STORAGE_FORMAT_FILE_BACKUP, DELETE_QUORUM_COMMITTED_FILE].contains(&name)
+        || [
+            DELETE_DATA_DIR_MARKER_PREFIX,
+            RESERVED_DELETE_DATA_DIR_MARKER_PREFIX,
+            RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX,
+        ]
+        .iter()
+        .any(|prefix| delete_transaction_marker_id(name, prefix).is_some())
+}
+
 impl DiscoverySession {
     /// Consume one bounded step, returning ownership only when traversal remains.
     fn advance(self, limit: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>)> {
-        self.advance_bounded(limit, super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)
+        self.advance_bounded(limit, DISCOVERY_STEP_ENTRIES, &|| false)
             .map(|(transactions, next, _)| (transactions, next))
     }
 
-    fn advance_bounded(mut self, limit: u32, max_entries: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>, u32)> {
+    /// Walk until `limit` transactions, `max_entries` charged entries, the end
+    /// of the scope, or `should_stop` (checked before every entry); any early
+    /// stop returns the remaining traversal as continuation state.
+    fn advance_bounded(
+        mut self,
+        limit: u32,
+        max_entries: u32,
+        should_stop: &dyn Fn() -> bool,
+    ) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>, u32)> {
         if self.frames.is_empty() {
             match std::fs::read_dir(&self.root) {
                 Ok(entries) => self.frames.push((String::new(), entries)),
@@ -66,10 +100,11 @@ impl DiscoverySession {
         let mut transactions = BTreeSet::new();
         let mut seen_candidates = HashSet::new();
         let mut visited = 0u32;
-        let mut plain_children = 0u32;
+        let mut skipped_by_name = 0u32;
         while !self.frames.is_empty()
             && transactions.len() < usize::try_from(limit).map_err(|_| std::io::Error::from(ErrorKind::InvalidInput))?
             && visited < max_entries
+            && !should_stop()
         {
             let (parent, entries) = self
                 .frames
@@ -90,13 +125,22 @@ impl DiscoverySession {
                 return Err(std::io::Error::from(ErrorKind::InvalidData));
             }
             // An object-scoped scan only ever descends into transaction (UUID)
-            // directories at the root. Plain child keys of a prefix directory are
-            // skipped by name without a stat; they are charged at one entry per
-            // readdir batch so the work stays bounded while deleting a missing key
-            // that names a prefix with thousands of children can still complete.
-            if self.object.is_some() && parent.is_empty() && !canonical_uuid(&name).is_some_and(|id| !id.is_nil()) {
-                plain_children += 1;
-                if plain_children.is_multiple_of(PLAIN_CHILDREN_PER_BUDGET_ENTRY) {
+            // directories at the root, and inside them only journal files and
+            // data-directory markers can be evidence. Every other name (plain
+            // child keys of a prefix directory, and the `xl.meta`, data
+            // directories or children of a UUID-named child key) is skipped by
+            // name without a stat and charged at one entry per readdir batch, so
+            // the work stays bounded while deleting a missing key that names a
+            // folder prefix with many children, UUID-named or not, can complete.
+            if self.object.is_some()
+                && if parent.is_empty() {
+                    !canonical_uuid(&name).is_some_and(|id| !id.is_nil())
+                } else {
+                    !may_be_delete_evidence(&name)
+                }
+            {
+                skipped_by_name += 1;
+                if skipped_by_name.is_multiple_of(PLAIN_CHILDREN_PER_BUDGET_ENTRY) {
                     visited += 1;
                 }
                 continue;
@@ -1391,8 +1435,16 @@ impl LocalDisk {
         self.validate_delete_discovery_owner(&session)?;
         // The blocking owner drops every directory handle even if the RPC waiter disappears.
         // A partial inventory is never evidence of absence and never enters DISCOVERY_SESSIONS.
-        os::run_blocking_namespace_operation(lease, move || {
-            let (transactions, next, visited) = session.advance_bounded(256, max_entries)?;
+        // A large folder prefix may take a while to walk, so the probe rides the
+        // durable-mutation lane like bounded inspection, never the one-worker
+        // foreground lane that every other mutation on the drive queues behind.
+        // It also stops, incomplete, when its waiter goes away or well before the
+        // drive timeout, so a slow walk answers SlowDown instead of marking the
+        // drive failed or holding a lane worker after the request is gone.
+        let deadline = Instant::now() + bounded_discovery_time_limit();
+        os::run_cancellable_blocking_namespace_operation_with_sync(lease, true, move |is_cancelled| {
+            let should_stop = || is_cancelled() || Instant::now() >= deadline;
+            let (transactions, next, visited) = session.advance_bounded(256, max_entries, &should_stop)?;
             Ok(super::super::BoundedDeleteRecoveryDiscovery {
                 transactions: if next.is_none() { transactions } else { Vec::new() },
                 visited,
@@ -6561,6 +6613,93 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn delete_recovery_bounded_discovery_charges_uuid_named_children_once_and_keeps_evidence() {
+        let (_temp, disk, _, _) = fixture().await;
+        let path = disk.delete_discovery_path("recovery", Some("prefix")).expect("prefix path");
+        std::fs::create_dir_all(&path).expect("physical prefix");
+        // UUID-named child keys of the folder: each holds its own `xl.meta` and a
+        // data directory, neither of which can be evidence for the folder's key.
+        const CHILDREN: u32 = 2000;
+        for _ in 0..CHILDREN {
+            let child = path.join(Uuid::new_v4().to_string());
+            std::fs::create_dir_all(child.join(Uuid::new_v4().to_string())).expect("child key data dir");
+            std::fs::write(child.join(STORAGE_FORMAT_FILE), b"meta").expect("child key metadata");
+        }
+        // A UUID-named sub-prefix with children of its own.
+        const SUB_PREFIX_CHILDREN: u32 = 640;
+        let sub_prefix = path.join(Uuid::new_v4().to_string());
+        for index in 0..SUB_PREFIX_CHILDREN {
+            std::fs::create_dir_all(sub_prefix.join(format!("child-{index}"))).expect("sub-prefix child");
+        }
+        // Evidence of two interrupted transactions of the key itself: a journaled
+        // rollback directory, and a data-directory marker whose rollback is gone.
+        let journaled = Uuid::new_v4();
+        std::fs::create_dir_all(path.join(journaled.to_string())).expect("rollback directory");
+        std::fs::write(path.join(journaled.to_string()).join(PREPARED), b"{}").expect("journal");
+        let marker_only = Uuid::new_v4();
+        let data_dir = path.join(Uuid::new_v4().to_string());
+        std::fs::create_dir_all(&data_dir).expect("residual data directory");
+        std::fs::write(data_dir.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{marker_only}")), [])
+            .expect("reservation marker");
+
+        let page = disk
+            .discover_delete_transactions_bounded(
+                "recovery",
+                "prefix",
+                super::super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES,
+            )
+            .await
+            .expect("bounded probe");
+        assert!(page.complete, "a folder of UUID-named keys is a complete inventory");
+        // One entry per descent (children, sub-prefix, rollback and data
+        // directory) and per evidence file; inert names at 1/64.
+        let inert = 2 * CHILDREN + SUB_PREFIX_CHILDREN;
+        assert_eq!(page.visited, CHILDREN + 3 + 2 + inert / PLAIN_CHILDREN_PER_BUDGET_ENTRY);
+        let found = page.transactions.iter().map(|id| id.transaction).collect::<BTreeSet<_>>();
+        assert!(found.contains(&journaled), "a journaled rollback directory is evidence");
+        assert!(found.contains(&marker_only), "a marker-only residue is evidence");
+        assert_eq!(found.len(), 2, "no child key is mistaken for a transaction");
+    }
+
+    #[tokio::test]
+    async fn delete_recovery_bounded_discovery_stops_on_request_and_skips_only_inert_irregular_entries() {
+        let (_temp, disk, _, _) = fixture().await;
+        let path = disk.delete_discovery_path("recovery", Some("prefix")).expect("prefix path");
+        let child = path.join(Uuid::new_v4().to_string());
+        std::fs::create_dir_all(&child).expect("UUID-named child key");
+        // A stop request (cancellation or the wall-clock limit) ends the walk
+        // before any entry, leaving continuation state: never evidence of absence.
+        let session = disk
+            .new_delete_discovery_session("recovery", Some("prefix"), &path)
+            .expect("private session");
+        let (transactions, next, visited) = session
+            .advance_bounded(256, super::super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, &|| true)
+            .expect("stopped walk");
+        assert!(transactions.is_empty());
+        assert_eq!(visited, 0);
+        assert!(next.is_some(), "a stopped walk is incomplete");
+        drop(next);
+        #[cfg(unix)]
+        {
+            // An irregular entry whose name cannot be evidence is skipped unread.
+            std::os::unix::fs::symlink(&path, child.join("inert-link")).expect("inert symlink");
+            let page = disk
+                .discover_delete_transactions_bounded("recovery", "prefix", 4096)
+                .await
+                .expect("inert irregular entries are not evidence");
+            assert!(page.complete && page.transactions.is_empty());
+            // One with an evidence name still fails closed.
+            std::os::unix::fs::symlink(&path, child.join(PREPARED)).expect("evidence-named symlink");
+            assert!(
+                disk.discover_delete_transactions_bounded("recovery", "prefix", 4096)
+                    .await
+                    .is_err(),
+                "an evidence-named symlink is corruption, not a skipped name"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn delete_recovery_bounded_discovery_refuses_partial_inventory_without_public_cursor() {
         let (_temp, disk, _, _) = fixture().await;
@@ -6663,8 +6802,11 @@ mod tests {
         let filler_id = Uuid::new_v4().to_string();
         let filler = path.join(&filler_id);
         fs::create_dir_all(&filler).await.expect("non-evidence directory");
-        for index in 0..5000 {
-            fs::write(filler.join(format!("part-{index:04}")), [])
+        // Inert names are skipped at 1/64, so fill the step with entries that
+        // pass the evidence-name filter but are directories: each costs a full
+        // budget entry and names no transaction.
+        for _ in 0..5000 {
+            fs::create_dir(filler.join(format!("{DELETE_DATA_DIR_MARKER_PREFIX}{}", Uuid::new_v4())))
                 .await
                 .expect("non-evidence entry");
         }

crates/ecstore/src/disk/local/multipart_index.rs

@@ -4,6 +4,11 @@
 //! Bucket-to-staging-root candidates: one marker per staging root, recorded
 //! before the root is published and pruned once the root is gone. Source
 //! metadata stays authoritative.
+//!
+//! A marker without a staging root is either a leftover or an upload whose
+//! root is still being published. The publisher proves the second case by
+//! holding a [`PublicationPin`] from before it records the marker until its
+//! filesystem work has returned; no elapsed time is taken as that proof.
 
 use super::{LocalDisk, read_metadata_only_std};
 use crate::disk::{
@@ -29,9 +34,6 @@ use uuid::Uuid;
 
 const INDEX: &str = ".rustfs.sys/multipart-index-v1";
 const REBUILD_DIRENTS_PER_JOB: usize = 64;
-/// A marker is recorded before its staging root is published. A marker younger
-/// than this without a root is an upload in flight, never a leftover to prune.
-const MULTIPART_INDEX_PRUNE_GRACE: std::time::Duration = std::time::Duration::from_secs(300);
 type DirectoryEntries = Box<dyn Iterator<Item = io::Result<String>> + Send>;
 type PhysicalRoot = (u64, [u8; 16]);
 static OWNERS: LazyLock<parking_lot::Mutex<HashMap<PhysicalRoot, Weak<IndexOwner>>>> =
@@ -56,9 +58,15 @@ pub(super) struct IndexOwner {
     // drop so a cancelled scan can be resumed by the next caller.
     driving: AtomicBool,
     disabled: AtomicBool,
+    // Markers whose staging root some publisher is creating right now, with
+    // the number of publishers. Boot-local like the index itself: every writer
+    // of this physical drive in this process shares this owner.
+    publishing: parking_lot::Mutex<HashMap<PathBuf, usize>>,
     #[cfg(test)]
     rebuild_head_pause: parking_lot::Mutex<Option<RebuildHeadPause>>,
     #[cfg(test)]
+    publication_pause: parking_lot::Mutex<Option<RebuildHeadPause>>,
+    #[cfg(test)]
     observed_rebuild_queue: parking_lot::Mutex<Option<Arc<os::DiskBlockingQueue>>>,
     #[cfg(test)]
     rebuild_single_job: AtomicBool,
@@ -120,23 +128,6 @@ impl IndexDirectory {
             std::fs::File::open(self.path.join(name))
         }
     }
-    fn modified(&self, name: &str) -> io::Result<std::time::SystemTime> {
-        #[cfg(unix)]
-        {
-            self.open_file(name)?.metadata()?.modified()
-        }
-        #[cfg(not(unix))]
-        {
-            std::fs::metadata(self.path.join(name))?.modified()
-        }
-    }
-    /// Whether the marker is still inside the publication grace period. A
-    /// marker dated in the future (clock stepped back) is never a leftover.
-    fn marker_is_young(&self, name: &str) -> bool {
-        self.modified(name)
-            .ok()
-            .is_some_and(|modified| modified.elapsed().map_or(true, |age| age < MULTIPART_INDEX_PRUNE_GRACE))
-    }
     fn remove_file(&self, name: &str) -> io::Result<()> {
         #[cfg(unix)]
         {
@@ -280,9 +271,12 @@ impl IndexOwner {
             initialized: OnceLock::new(),
             driving: AtomicBool::new(false),
             disabled: AtomicBool::new(false),
+            publishing: parking_lot::Mutex::new(HashMap::new()),
             #[cfg(test)]
             rebuild_head_pause: parking_lot::Mutex::new(None),
             #[cfg(test)]
+            publication_pause: parking_lot::Mutex::new(None),
+            #[cfg(test)]
             observed_rebuild_queue: parking_lot::Mutex::new(None),
             #[cfg(test)]
             rebuild_single_job: AtomicBool::new(false),
@@ -394,8 +388,8 @@ impl IndexOwner {
             // The index is boot-local: every start rescans the staging area
             // before serving it, so a marker lost to a crash is recreated before
             // it can be consulted. Creation is enough; no fsync of the marker or
-            // its ancestors. An existing marker is touched so pruning sees the
-            // new upload's age, not the previous one's.
+            // its ancestors. An existing marker needs nothing: the caller's
+            // publication pin, not the marker's age, keeps it from being pruned.
             match openat(
                 &parent,
                 source_root,
@@ -407,16 +401,7 @@ impl IndexOwner {
                         return Err(io::Error::other("index marker is not a file"));
                     }
                 }
-                Err(rustix::io::Errno::EXIST) => {
-                    // Best effort: a filesystem that refuses to set times keeps
-                    // the previous upload's age, which only shortens the prune
-                    // grace for this key; index completeness does not depend on it.
-                    if let Ok(marker) =
-                        openat(&parent, source_root, OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, Mode::empty())
-                    {
-                        let _ = std::fs::File::from(marker).set_modified(std::time::SystemTime::now());
-                    }
-                }
+                Err(rustix::io::Errno::EXIST) => {}
                 Err(error) => return Err(error.into()),
             }
         }
@@ -432,6 +417,22 @@ impl IndexOwner {
         Ok(())
     }
 
+    /// Declare that the caller is about to publish the staging root of
+    /// `marker`. Must precede the marker lease the caller records under, so a
+    /// prune that found no pin has released that lease before the marker is
+    /// (re)created.
+    fn pin_publication(self: &Arc<Self>, marker: PathBuf) -> PublicationPin {
+        *self.publishing.lock().entry(marker.clone()).or_default() += 1;
+        PublicationPin {
+            owner: Arc::clone(self),
+            marker,
+        }
+    }
+
+    fn is_publishing(&self, marker: &Path) -> bool {
+        self.publishing.lock().contains_key(marker)
+    }
+
     fn ensure_current_root(&self) -> io::Result<()> {
         #[cfg(unix)]
         let current = {
@@ -513,9 +514,12 @@ impl IndexOwner {
         }
     }
 
-    pub(super) async fn record_info(self: &Arc<Self>, path: &str, info: &FileInfo) -> Result<()> {
+    /// Record the marker of `path`'s staging root. The returned pin must stay
+    /// alive until the filesystem work that publishes the root has returned,
+    /// including work that outlives a cancelled caller.
+    pub(super) async fn record_info(self: &Arc<Self>, path: &str, info: &FileInfo) -> Result<Option<PublicationPin>> {
         if canonical_upload(path).is_none() {
-            return Ok(());
+            return Ok(None);
         }
         let owner = self.clone();
         let path = path.to_owned();
@@ -531,6 +535,7 @@ impl IndexOwner {
                     .join(path.split('/').next().unwrap_or_default())
             })
             .unwrap_or_else(|| owner.base.join(INDEX));
+        let pin = self.pin_publication(marker.clone());
         let lease = os::acquire_namespace_mutation_lease_with_owner(&marker, None).await;
         os::run_blocking_namespace_operation_with_sync(lease, true, move || {
             match owner.record(&path, &info) {
@@ -544,7 +549,7 @@ impl IndexOwner {
             }
         })
         .await?;
-        Ok(())
+        Ok(Some(pin))
     }
 
     pub(super) async fn prepare_record_head(self: &Arc<Self>, path: &str, bytes: &[u8]) -> Option<PreparedHeadRecord> {
@@ -578,20 +583,23 @@ impl IndexOwner {
                     .join(upload.split('/').next().unwrap_or_default())
             })
             .unwrap_or_else(|| self.base.join(INDEX));
+        let pin = self.pin_publication(marker.clone());
         let lease = os::acquire_namespace_mutation_lease_with_owner(&marker, None).await;
         Some(PreparedHeadRecord {
-            owner: Arc::clone(self),
             upload: upload.to_owned(),
             info,
+            pin,
             _lease: lease,
         })
     }
 
-    pub(super) async fn record_head(self: &Arc<Self>, path: &str, bytes: &[u8]) -> Result<()> {
+    /// Record the marker of the staging root that `path` belongs to. See
+    /// [`Self::record_info`] for how long the returned pin must live.
+    pub(super) async fn record_head(self: &Arc<Self>, path: &str, bytes: &[u8]) -> Result<Option<PublicationPin>> {
         let Some(prepared) = self.prepare_record_head(path, bytes).await else {
-            return Ok(());
+            return Ok(None);
         };
-        prepared.run().await
+        prepared.run().await.map(Some)
     }
 
     async fn page(self: &Arc<Self>, bucket: &str, marker: Option<&str>, disk_id: Uuid) -> Result<MultipartIndexPage> {
@@ -642,17 +650,20 @@ impl IndexOwner {
             // to every key ever multipart-initiated on this drive.
             let mut live = Vec::with_capacity(roots.len());
             let mut stale = Vec::new();
+            let markers = owner.base.join(INDEX).join(sha(&bucket));
             for root in roots {
+                // The marker precedes its staging root: a pinned one is an
+                // upload in flight and stays a candidate. The pin is read
+                // before the root, so a publisher that finished in between has
+                // already made its root visible.
+                if owner.is_publishing(&markers.join(&root)) {
+                    live.push(root);
+                    continue;
+                }
                 match owner.directory(&Path::new(RUSTFS_META_MULTIPART_BUCKET).join(&root)) {
                     Ok(_) => live.push(root),
                     Err(error) if matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::NotADirectory) => {
-                        // The marker precedes its staging root; a young one is an
-                        // upload in flight and stays a candidate.
-                        if directory.as_ref().is_some_and(|index| index.marker_is_young(&root)) {
-                            live.push(root);
-                        } else {
-                            stale.push(root);
-                        }
+                        stale.push(root);
                     }
                     Err(error) => return Err(DiskError::from(error)),
                 }
@@ -677,9 +688,11 @@ impl IndexOwner {
     }
 
     /// Remove one marker whose staging root was gone when a page examined it.
-    /// The marker's own namespace lease serializes this with `record_info`; the
-    /// staging root and the marker's age are re-checked under that lease, so an
-    /// upload recorded in between keeps its marker.
+    /// The marker's own namespace lease serializes this with recording. Under
+    /// that lease the publication pin is read first and the staging root
+    /// second: a publisher pinned before the read keeps the marker, one that
+    /// already finished has made its root visible, and one that pins later
+    /// records its marker only after this lease is released.
     async fn prune_marker(self: &Arc<Self>, bucket: &str, root: &str) -> Result<()> {
         let owner = self.clone();
         let bucket_directory = Path::new(INDEX).join(sha(bucket));
@@ -690,6 +703,9 @@ impl IndexOwner {
         // not queue behind every other mutation on the one-worker foreground lane.
         os::run_blocking_namespace_operation_with_sync(lease, true, move || {
             owner.ensure_current_root()?;
+            if owner.is_publishing(&marker) {
+                return Ok(());
+            }
             match owner.directory(&Path::new(RUSTFS_META_MULTIPART_BUCKET).join(&root)) {
                 Ok(_) => return Ok(()),
                 Err(error) if matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::NotADirectory) => {}
@@ -700,9 +716,6 @@ impl IndexOwner {
                 Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
                 Err(error) => return Err(error),
             };
-            if directory.marker_is_young(&root) {
-                return Ok(());
-            }
             match directory.remove_file(&root) {
                 Err(error) if error.kind() != io::ErrorKind::NotFound => Err(error),
                 _ => Ok(()),
@@ -713,34 +726,69 @@ impl IndexOwner {
     }
 }
 
-pub(super) struct PreparedHeadRecord {
+/// Proof that the staging root of one marker is being published. While any pin
+/// for a marker is alive, pruning leaves that marker alone.
+#[must_use = "dropping the pin lets a concurrent listing prune the marker before its staging root exists"]
+pub(super) struct PublicationPin {
     owner: Arc<IndexOwner>,
+    marker: PathBuf,
+}
+
+impl PublicationPin {
+    /// Test seam between recording the marker and publishing the staging root.
+    #[cfg(test)]
+    pub(super) fn pause_for_test(&self) {
+        let pause = self.owner.publication_pause.lock().take();
+        if let Some(pause) = pause {
+            let _pause = os::enter_blocking_test_hook();
+            let _ = pause.started.send(());
+            let _ = pause.release.recv_timeout(std::time::Duration::from_secs(10));
+        }
+    }
+}
+
+impl Drop for PublicationPin {
+    fn drop(&mut self) {
+        let mut publishing = self.owner.publishing.lock();
+        if let Some(count) = publishing.get_mut(&self.marker) {
+            *count -= 1;
+            if *count == 0 {
+                publishing.remove(&self.marker);
+            }
+        }
+    }
+}
+
+pub(super) struct PreparedHeadRecord {
     upload: String,
     info: FileInfo,
+    pin: PublicationPin,
     _lease: Arc<os::NamespaceMutationLease>,
 }
 
 impl PreparedHeadRecord {
-    pub(super) fn run_std(self) -> io::Result<()> {
+    /// Record the marker and release its lease. The caller publishes the
+    /// staging root while it holds the returned pin.
+    pub(super) fn run_std(self) -> io::Result<PublicationPin> {
         let Self {
-            owner,
             upload,
             info,
+            pin,
             _lease,
         } = self;
-        match owner.record(&upload, &info) {
+        match pin.owner.record(&upload, &info) {
             Err(error) if error.kind() == io::ErrorKind::InvalidData => {
-                owner.disabled.store(true, Ordering::Release);
-                Ok(())
+                pin.owner.disabled.store(true, Ordering::Release);
+                Ok(pin)
             }
-            result => result,
+            Err(error) => Err(error),
+            Ok(()) => Ok(pin),
         }
     }
 
-    async fn run(self) -> Result<()> {
+    async fn run(self) -> Result<PublicationPin> {
         let lease = Arc::clone(&self._lease);
-        os::run_blocking_namespace_operation_with_sync(lease, true, move || self.run_std()).await?;
-        Ok(())
+        Ok(os::run_blocking_namespace_operation_with_sync(lease, true, move || self.run_std()).await?)
     }
 }
 
@@ -958,47 +1006,274 @@ mod tests {
         let live = format!("{:064x}", 1);
         let finished = format!("{:064x}", 2);
         let in_flight = format!("{:064x}", 3);
-        let clock_stepped = format!("{:064x}", 4);
-        for name in [&live, &finished, &in_flight, &clock_stepped] {
+        for name in [&live, &finished, &in_flight] {
             std::fs::write(directory.join(name), b"1").expect("marker");
         }
         // Completion and abort remove the staging root and leave the marker.
         std::fs::create_dir_all(root.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&live)).expect("live staging root");
-        set_marker_modified(&directory.join(&finished), std::time::SystemTime::now() - MULTIPART_INDEX_PRUNE_GRACE * 2);
-        // A marker dated after a backwards clock step is never a leftover.
-        set_marker_modified(
-            &directory.join(&clock_stepped),
-            std::time::SystemTime::now() + MULTIPART_INDEX_PRUNE_GRACE * 2,
-        );
-        let page = disk.multipart_index.page("bucket", None, Uuid::new_v4()).await.expect("page");
-        // The in-flight marker has no staging root yet but is young: a candidate, not a leftover.
-        assert_eq!(page.roots, vec![live.clone(), in_flight.clone(), clock_stepped.clone()]);
+        let owner = &disk.multipart_index;
+        let publishing = owner.pin_publication(owner.base.join(INDEX).join(sha("bucket")).join(&in_flight));
+        let second_publisher = owner.pin_publication(owner.base.join(INDEX).join(sha("bucket")).join(&in_flight));
+        let page = owner.page("bucket", None, Uuid::new_v4()).await.expect("page");
+        // The in-flight marker has no staging root yet but is pinned: a candidate, not a leftover.
+        assert_eq!(page.roots, vec![live.clone(), in_flight.clone()]);
         assert!(page.next_marker.is_none());
         assert!(directory.join(&live).exists(), "a live root keeps its marker");
-        assert!(directory.join(&in_flight).exists(), "a marker inside the grace period is kept");
-        assert!(directory.join(&clock_stepped).exists(), "a future-dated marker is kept");
-        assert!(!directory.join(&finished).exists(), "a finished root's marker is pruned");
-        // The next page no longer pays for the pruned marker.
-        let page = disk
-            .multipart_index
+        assert!(directory.join(&in_flight).exists(), "a pinned marker is kept");
+        assert!(!directory.join(&finished).exists(), "a finished root's marker is pruned at once");
+        // One of two publishers of the same root finishing does not unpin it.
+        drop(publishing);
+        let page = owner
             .page("bucket", None, Uuid::new_v4())
             .await
-            .expect("page after pruning");
-        assert_eq!(page.roots, vec![live, in_flight, clock_stepped]);
+            .expect("page with one publisher left");
+        assert_eq!(page.roots, vec![live.clone(), in_flight.clone()]);
+        assert!(directory.join(&in_flight).exists(), "the remaining publisher keeps the marker");
+        // A publication that ended without a staging root leaves a leftover.
+        drop(second_publisher);
+        let page = owner
+            .page("bucket", None, Uuid::new_v4())
+            .await
+            .expect("page after the failed publication");
+        assert_eq!(page.roots, vec![live]);
+        assert!(!directory.join(&in_flight).exists(), "an unpinned marker without a root is pruned");
+        assert!(owner.publishing.lock().is_empty(), "finished publishers leave no pin behind");
     }
 
-    /// Move a marker's modification time outside the publication grace period.
-    fn age_marker(marker: &Path) {
-        set_marker_modified(marker, std::time::SystemTime::now() - MULTIPART_INDEX_PRUNE_GRACE * 2);
+    fn pause_next_publication(owner: &IndexOwner) -> (tokio::sync::oneshot::Receiver<()>, std::sync::mpsc::Sender<()>) {
+        let (started, waiting) = tokio::sync::oneshot::channel();
+        let (release, released) = std::sync::mpsc::channel();
+        *owner.publication_pause.lock() = Some(RebuildHeadPause {
+            started,
+            release: released,
+        });
+        (waiting, release)
     }
 
-    fn set_marker_modified(marker: &Path, modified: std::time::SystemTime) {
+    /// The window the pin closes: the marker is recorded and its lease is
+    /// released, the staging root is not published yet, and a listing examines
+    /// the bucket. The marker is dated a day back so that no grace period could
+    /// be what keeps it.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn multipart_index_listing_during_source_publication_keeps_the_new_upload_listed() {
+        let root = tempfile::tempdir().expect("disk root");
+        let disk = Arc::new(open(root.path()).await);
+        let (path, info) = upload("bucket", "published-while-listed");
+        let source_root = path.split('/').next().expect("source sha").to_owned();
+        let marker = root.path().join(INDEX).join(sha("bucket")).join(&source_root);
+        let source_head = root
+            .path()
+            .join(RUSTFS_META_MULTIPART_BUCKET)
+            .join(&path)
+            .join(STORAGE_FORMAT_FILE);
+        let (paused, release) = pause_next_publication(&disk.multipart_index);
+        let writer = tokio::spawn({
+            let disk = Arc::clone(&disk);
+            let path = path.clone();
+            async move { disk.write_metadata("bucket", RUSTFS_META_MULTIPART_BUCKET, &path, info).await }
+        });
+        tokio::time::timeout(std::time::Duration::from_secs(5), paused)
+            .await
+            .expect("the publication reaches the pause")
+            .expect("pause sender");
+        assert!(marker.exists(), "the marker precedes the staging root");
+        assert!(!source_head.exists(), "the staging root is not published yet");
         std::fs::File::options()
             .write(true)
-            .open(marker)
+            .open(&marker)
             .expect("marker")
-            .set_modified(modified)
-            .expect("set marker time");
+            .set_modified(std::time::SystemTime::now() - std::time::Duration::from_secs(86_400))
+            .expect("age the marker");
+
+        let identity = Uuid::new_v4();
+        let page = disk
+            .multipart_index
+            .page("bucket", None, identity)
+            .await
+            .expect("page during publication");
+        assert_eq!(page.roots, vec![source_root.clone()], "an upload in flight stays a candidate");
+        assert!(marker.exists(), "a listing must not prune the marker of an upload in flight");
+
+        release.send(()).expect("resume the publication");
+        writer.await.expect("writer task").expect("the upload metadata is published");
+        assert!(source_head.exists());
+        let page = disk
+            .multipart_index
+            .page("bucket", None, identity)
+            .await
+            .expect("page after publication");
+        assert_eq!(page.roots, vec![source_root], "the published upload is listed");
+        assert!(marker.exists());
+        assert!(disk.multipart_index.publishing.lock().is_empty());
+    }
+
+    /// A timed-out caller drops its future while the filesystem work goes on.
+    /// The pin belongs to that work, so the marker survives until the staging
+    /// root the detached work publishes is visible.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn multipart_index_cancelled_publisher_keeps_its_marker_until_the_detached_work_returns() {
+        let root = tempfile::tempdir().expect("disk root");
+        let disk = Arc::new(open(root.path()).await);
+        let (path, info) = upload("bucket", "caller-timed-out");
+        let source_root = path.split('/').next().expect("source sha").to_owned();
+        let marker = root.path().join(INDEX).join(sha("bucket")).join(&source_root);
+        let source_head = root
+            .path()
+            .join(RUSTFS_META_MULTIPART_BUCKET)
+            .join(&path)
+            .join(STORAGE_FORMAT_FILE);
+        let (paused, release) = pause_next_publication(&disk.multipart_index);
+        let writer = tokio::spawn({
+            let disk = Arc::clone(&disk);
+            let path = path.clone();
+            async move { disk.write_metadata("bucket", RUSTFS_META_MULTIPART_BUCKET, &path, info).await }
+        });
+        tokio::time::timeout(std::time::Duration::from_secs(5), paused)
+            .await
+            .expect("the publication reaches the pause")
+            .expect("pause sender");
+        writer.abort();
+        assert!(writer.await.expect_err("the caller is gone").is_cancelled());
+
+        let identity = Uuid::new_v4();
+        let page = disk
+            .multipart_index
+            .page("bucket", None, identity)
+            .await
+            .expect("page after the caller left");
+        assert_eq!(page.roots, vec![source_root.clone()]);
+        assert!(marker.exists(), "the detached publication still owns the marker");
+
+        release.send(()).expect("resume the detached publication");
+        tokio::time::timeout(std::time::Duration::from_secs(10), async {
+            while !source_head.exists() || !disk.multipart_index.publishing.lock().is_empty() {
+                tokio::time::sleep(std::time::Duration::from_millis(5)).await;
+            }
+        })
+        .await
+        .expect("the detached publication completes");
+        let page = disk
+            .multipart_index
+            .page("bucket", None, identity)
+            .await
+            .expect("page after the detached publication");
+        assert_eq!(page.roots, vec![source_root], "the upload the detached work published is listed");
+        assert!(marker.exists());
+    }
+
+    /// A page classifies a marker as stale before the publisher pins it; the
+    /// prune that follows must read the pin again under the marker lease.
+    #[tokio::test]
+    async fn multipart_index_prune_rechecks_the_pin_under_the_marker_lease() {
+        let root = tempfile::tempdir().expect("disk root");
+        let disk = open(root.path()).await;
+        let owner = &disk.multipart_index;
+        let directory = root.path().join(INDEX).join(sha("bucket"));
+        std::fs::create_dir_all(&directory).expect("marker directory");
+        let name = format!("{:064x}", 9);
+        std::fs::write(directory.join(&name), b"1").expect("marker");
+
+        let pin = owner.pin_publication(owner.base.join(INDEX).join(sha("bucket")).join(&name));
+        owner.prune_marker("bucket", &name).await.expect("prune of a pinned marker");
+        assert!(directory.join(&name).exists(), "a publisher that pinned after the page keeps its marker");
+
+        drop(pin);
+        owner.prune_marker("bucket", &name).await.expect("prune of a leftover");
+        assert!(!directory.join(&name).exists(), "the same marker is a leftover once nobody publishes it");
+    }
+
+    /// The publisher pins before it queues on the marker lease. A prune that
+    /// holds the lease and found no pin has therefore unlinked the marker
+    /// before the publisher records it, and the record recreates it.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn multipart_index_publisher_pins_before_it_waits_for_the_marker_lease() {
+        let root = tempfile::tempdir().expect("disk root");
+        let disk = Arc::new(open(root.path()).await);
+        let owner = Arc::clone(&disk.multipart_index);
+        let (path, info) = upload("bucket", "queued-behind-a-prune");
+        let source_root = path.split('/').next().expect("source sha").to_owned();
+        let marker = owner.base.join(INDEX).join(sha("bucket")).join(&source_root);
+        let visible_marker = root.path().join(INDEX).join(sha("bucket")).join(&source_root);
+
+        // Stand in for a prune that owns the marker lease and already read "no pin".
+        let prune_lease = os::acquire_namespace_mutation_lease_with_owner(&marker, None).await;
+        assert!(!owner.is_publishing(&marker));
+        let writer = tokio::spawn({
+            let disk = Arc::clone(&disk);
+            let path = path.clone();
+            async move { disk.write_metadata("bucket", RUSTFS_META_MULTIPART_BUCKET, &path, info).await }
+        });
+        tokio::time::timeout(std::time::Duration::from_secs(5), async {
+            while !owner.is_publishing(&marker) {
+                tokio::time::sleep(std::time::Duration::from_millis(2)).await;
+            }
+        })
+        .await
+        .expect("the publisher pins while the marker lease is still held by the prune");
+        assert!(!visible_marker.exists(), "the marker is recorded only under the lease");
+
+        drop(prune_lease);
+        writer.await.expect("writer task").expect("the upload metadata is published");
+        assert!(visible_marker.exists(), "the record after the prune recreates the marker");
+        let page = owner.page("bucket", None, Uuid::new_v4()).await.expect("page");
+        assert_eq!(page.roots, vec![source_root]);
+    }
+
+    /// Stale markers at the start of a page, at its cursor, and filling a whole
+    /// page: every page still satisfies the receiver's contract and advances.
+    #[tokio::test]
+    async fn multipart_index_pages_of_stale_markers_keep_valid_cursors() {
+        let root = tempfile::tempdir().expect("disk root");
+        let disk = open(root.path()).await;
+        let directory = root.path().join(INDEX).join(sha("bucket"));
+        std::fs::create_dir_all(&directory).expect("marker directory");
+        // Page one is entirely stale. Page two is live except its first name
+        // and the name its cursor lands on. Two live names follow.
+        let total = MULTIPART_INDEX_PAGE_SIZE * 2 + 2;
+        let stale = |i: usize| i <= MULTIPART_INDEX_PAGE_SIZE || i == MULTIPART_INDEX_PAGE_SIZE * 2 - 1;
+        for i in 0..total {
+            let name = format!("{i:064x}");
+            std::fs::write(directory.join(&name), b"1").expect("marker");
+            if !stale(i) {
+                std::fs::create_dir_all(root.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&name)).expect("staging root");
+            }
+        }
+        let disk_id = Uuid::new_v4();
+        let first = disk.multipart_index.page("bucket", None, disk_id).await.expect("first page");
+        first
+            .validate("bucket", None, disk_id)
+            .expect("an empty page with a cursor validates");
+        assert!(first.roots.is_empty(), "a fully stale page has no roots");
+        let cursor = first.next_marker.clone().expect("more markers follow a fully stale page");
+        assert_eq!(cursor, format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE - 1));
+
+        let second = disk
+            .multipart_index
+            .page("bucket", Some(&cursor), disk_id)
+            .await
+            .expect("second page");
+        second
+            .validate("bucket", Some(&cursor), disk_id)
+            .expect("stale first and cursor names validate");
+        assert_eq!(second.roots.len(), MULTIPART_INDEX_PAGE_SIZE - 2);
+        assert_eq!(second.roots.first(), Some(&format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE + 1)));
+        let cursor = second
+            .next_marker
+            .clone()
+            .expect("the cursor is the stale last examined name");
+        assert_eq!(cursor, format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE * 2 - 1));
+        assert!(second.roots.last() < Some(&cursor));
+
+        let third = disk
+            .multipart_index
+            .page("bucket", Some(&cursor), disk_id)
+            .await
+            .expect("third page");
+        third.validate("bucket", Some(&cursor), disk_id).expect("last page validates");
+        assert_eq!(third.roots.len(), 2);
+        assert!(third.next_marker.is_none());
+        let remaining = std::fs::read_dir(&directory).expect("marker directory").count();
+        assert_eq!(remaining, total - MULTIPART_INDEX_PAGE_SIZE - 2, "every stale marker is pruned");
     }
 
     #[tokio::test]
@@ -1015,7 +1290,6 @@ mod tests {
                 std::fs::create_dir_all(root.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&name)).expect("staging root");
             }
         }
-        age_marker(&directory.join(&stale));
         let disk_id = Uuid::new_v4();
         let first = disk.multipart_index.page("bucket", None, disk_id).await.expect("first page");
         first

crates/ecstore/src/disk/local/multipart_source_state.rs

@@ -813,7 +813,8 @@ impl SourceStore {
         Ok(())
     }
     pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
-        self.multipart_index
+        let index_pin = self
+            .multipart_index
             .record_head(&format!("{}/xl.meta", self.path), bytes)
             .await?;
         invalidate_active_multipart_source_head(&self.source);
@@ -822,6 +823,7 @@ impl SourceStore {
         let root = self.publication_root.clone();
         let bytes = bytes.to_vec();
         os::run_blocking_namespace_operation(self.lease.clone(), move || {
+            let _index_pin = index_pin;
             os::replace_multipart_source_metadata(&path, &bytes, &base, &root)
         })
         .await

crates/ecstore/src/disk/mod.rs

@@ -2585,7 +2585,13 @@ pub struct BoundedDeleteRecoveryDiscovery {
     pub complete: bool,
 }
 
-pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 4096;
+/// Entries one bounded discovery call may charge on one drive. Object-scoped
+/// discovery charges each transaction-shaped (UUID) child of the key's
+/// directory once and every name that cannot be delete evidence at
+/// 1/`PLAIN_CHILDREN_PER_BUDGET_ENTRY`, so this admits a missing-key delete of a
+/// folder prefix with this many UUID-named children per drive. The request
+/// budget (`DeleteRecoveryWorkBudget`) still bounds the total across keys.
+pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 262_144;
 pub const DELETE_RECOVERY_MAX_INSPECTION_ENTRIES: u32 = 65536;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

crates/ecstore/src/object_api/types.rs

@@ -975,9 +975,11 @@ pub struct DeleteRecoveryWorkBudget {
     remaining: std::sync::atomic::AtomicU32,
 }
 
-/// Entries one disk may visit per recovery step (discovery plus inspection).
-/// Every disk of every set a request fans out to is admitted up to this much
-/// and charged on completion, so a budget must cover that many disks.
+/// Per-drive unit a request budget is sized by (a typical discovery plus one
+/// inspection). Admission charges what a call visited, not this unit, so it only
+/// scales the request total with the drives a request can reach; it is kept
+/// independent of the per-call caps, which may let one key's probe use more of
+/// the same total on a large folder.
 const DELETE_RECOVERY_RESERVATION_PER_DISK: u32 = 4096 + 65_536;
 
 impl Default for DeleteRecoveryWorkBudget {

crates/ecstore/src/set_disk/mod.rs

@@ -11040,7 +11040,12 @@ mod tests {
         let live = Uuid::new_v4();
         let orphan = Uuid::new_v4();
         let object_dir = dir.path().join("bucket/obj");
-        write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &[live]).await;
+        // Part files are inert to discovery and cost 1/64 of an entry, so the
+        // first step is exhausted by what still costs a full entry each: the
+        // data directories of many versions, all referenced by the metadata.
+        let versions = (0..4200).map(|_| Uuid::new_v4()).collect::<Vec<_>>();
+        let referenced = std::iter::once(live).chain(versions.iter().copied()).collect::<Vec<_>>();
+        write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &referenced).await;
         let live_dir = object_dir.join(live.to_string());
         let orphan_dir = object_dir.join(orphan.to_string());
         fs::create_dir_all(&live_dir).await.expect("create live multipart directory");
@@ -11049,10 +11054,14 @@ mod tests {
             .await
             .expect("write orphan payload");
         let parts_dir = live_dir.clone();
+        let versions_root = object_dir.clone();
         tokio::task::spawn_blocking(move || {
             for part in 1..=4200 {
                 std::fs::write(parts_dir.join(format!("part.{part}")), b"live").expect("write live multipart entry");
             }
+            for version in versions {
+                std::fs::create_dir(versions_root.join(version.to_string())).expect("create referenced version directory");
+            }
         })
         .await
         .expect("create multipart fixture");
@@ -11062,7 +11071,10 @@ mod tests {
             .await
             .expect("read first discovery page");
         assert!(first.transactions.is_empty());
-        assert!(first.next.is_some(), "ordinary multipart entries must exhaust the first scan budget");
+        assert!(
+            first.next.is_some(),
+            "the data directories of many versions must exhaust the first scan step"
+        );
         let last = disk
             .discover_delete_transactions("bucket", Some("obj"), first.next, 1)
             .await

crates/ecstore/src/set_disk/ops/delete_recovery.rs

@@ -1709,6 +1709,37 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn delete_recovery_missing_key_naming_a_folder_of_uuid_named_keys_completes() {
+        let bucket = "recover-uuid-named-folder";
+        let (dirs, store, _) = fixture(bucket).await;
+        // More UUID-named child keys per drive than the former 4096-entry probe
+        // could cover once each child's `xl.meta` and data directory were charged
+        // (three entries per child).
+        const CHILDREN: usize = 1500;
+        for dir in &dirs {
+            let prefix = dir.path().join(bucket).join("uploads");
+            for _ in 0..CHILDREN {
+                let child = prefix.join(Uuid::new_v4().to_string());
+                std::fs::create_dir_all(child.join(Uuid::new_v4().to_string())).expect("child key data dir");
+                std::fs::write(child.join(crate::disk::STORAGE_FORMAT_FILE), b"meta").expect("child key metadata");
+            }
+        }
+        let error = store
+            .delete_object(bucket, "uploads", ObjectOptions::default())
+            .await
+            .expect_err("the folder's own key does not exist");
+        assert!(
+            crate::error::is_err_object_not_found(&error),
+            "a complete inventory keeps the missing-key no-op semantics: {error:?}"
+        );
+        assert!(dirs.iter().all(|dir| {
+            std::fs::read_dir(dir.path().join(bucket).join("uploads"))
+                .map(|entries| entries.count() == CHILDREN)
+                .unwrap_or(false)
+        }));
+    }
+
     #[tokio::test]
     async fn delete_recovery_batch_budget_survives_pool_and_set_fanout() {
         use crate::layout::endpoint::Endpoint;

crates/gateway-adapter/Cargo.toml

@@ -0,0 +1,48 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[package]
+name = "rustfs-gateway-adapter"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+homepage.workspace = true
+description = "RustFS implementations of the rustfs-gateway extension traits (credentials, policy facts, virtual hosts, bucket owner and request id)."
+keywords = ["rustfs", "gateway", "s3", "iam"]
+categories = ["web-programming", "authentication"]
+authors.workspace = true
+publish = false
+
+[package.metadata.gateway]
+# Documentary only: this crate is a ring-2 consumer of the gateway (it depends on
+# RustFS business crates). The RustFS repository does not run the gateway's ring guard.
+ring = 2
+
+[lints]
+workspace = true
+
+[dependencies]
+http = { workspace = true }
+rustfs-gateway = { workspace = true }
+rustfs-credentials = { workspace = true }
+rustfs-iam = { workspace = true }
+rustfs-policy = { workspace = true }
+serde_json = { workspace = true }
+
+[dev-dependencies]
+rustfs-gateway-http = { workspace = true }
+time = { workspace = true }
+tokio = { workspace = true, features = ["macros", "rt"] }

crates/gateway-adapter/src/authz_audit.rs

@@ -0,0 +1,548 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! The gateway's [`AuthzAuditSink`]: the legacy denial record for a refusal the gateway enforced.
+//!
+//! Responsible for: one [`GatewayDenialRecord`] for each authorization decision the gateway made
+//! at an enforcing stage that ended the request with a refusal (`Deny` or `Indeterminate`), after
+//! the legacy audit switch and the legacy denial rate limiter have both admitted it.
+//! NOT responsible for: the authoritative authorization decision. On the gateway stack that is the
+//! legacy access check in owned form, run by the handler before the use case, and it writes its own
+//! denial record (`DenialContext::record`, `rustfs/src/storage/access.rs`). The gateway
+//! `Authorizer` is only an earlier gate that may refuse what the legacy check certainly refuses, so
+//! a request it lets through is not decided yet, and a request it refuses never reaches the check.
+//! Nor is this module responsible for building the `rustfs_audit::AuditEntry`, filling its
+//! request-line fields, or dispatching it: that is the binary's [`DenialAuditSink`].
+//! Upstream: the gateway service, once per decision (`AuthzAuditSink::on_decision`). Downstream:
+//! the binary's [`DenialAuditSink`].
+//!
+//! # What is recorded, and what is not
+//!
+//! - An `Allow` is never recorded: at the early gate it is not the final decision, and a handler
+//!   denial after it would contradict the record.
+//! - The missing-object visibility probe is never recorded. For `GetObject` the gateway also asks
+//!   `s3:ListBucket`, only to choose between `404` and `403` for an absent object; that answer
+//!   refuses nothing, yet it reaches the sink as an `Input` event of its own. Legacy keeps the same
+//!   probe out of the audit (`suppress_denial_audit`, `rustfs/src/app/object/head.rs`).
+//! - Every other `Deny` or `Indeterminate` ended its request: a route-stage refusal is answered
+//!   before the body is read, and an input-stage refusal either fails the stage or prevents the
+//!   gateway core from constructing the authorized request. One refused request therefore yields
+//!   exactly one record.
+//!
+//! # Telling the visibility probe apart
+//!
+//! At the pinned gateway rev every event's `resources` starts with the request the decision was
+//! about. For the route and input events that is the route target, whose `action` equals its
+//! `route_action` (`crates/gateway/src/service.rs`, the route request and every asked question,
+//! and the input stage's route request). The visibility event alone is built from the route
+//! target with only `action` replaced by `s3:ListBucket`, keeping the target's `route_action`
+//! (`crates/gateway/src/ext/authz_audit.rs`, `emit_input_safely`; `crates/gateway/src/ext/authorizer.rs`,
+//! `InputAuthzRequest::new`). An `Input` event whose first resource asks an action other than its
+//! route action is the visibility event.
+//!
+//! # Known limit: the denied action and resource
+//!
+//! `trigger` (and `api.name`), `api.bucket` and `api.object` are the event's `action`, `bucket` and
+//! `key`: the operation's declared action and primary target. The event carries no per-resource
+//! decision, so the sink cannot name a derived resource that was refused. This is exact only while
+//! every early-gate refusal concerns the declared action and target. An operation whose denial can
+//! concern a second resource or a version-specific action (a copy source, a versioned read) must
+//! therefore be decided by the handler, never refused by the early gate: legacy records those with
+//! the source bucket and object, and with `s3:GetObjectVersion`.
+//!
+//! # What may not reach a record
+//!
+//! The record is built field by field from the event's `action`, `bucket`, `key`, `request_id`,
+//! `decision` and the verified identity's access key id. Nothing is copied from `resources`,
+//! `subject`, `auth_scheme`, a version id or a copy-source identity, and nothing is formatted from a
+//! `Debug` implementation. The event carries no secret key, session token or signature.
+
+use rustfs_gateway::{AuthzAuditEvent, AuthzAuditSink, AuthzStage, Decision};
+
+/// `tags.denialReason` of a `Deny` the gateway enforced. The event carries the decision, not one of
+/// the legacy reasons (`iam_explicit_deny`, `no_policy_allows_action`, ...).
+pub const DENY_REASON: &str = "authorization_denied";
+/// `tags.denialReason` of an `Indeterminate` the gateway enforced: no policy reading could answer.
+pub const INDETERMINATE_REASON: &str = "authorization_indeterminate";
+
+/// What the legacy denial rate limiter answered for one refusal.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum DenialAdmission {
+    /// Record it; `suppressed` refusals of the same caller class were dropped since the previous
+    /// record, and are reported as `tags.suppressedDenials` when non-zero.
+    Emit {
+        /// Refusals dropped since the previous record of this caller class.
+        suppressed: u64,
+    },
+    /// Drop it.
+    Suppress,
+}
+
+/// One refusal the gateway enforced, in the fields of the legacy authorization-denial entry the
+/// event can supply.
+///
+/// The binary builds the entry as legacy `authorization_denial_entry` (`rustfs/src/storage/helper.rs`)
+/// does: `version` `1.0`, `event` `AuditS3Operation`, `trigger` and `api.name` [`Self::action`],
+/// `api.status` `failure`, `api.status_code` `403`, `error` `Access Denied`, `tags.denialReason`
+/// [`Self::denial_reason`], and `tags.suppressedDenials` only when [`Self::suppressed_denials`] is
+/// non-zero.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct GatewayDenialRecord {
+    /// `trigger` and `api.name`: the operation's declared IAM action, `s3:GetObject`.
+    pub action: String,
+    /// `api.bucket`, when the request addressed one.
+    pub bucket: Option<String>,
+    /// `api.object`, when the request addressed one.
+    pub object: Option<String>,
+    /// `requestID`: the identifier the caller received.
+    pub request_id: String,
+    /// `accessKey`: the verified caller; `None` for an anonymous caller, as legacy omits it.
+    pub access_key: Option<String>,
+    /// `tags.denialReason`: [`DENY_REASON`] or [`INDETERMINATE_REASON`].
+    pub denial_reason: &'static str,
+    /// `tags.suppressedDenials`, from [`DenialAdmission::Emit`].
+    pub suppressed_denials: u64,
+}
+
+/// Where admitted refusals go. Implemented by the binary over the legacy denial auditor.
+///
+/// The three methods are called in order, at most once each per refusal, on the request's task,
+/// and must neither block nor panic.
+///
+/// # What the implementation owes the legacy entry
+///
+/// - [`Self::enabled`] is the legacy audit switch (`is_audit_module_enabled`), and [`Self::admit`]
+///   the process-wide legacy `DenialRateLimiter` itself, not a second instance: gateway and handler
+///   refusals share one token bucket per caller class.
+/// - `time`, `parentUser` and `requestClaims` are filled as legacy fills them.
+/// - `remotehost`, `userAgent`, `requestHost`, `requestPath` and `requestQuery` come from the
+///   request the outer layer saw. **`requestQuery` is not the URI query.** Legacy fills it with
+///   `rustfs_targets::net::extract_params_header` over the request *headers*, which replaces the
+///   value of every sensitive header (`authorization`, `x-amz-security-token`, the SSE-C key
+///   headers, ...) with the redaction marker. The raw URI query must never be put there: on a
+///   presigned request it carries `X-Amz-Signature`, `X-Amz-Credential` and
+///   `X-Amz-Security-Token`.
+pub trait DenialAuditSink: Send + Sync + 'static {
+    /// Whether denial auditing is switched on. Asked first; `false` drops the refusal before the
+    /// rate limiter is consulted.
+    fn enabled(&self) -> bool;
+    /// Asks the rate limiter for one refusal of this caller class. Asked before any record exists.
+    fn admit(&self, anonymous: bool) -> DenialAdmission;
+    /// Receives one admitted record.
+    fn record(&self, record: GatewayDenialRecord);
+}
+
+/// An [`AuthzAuditSink`] recording only the refusals the gateway enforced.
+pub struct RustfsAuthzAudit<S> {
+    sink: S,
+}
+
+impl<S: DenialAuditSink> RustfsAuthzAudit<S> {
+    /// An audit sink reporting into `sink`.
+    #[must_use]
+    pub fn new(sink: S) -> Self {
+        Self { sink }
+    }
+}
+
+impl<S: DenialAuditSink> AuthzAuditSink for RustfsAuthzAudit<S> {
+    fn on_decision(&self, event: &AuthzAuditEvent<'_>) {
+        let denial_reason = match event.decision {
+            Decision::Allow => return,
+            Decision::Deny => DENY_REASON,
+            Decision::Indeterminate => INDETERMINATE_REASON,
+        };
+        if is_visibility_probe(event) || !self.sink.enabled() {
+            return;
+        }
+        let anonymous = event.identity.is_none();
+        let DenialAdmission::Emit { suppressed } = self.sink.admit(anonymous) else {
+            return;
+        };
+        self.sink.record(GatewayDenialRecord {
+            action: event.action.to_owned(),
+            bucket: event.bucket.map(|bucket| bucket.as_str().to_owned()),
+            object: event.key.map(|key| key.as_str().to_owned()),
+            request_id: event.request_id.as_str().to_owned(),
+            access_key: event.identity.map(|identity| identity.access_key_id().to_owned()),
+            denial_reason,
+            suppressed_denials: suppressed,
+        });
+    }
+}
+
+/// Whether `event` is the missing-object visibility probe, which refuses nothing. See the module
+/// documentation for why the first resource identifies it.
+fn is_visibility_probe(event: &AuthzAuditEvent<'_>) -> bool {
+    event.stage == AuthzStage::Input
+        && event
+            .resources
+            .first()
+            .is_some_and(|first| first.action != first.route_action)
+}
+
+#[cfg(test)]
+#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
+mod tests {
+    use super::*;
+    use core::time::Duration;
+    use rustfs_gateway::{
+        AuthSchemeRef, AuthzRequest, BucketName, Identity, ObjectKey, PolicySnapshot, RequestId, ResourceShape, SessionBinding,
+        Subject, TargetOrigin,
+    };
+    use std::sync::Mutex;
+
+    const ACCESS_KEY: &str = "AKIDCALLER0001";
+    const BUCKET: &str = "audit-bucket";
+    const KEY: &str = "reports/q3.csv";
+
+    /// A sink with a scripted switch and limiter that records every call it receives.
+    struct Probe {
+        enabled: bool,
+        admission: DenialAdmission,
+        switch_asked: Mutex<usize>,
+        limiter_asked: Mutex<Vec<bool>>,
+        records: Mutex<Vec<GatewayDenialRecord>>,
+    }
+
+    impl Probe {
+        fn new(enabled: bool, admission: DenialAdmission) -> Self {
+            Self {
+                enabled,
+                admission,
+                switch_asked: Mutex::new(0),
+                limiter_asked: Mutex::new(Vec::new()),
+                records: Mutex::new(Vec::new()),
+            }
+        }
+
+        fn open() -> Self {
+            Self::new(true, DenialAdmission::Emit { suppressed: 0 })
+        }
+
+        fn records(&self) -> Vec<GatewayDenialRecord> {
+            self.records.lock().expect("not poisoned").clone()
+        }
+
+        fn switch_asked(&self) -> usize {
+            *self.switch_asked.lock().expect("not poisoned")
+        }
+
+        fn limiter_asked(&self) -> Vec<bool> {
+            self.limiter_asked.lock().expect("not poisoned").clone()
+        }
+    }
+
+    impl DenialAuditSink for Probe {
+        fn enabled(&self) -> bool {
+            *self.switch_asked.lock().expect("not poisoned") += 1;
+            self.enabled
+        }
+
+        fn admit(&self, anonymous: bool) -> DenialAdmission {
+            self.limiter_asked.lock().expect("not poisoned").push(anonymous);
+            self.admission
+        }
+
+        fn record(&self, record: GatewayDenialRecord) {
+            self.records.lock().expect("not poisoned").push(record);
+        }
+    }
+
+    struct Fixture {
+        request_id: RequestId,
+        bucket: BucketName,
+        key: ObjectKey,
+        identity: Identity,
+        snapshot: PolicySnapshot,
+    }
+
+    fn fixture() -> Fixture {
+        Fixture {
+            request_id: RequestId::from_bits(0x5EED),
+            bucket: BucketName::new(BUCKET).expect("a valid bucket"),
+            key: ObjectKey::new(KEY).expect("a valid key"),
+            identity: Identity::new(ACCESS_KEY).expect("a valid access key"),
+            snapshot: PolicySnapshot::empty(),
+        }
+    }
+
+    /// The route target of a GetObject, as the gateway builds it: its action is its route action.
+    fn target<'a>(f: &'a Fixture, identity: Option<&'a Identity>) -> AuthzRequest<'a> {
+        AuthzRequest {
+            operation: "GetObject",
+            action: "s3:GetObject",
+            resource: ResourceShape::Object,
+            bucket: Some(&f.bucket),
+            key: Some(&f.key),
+            copy_source_identity: None,
+            version_id: None,
+            route_action: "s3:GetObject",
+            route_bucket: Some(&f.bucket),
+            route_key: Some(&f.key),
+            identity,
+            target_origin: TargetOrigin::Path,
+            subject: None,
+        }
+    }
+
+    /// The visibility question, built from the route target as `InputAuthzRequest::new` builds it.
+    fn visibility<'a>(route: &AuthzRequest<'a>) -> AuthzRequest<'a> {
+        AuthzRequest {
+            action: "s3:ListBucket",
+            resource: ResourceShape::Bucket,
+            copy_source_identity: None,
+            version_id: None,
+            ..*route
+        }
+    }
+
+    /// A route or input event of a GetObject about `resources`.
+    fn event<'a>(
+        f: &'a Fixture,
+        stage: AuthzStage,
+        decision: Decision,
+        resources: &'a [AuthzRequest<'a>],
+        identity: Option<&'a Identity>,
+    ) -> AuthzAuditEvent<'a> {
+        AuthzAuditEvent {
+            request_id: &f.request_id,
+            stage,
+            operation: "GetObject",
+            action: "s3:GetObject",
+            resource: ResourceShape::Object,
+            bucket: Some(&f.bucket),
+            key: Some(&f.key),
+            resources,
+            auth_scheme: if identity.is_some() {
+                AuthSchemeRef::SigV4Header
+            } else {
+                AuthSchemeRef::Anonymous
+            },
+            identity,
+            target_origin: TargetOrigin::Path,
+            policy_snapshot: Some(f.snapshot.id()),
+            decision,
+            elapsed: Duration::ZERO,
+        }
+    }
+
+    /// The visibility event, as `emit_input_safely` builds it from the input event.
+    fn visibility_event<'a>(
+        input: &AuthzAuditEvent<'a>,
+        request: &'a AuthzRequest<'a>,
+        decision: Decision,
+    ) -> AuthzAuditEvent<'a> {
+        AuthzAuditEvent {
+            stage: AuthzStage::Input,
+            action: request.action,
+            resource: request.resource,
+            bucket: request.bucket,
+            key: request.key,
+            resources: std::slice::from_ref(request),
+            decision,
+            ..*input
+        }
+    }
+
+    /// Drives the events of one GetObject through `audit` in the gateway's order: the route stage,
+    /// then (when the route allowed) the input stage followed by its visibility question.
+    fn get_object(
+        audit: &RustfsAuthzAudit<Probe>,
+        f: &Fixture,
+        identity: Option<&Identity>,
+        route: Decision,
+        input: Decision,
+        probe: Decision,
+    ) {
+        let route_target = target(f, identity);
+        let asked = [route_target];
+        audit.on_decision(&event(f, AuthzStage::Route, route, &asked, identity));
+        if route != Decision::Allow {
+            return;
+        }
+        let input_event = event(f, AuthzStage::Input, input, &asked, identity);
+        audit.on_decision(&input_event);
+        let question = visibility(&route_target);
+        audit.on_decision(&visibility_event(&input_event, &question, probe));
+    }
+
+    fn expected(denial_reason: &'static str, access_key: Option<&str>, suppressed_denials: u64) -> GatewayDenialRecord {
+        GatewayDenialRecord {
+            action: "s3:GetObject".to_owned(),
+            bucket: Some(BUCKET.to_owned()),
+            object: Some(KEY.to_owned()),
+            request_id: RequestId::from_bits(0x5EED).as_str().to_owned(),
+            access_key: access_key.map(str::to_owned),
+            denial_reason,
+            suppressed_denials,
+        }
+    }
+
+    /// Positive — a route-stage refusal is one record carrying the legacy denial entry's values,
+    /// and `Indeterminate` keeps a reason of its own.
+    #[test]
+    fn p_route_refusal_is_one_record_with_the_legacy_values() {
+        for (decision, reason) in [(Decision::Deny, DENY_REASON), (Decision::Indeterminate, INDETERMINATE_REASON)] {
+            let f = fixture();
+            let audit = RustfsAuthzAudit::new(Probe::open());
+            get_object(&audit, &f, Some(&f.identity), decision, Decision::Allow, Decision::Allow);
+            assert_eq!(audit.sink.records(), [expected(reason, Some(ACCESS_KEY), 0)], "{decision:?}");
+            assert_eq!(audit.sink.limiter_asked(), [false], "{decision:?}");
+        }
+    }
+
+    /// Positive — an input-stage refusal of a GetObject is exactly one record, though its
+    /// visibility question was refused as well.
+    #[test]
+    fn p_enforcing_input_refusal_is_recorded_once() {
+        let f = fixture();
+        let audit = RustfsAuthzAudit::new(Probe::open());
+        get_object(&audit, &f, Some(&f.identity), Decision::Allow, Decision::Deny, Decision::Deny);
+        assert_eq!(audit.sink.records(), [expected(DENY_REASON, Some(ACCESS_KEY), 0)]);
+        assert_eq!(audit.sink.limiter_asked(), [false]);
+    }
+
+    /// Positive — the refusals the limiter dropped before this one are carried on the record.
+    #[test]
+    fn p_suppressed_count_is_carried() {
+        let f = fixture();
+        let audit = RustfsAuthzAudit::new(Probe::new(true, DenialAdmission::Emit { suppressed: 7 }));
+        get_object(&audit, &f, Some(&f.identity), Decision::Deny, Decision::Allow, Decision::Allow);
+        assert_eq!(audit.sink.records(), [expected(DENY_REASON, Some(ACCESS_KEY), 7)]);
+    }
+
+    /// Negative — an `Allow` at either stage records nothing and asks neither the switch nor the
+    /// limiter, so it cannot spend a token a real refusal needs.
+    #[test]
+    fn n_allow_is_never_recorded() {
+        let f = fixture();
+        let audit = RustfsAuthzAudit::new(Probe::open());
+        let asked = [target(&f, Some(&f.identity))];
+        for stage in [AuthzStage::Route, AuthzStage::Input] {
+            audit.on_decision(&event(&f, stage, Decision::Allow, &asked, Some(&f.identity)));
+        }
+        assert!(audit.sink.records().is_empty());
+        assert_eq!((audit.sink.switch_asked(), audit.sink.limiter_asked()), (0, Vec::new()));
+    }
+
+    /// Negative — a GetObject that succeeds for a caller refused `s3:ListBucket` records nothing:
+    /// the visibility question refuses no request, whatever it answered.
+    #[test]
+    fn n_visibility_probe_is_never_recorded() {
+        for probe in [Decision::Deny, Decision::Indeterminate] {
+            let f = fixture();
+            let audit = RustfsAuthzAudit::new(Probe::open());
+            get_object(&audit, &f, Some(&f.identity), Decision::Allow, Decision::Allow, probe);
+            assert!(audit.sink.records().is_empty(), "{probe:?}");
+            assert_eq!((audit.sink.switch_asked(), audit.sink.limiter_asked()), (0, Vec::new()), "{probe:?}");
+        }
+    }
+
+    /// Negative — with the audit switch off a refusal records nothing and the limiter is never
+    /// asked, as legacy checks the switch first.
+    #[test]
+    fn n_audit_switch_off_records_nothing() {
+        let f = fixture();
+        let audit = RustfsAuthzAudit::new(Probe::new(false, DenialAdmission::Emit { suppressed: 0 }));
+        get_object(&audit, &f, Some(&f.identity), Decision::Deny, Decision::Allow, Decision::Allow);
+        assert!(audit.sink.records().is_empty());
+        assert_eq!((audit.sink.switch_asked(), audit.sink.limiter_asked()), (1, Vec::new()));
+    }
+
+    /// Negative — a refusal the limiter suppresses records nothing.
+    #[test]
+    fn n_rate_limiter_suppression_records_nothing() {
+        let f = fixture();
+        let audit = RustfsAuthzAudit::new(Probe::new(true, DenialAdmission::Suppress));
+        get_object(&audit, &f, Some(&f.identity), Decision::Deny, Decision::Allow, Decision::Allow);
+        assert!(audit.sink.records().is_empty());
+        assert_eq!(audit.sink.limiter_asked(), [false]);
+    }
+
+    /// Negative — an anonymous caller has no access key, not even an empty one, and is limited as
+    /// the anonymous class; the authenticated refusal beside it is limited as the other class.
+    #[test]
+    fn n_anonymous_caller_has_no_access_key() {
+        let f = fixture();
+        let audit = RustfsAuthzAudit::new(Probe::open());
+        get_object(&audit, &f, None, Decision::Deny, Decision::Allow, Decision::Allow);
+        get_object(&audit, &f, Some(&f.identity), Decision::Deny, Decision::Allow, Decision::Allow);
+        assert_eq!(
+            audit.sink.records(),
+            [expected(DENY_REASON, None, 0), expected(DENY_REASON, Some(ACCESS_KEY), 0)]
+        );
+        assert_eq!(audit.sink.limiter_asked(), [true, false]);
+    }
+
+    /// Negative — values planted in the event's `resources`, version ids, subject, auth scheme,
+    /// target origin and session binding leave the record exactly as it is without them, and none
+    /// of their text reaches its `Debug` output.
+    #[test]
+    fn n_event_fields_outside_the_record_do_not_reach_it() {
+        const SECRETS: [&str; 3] = [
+            "wJalrXUtnFEMI/K7MDENG/bPxRfiCYSECRETKEY",
+            "AQoDYXdzEJrSESSIONTOKEN",
+            "fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024",
+        ];
+        const PLANTED_BUCKET: &str = "planted-source-bucket";
+        const PLANTED_KEY: &str = "planted/source-key";
+        let f = fixture();
+        let source_bucket = BucketName::new(PLANTED_BUCKET).expect("a valid bucket");
+        let source_key = ObjectKey::new(PLANTED_KEY).expect("a valid key");
+        let binding = SessionBinding::new(SECRETS[1], 4_000_000_000).expect("a binding");
+        let session = Identity::new(ACCESS_KEY).expect("a key").with_session_binding(binding);
+        let subject = Subject::Caller;
+        let planted_target = AuthzRequest {
+            version_id: Some(SECRETS[0]),
+            target_origin: TargetOrigin::Host,
+            subject: Some(&subject),
+            ..target(&f, Some(&session))
+        };
+        let derived = AuthzRequest {
+            action: "s3:GetObjectVersion",
+            bucket: Some(&source_bucket),
+            key: Some(&source_key),
+            version_id: Some(SECRETS[2]),
+            ..planted_target
+        };
+        let planted_resources = [planted_target, derived];
+        let plain_resources = [target(&f, Some(&f.identity))];
+        for stage in [AuthzStage::Route, AuthzStage::Input] {
+            let plain = RustfsAuthzAudit::new(Probe::open());
+            plain.on_decision(&event(&f, stage, Decision::Deny, &plain_resources, Some(&f.identity)));
+            let planted = RustfsAuthzAudit::new(Probe::open());
+            planted.on_decision(&AuthzAuditEvent {
+                auth_scheme: AuthSchemeRef::SigV4Presigned,
+                target_origin: TargetOrigin::Host,
+                ..event(&f, stage, Decision::Deny, &planted_resources, Some(&session))
+            });
+            let records = planted.sink.records();
+            assert_eq!(records, plain.sink.records(), "{stage:?}");
+            assert_eq!(records.len(), 1, "{stage:?}");
+            let rendered = format!("{records:?}");
+            for planted_text in SECRETS.into_iter().chain([
+                PLANTED_BUCKET,
+                PLANTED_KEY,
+                "GetObjectVersion",
+                "Presigned",
+                "Host",
+                "Caller",
+                "Authz",
+            ]) {
+                assert!(!rendered.contains(planted_text), "{planted_text} reached the record: {rendered}");
+            }
+        }
+    }
+}

crates/gateway-adapter/src/bucket_owner.rs

@@ -0,0 +1,215 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! The gateway's [`BucketOwnerSource`]: the one owner RustFS reports, for a bucket that exists.
+//!
+//! Responsible for: confirming through a [`BucketLookup`] that a bucket exists, answering the
+//! single owner id the deployment reports ([`RustfsBucketOwners::new`]) for it, and turning every
+//! outcome that is not a usable owner into the gateway's one refusal, [`BucketOwnerError`].
+//! NOT responsible for: choosing the owner id (the binary passes the constant it already reports
+//! as the owner in listings), comparing it with `x-amz-expected-bucket-owner` (the gateway does,
+//! byte for byte), or authorizing anything.
+//! Upstream: the gateway's expected-owner check, which consults this source only when a request
+//! carries the header, after authorization allowed it. Downstream: the lookup the binary supplies
+//! over the ecstore bucket metadata system.
+//!
+//! # What an owner is in RustFS
+//!
+//! RustFS stores no per-bucket owner: bucket ACL writes are no-ops and nothing writes the ACL
+//! document, so the only owner RustFS reports anywhere (ListBuckets, ACL and multipart listings)
+//! is one constant. The source answers exactly that constant. A caller therefore passes the
+//! gateway's check only by presenting the owner id RustFS lists, which separates no tenants; it is
+//! a compatibility answer, and the constant's own documentation says it must not decide
+//! authorization.
+//!
+//! # Difference from legacy (ledger entry R-071)
+//!
+//! Legacy RustFS ignores `x-amz-expected-bucket-owner` entirely. On the gateway stack a request
+//! carrying it is served only when the value equals the reported owner id; otherwise it is `403
+//! AccessDenied`. A missing bucket and a backend failure with the header present are also `403`
+//! at the pinned gateway, where legacy answers `404` and `5xx`: [`BucketOwnerError`] has one value,
+//! so it cannot express either. [`RustfsBucketOwners::resolve`] keeps the two apart at this
+//! adapter's boundary so the gateway can surface them once it grows the classes.
+
+use std::sync::Arc;
+
+use rustfs_gateway::{BoxFuture, BucketName, BucketOwnerError, BucketOwnerSource};
+
+/// Why a bucket yielded no owner.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum OwnerLookupError {
+    /// The bucket does not exist.
+    NoSuchBucket,
+    /// The metadata backend failed, could not be reached, or the owner is not usable (an empty
+    /// owner id is a misconfiguration), so the bucket's state is unknown.
+    Backend,
+}
+
+/// The narrow slice of RustFS bucket metadata an owner answer needs: whether a bucket exists.
+///
+/// Exists so the source can be tested against a fake. The binary implements it over the ecstore
+/// bucket metadata system, mapping `BucketNotFound` to [`OwnerLookupError::NoSuchBucket`] and every
+/// other error to [`OwnerLookupError::Backend`].
+pub trait BucketLookup: Send + Sync + 'static {
+    /// `Ok(())` when `bucket` exists.
+    fn check<'a>(&'a self, bucket: &'a str) -> BoxFuture<'a, Result<(), OwnerLookupError>>;
+}
+
+/// A [`BucketOwnerSource`] answering one owner id for every bucket that exists.
+pub struct RustfsBucketOwners<L> {
+    lookup: L,
+    owner: Arc<str>,
+}
+
+impl<L: BucketLookup> RustfsBucketOwners<L> {
+    /// A source answering `owner` for every bucket `lookup` finds.
+    ///
+    /// An empty `owner` is no owner: every answer is then [`OwnerLookupError::Backend`].
+    #[must_use]
+    pub fn new(lookup: L, owner: impl Into<Arc<str>>) -> Self {
+        Self {
+            lookup,
+            owner: owner.into(),
+        }
+    }
+
+    /// The owner of `bucket`, or why there is none. Missing and failing stay distinct here.
+    ///
+    /// # Errors
+    ///
+    /// [`OwnerLookupError::NoSuchBucket`] when the bucket does not exist;
+    /// [`OwnerLookupError::Backend`] on any lookup failure or when the configured owner is empty.
+    pub async fn resolve(&self, bucket: &str) -> Result<Arc<str>, OwnerLookupError> {
+        self.lookup.check(bucket).await?;
+        if self.owner.is_empty() {
+            return Err(OwnerLookupError::Backend);
+        }
+        Ok(Arc::clone(&self.owner))
+    }
+}
+
+impl<L: BucketLookup> BucketOwnerSource for RustfsBucketOwners<L> {
+    fn owner<'a>(&'a self, bucket: &'a BucketName) -> BoxFuture<'a, Result<Arc<str>, BucketOwnerError>> {
+        Box::pin(async move {
+            // The pinned gateway collapses both classes (R-071), so both are the one refusal.
+            self.resolve(bucket.as_str())
+                .await
+                .map_err(|_: OwnerLookupError| BucketOwnerError::unavailable())
+        })
+    }
+}
+
+#[cfg(test)]
+#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
+mod tests {
+    use super::*;
+    use std::sync::Mutex;
+
+    /// Stands for the owner id RustFS reports; the binary passes its own constant.
+    const OWNER: &str = "reported-owner-id";
+
+    struct Fake {
+        answer: Result<(), OwnerLookupError>,
+        asked: Mutex<Vec<String>>,
+    }
+
+    impl Fake {
+        fn answering(answer: Result<(), OwnerLookupError>) -> Self {
+            Self {
+                answer,
+                asked: Mutex::new(Vec::new()),
+            }
+        }
+    }
+
+    impl BucketLookup for Fake {
+        fn check<'a>(&'a self, bucket: &'a str) -> BoxFuture<'a, Result<(), OwnerLookupError>> {
+            self.asked.lock().unwrap().push(bucket.to_owned());
+            let answer = self.answer;
+            Box::pin(async move { answer })
+        }
+    }
+
+    fn source(answer: Result<(), OwnerLookupError>) -> RustfsBucketOwners<Fake> {
+        RustfsBucketOwners::new(Fake::answering(answer), OWNER)
+    }
+
+    async fn gateway_answer(answer: Result<(), OwnerLookupError>) -> Result<Arc<str>, BucketOwnerError> {
+        source(answer).owner(&BucketName::new("photos").expect("a bucket name")).await
+    }
+
+    /// Positive: an existing bucket's owner is the configured id, byte for byte.
+    #[tokio::test]
+    async fn an_existing_bucket_has_the_configured_owner() {
+        assert_eq!(gateway_answer(Ok(())).await.as_deref(), Ok(OWNER));
+    }
+
+    /// Positive: every bucket gets the same owner, since RustFS stores none per bucket.
+    #[tokio::test]
+    async fn every_bucket_has_the_same_owner() {
+        let source = source(Ok(()));
+        assert_eq!(source.resolve("photos").await.as_deref(), Ok(OWNER));
+        assert_eq!(source.resolve("backups").await.as_deref(), Ok(OWNER));
+    }
+
+    /// Positive: the lookup is asked about exactly the bucket the gateway named.
+    #[tokio::test]
+    async fn the_lookup_is_asked_about_the_named_bucket() {
+        let source = source(Ok(()));
+        source
+            .owner(&BucketName::new("photos").expect("a bucket name"))
+            .await
+            .expect("an owner");
+        assert_eq!(*source.lookup.asked.lock().unwrap(), vec!["photos".to_owned()]);
+    }
+
+    /// Negative: a missing bucket is reported as missing at the adapter boundary.
+    #[tokio::test]
+    async fn a_missing_bucket_is_reported_as_missing() {
+        assert_eq!(
+            source(Err(OwnerLookupError::NoSuchBucket)).resolve("photos").await,
+            Err(OwnerLookupError::NoSuchBucket)
+        );
+    }
+
+    /// Negative: a failing backend is reported as a failure, never as an owner and never as missing.
+    #[tokio::test]
+    async fn a_backend_failure_is_reported_as_a_failure() {
+        assert_eq!(
+            source(Err(OwnerLookupError::Backend)).resolve("photos").await,
+            Err(OwnerLookupError::Backend)
+        );
+    }
+
+    /// Negative: the pinned gateway sees one refusal for both, and never an owner.
+    #[tokio::test]
+    async fn the_gateway_is_refused_in_both_cases() {
+        assert_eq!(
+            gateway_answer(Err(OwnerLookupError::NoSuchBucket)).await,
+            Err(BucketOwnerError::unavailable())
+        );
+        assert_eq!(gateway_answer(Err(OwnerLookupError::Backend)).await, Err(BucketOwnerError::unavailable()));
+    }
+
+    /// Negative: an empty configured owner is no owner; it must not equal an empty assertion.
+    #[tokio::test]
+    async fn an_empty_owner_is_refused() {
+        let source = RustfsBucketOwners::new(Fake::answering(Ok(())), "");
+        assert_eq!(source.resolve("photos").await, Err(OwnerLookupError::Backend));
+        assert_eq!(
+            source.owner(&BucketName::new("photos").expect("a bucket name")).await,
+            Err(BucketOwnerError::unavailable())
+        );
+    }
+}

crates/gateway-adapter/src/credentials.rs

@@ -0,0 +1,446 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! The gateway's [`CredentialProvider`], answered by RustFS IAM.
+//!
+//! Responsible for: resolving one access key id to the secret the gateway verifies a signature
+//! with, and classifying every other outcome as either a business result
+//! ([`CredentialLookup::NotFound`], eligible for the gateway's negative cache) or an operational
+//! fault ([`ProviderError`], counted and never cached).
+//! NOT responsible for: authorization of any kind, session-token claim validation, or deciding
+//! what the client is answered with. The gateway answers every unusable credential identically.
+//!
+//! # Secret handling
+//!
+//! Secrets travel only inside the gateway's [`Credentials`], whose `Debug` redacts them. No type
+//! in this module derives or implements `Debug`, and no error produced here carries text: the
+//! IAM error is classified and dropped. A secret that fails the crate's effective-length rule
+//! (`crate::MIN_SECRET_KEY_LEN` bytes once trailing NULs are removed) is never handed to the
+//! gateway, whatever the lookup says about its validity.
+
+use std::sync::Arc;
+
+use rustfs_gateway::{BoxFuture, CredentialLookup, CredentialProvider, Credentials, ProviderError, SessionBinding};
+use rustfs_iam::error::Error as IamError;
+use rustfs_iam::store::Store;
+use rustfs_iam::sys::IamSys;
+use rustfs_policy::auth::{ACCOUNT_OFF, UserIdentity};
+
+use crate::root::{RootSource, is_usable_secret};
+
+/// The opaque issuer recorded in the [`rustfs_gateway::SessionBinding`] of every RustFS STS
+/// credential.
+pub const STS_SESSION_ISSUER: &str = "rustfs-sts";
+
+/// Access keys with this prefix belong to Keystone token authentication, which never verifies an
+/// AWS signature.
+const KEYSTONE_ACCESS_KEY_PREFIX: &str = "keystone:";
+
+/// The narrow slice of IAM that a credential lookup needs.
+///
+/// Exists so the adapter can be tested against a fake: IAM exports no in-memory backend.
+/// Implemented for [`IamSys`], whose `check_key` loads a user from storage on a cache miss.
+pub trait IamKeyLookup: Send + Sync + 'static {
+    /// Whether the IAM store has finished loading.
+    fn is_ready(&self) -> bool;
+
+    /// Resolves an access key, with [`IamSys::check_key`]'s contract: `Ok((None, _))` for an
+    /// unknown key, `Ok((Some(identity), valid))` otherwise, where `valid` is false for a
+    /// disabled, expired, or malformed credential.
+    fn check_key<'a>(&'a self, access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>>;
+}
+
+impl<T: Store> IamKeyLookup for IamSys<T> {
+    fn is_ready(&self) -> bool {
+        IamSys::is_ready(self)
+    }
+
+    fn check_key<'a>(&'a self, access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
+        Box::pin(IamSys::check_key(self, access_key))
+    }
+}
+
+/// A [`CredentialProvider`] over RustFS IAM and the live root credentials.
+///
+/// Built only by [`crate::IamAdapters::new`], so it always reads the same [`RootSource`] as the
+/// policy source it was assembled with.
+pub struct IamCredentials {
+    iam: Arc<dyn IamKeyLookup>,
+    root: Arc<dyn RootSource>,
+}
+
+impl IamCredentials {
+    pub(crate) fn new(iam: Arc<dyn IamKeyLookup>, root: Arc<dyn RootSource>) -> Self {
+        Self { iam, root }
+    }
+
+    async fn resolve(&self, access_key: &str) -> Result<CredentialLookup, ProviderError> {
+        // Deliberate change from the legacy s3s path (`rustfs/src/auth.rs` `get_secret_key`),
+        // which answered a `keystone:` key with an empty secret. Keystone authenticates by token,
+        // so there is no AWS secret to verify a signature against: the key is unknown here.
+        if access_key.starts_with(KEYSTONE_ACCESS_KEY_PREFIX) {
+            return Ok(CredentialLookup::NotFound);
+        }
+
+        // Legacy answers every key with an internal error while the root credentials are not
+        // initialized; without them no key can be classified as root or not.
+        let Some(root) = self.root.current() else {
+            return Err(ProviderError::Unavailable);
+        };
+        // Byte-exact, as the trait requires: the access key id is signed material.
+        if root.access_key() == access_key {
+            return Ok(CredentialLookup::Found(root.credentials().clone_credentials()));
+        }
+
+        if !self.iam.is_ready() {
+            return Err(ProviderError::Unavailable);
+        }
+
+        match self.iam.check_key(access_key).await {
+            // IAM refuses such a secret through `valid`, but this provider does not rely on every
+            // lookup doing so: a short secret is never usable, so it is not a credential at all.
+            Ok((Some(identity), _)) if !is_usable_secret(identity.credentials.secret_key.as_bytes()) => {
+                Ok(CredentialLookup::NotFound)
+            }
+            Ok((Some(identity), valid)) => to_gateway_credentials(access_key, &identity, valid).map(CredentialLookup::Found),
+            Ok((None, _)) => Ok(CredentialLookup::NotFound),
+            Err(error) => classify_iam_error(&error),
+        }
+    }
+}
+
+/// Classifies an IAM failure without keeping any of its text.
+fn classify_iam_error(error: &IamError) -> Result<CredentialLookup, ProviderError> {
+    match error {
+        IamError::NoSuchUser(_)
+        | IamError::NoSuchServiceAccount(_)
+        | IamError::NoSuchTempAccount(_)
+        | IamError::NoSuchAccount(_) => Ok(CredentialLookup::NotFound),
+        IamError::IamSysNotInitialized => Err(ProviderError::Unavailable),
+        _ => Err(ProviderError::Backend),
+    }
+}
+
+/// Builds the gateway credential for one IAM identity. No authorization happens here: a
+/// disabled or invalid principal still resolves, marked disabled, and the gateway refuses it
+/// exactly as it refuses an unknown key.
+fn to_gateway_credentials(access_key: &str, identity: &UserIdentity, valid: bool) -> Result<Credentials, ProviderError> {
+    let cred = &identity.credentials;
+    let mut credentials = Credentials::new(access_key, cred.secret_key.as_bytes()).map_err(|_| ProviderError::Backend)?;
+
+    // A service account carries a claims token internally, but its clients never present it, so
+    // it authenticates as a long-term credential (as `check_session_token` treats it in legacy).
+    if !cred.session_token.is_empty() && !cred.is_service_account() {
+        // A temporary credential without an expiry cannot become a gateway session, which always
+        // ends. Refusing it as a fault is the fail-closed answer; a never-ending session is not.
+        let expiration = cred.expiration.ok_or(ProviderError::Backend)?;
+        let binding = SessionBinding::new(STS_SESSION_ISSUER, expiration.unix_timestamp()).map_err(|_| ProviderError::Backend)?;
+        credentials = credentials
+            .with_session(&cred.session_token, binding)
+            .map_err(|_| ProviderError::Backend)?;
+    }
+
+    if !valid || cred.status == ACCOUNT_OFF {
+        credentials = credentials.disable();
+    }
+    Ok(credentials)
+}
+
+impl CredentialProvider for IamCredentials {
+    fn lookup<'a>(&'a self, access_key_id: &'a str) -> BoxFuture<'a, Result<CredentialLookup, ProviderError>> {
+        Box::pin(self.resolve(access_key_id))
+    }
+}
+
+#[cfg(test)]
+#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
+mod tests {
+    use super::*;
+    use crate::root::{RootCell, RootCredentials};
+    use rustfs_credentials::Credentials as IamCredential;
+    use rustfs_gateway::RequestNow;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use time::OffsetDateTime;
+
+    const ACCESS_KEY: &str = "AKIDUSER0001";
+    const SECRET: &str = "user-secret-key-0001";
+
+    type Answer = Box<dyn Fn() -> Result<(Option<UserIdentity>, bool), IamError> + Send + Sync>;
+
+    struct FakeIam {
+        ready: bool,
+        answer: Answer,
+        calls: AtomicUsize,
+    }
+
+    impl IamKeyLookup for FakeIam {
+        fn is_ready(&self) -> bool {
+            self.ready
+        }
+
+        fn check_key<'a>(&'a self, _access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
+            self.calls.fetch_add(1, Ordering::SeqCst);
+            let answer = (self.answer)();
+            Box::pin(async move { answer })
+        }
+    }
+
+    fn fake(
+        ready: bool,
+        answer: impl Fn() -> Result<(Option<UserIdentity>, bool), IamError> + Send + Sync + 'static,
+    ) -> Arc<FakeIam> {
+        Arc::new(FakeIam {
+            ready,
+            answer: Box::new(answer),
+            calls: AtomicUsize::new(0),
+        })
+    }
+
+    fn user(status: &str) -> IamCredential {
+        IamCredential {
+            access_key: ACCESS_KEY.to_owned(),
+            secret_key: SECRET.to_owned(),
+            status: status.to_owned(),
+            ..Default::default()
+        }
+    }
+
+    fn found(cred: IamCredential, valid: bool) -> Result<(Option<UserIdentity>, bool), IamError> {
+        Ok((Some(UserIdentity::new(cred)), valid))
+    }
+
+    fn root_cell() -> Arc<RootCell> {
+        let root = RootCredentials::new("rootadmin", "root-secret-key", "sts-signing-key").expect("valid root credentials");
+        Arc::new(RootCell::new(Some(root)))
+    }
+
+    async fn lookup(iam: Arc<FakeIam>, access_key: &str) -> Result<CredentialLookup, ProviderError> {
+        IamCredentials::new(iam, root_cell()).lookup(access_key).await
+    }
+
+    fn expect_found(result: Result<CredentialLookup, ProviderError>) -> Credentials {
+        match result {
+            Ok(CredentialLookup::Found(credentials)) => credentials,
+            Ok(CredentialLookup::NotFound) => panic!("expected Found, got NotFound"),
+            Err(error) => panic!("expected Found, got {error:?}"),
+        }
+    }
+
+    fn is_not_found(result: &Result<CredentialLookup, ProviderError>) -> bool {
+        matches!(result, Ok(CredentialLookup::NotFound))
+    }
+
+    /// Positive — an enabled user resolves to exactly its secret, as a long-term credential.
+    #[tokio::test]
+    async fn credentials_found_user_resolves_to_its_secret() {
+        let iam = fake(true, || found(user("on"), true));
+        let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
+        assert_eq!(credentials.secret().expose(), SECRET.as_bytes());
+        assert_eq!(credentials.identity().access_key_id(), ACCESS_KEY);
+        assert!(!credentials.is_disabled());
+        assert!(credentials.session_binding().is_none());
+    }
+
+    /// Negative — a key IAM does not know is a business result, not a fault.
+    #[tokio::test]
+    async fn credentials_unknown_key_is_not_found() {
+        let iam = fake(true, || Ok((None, false)));
+        assert!(is_not_found(&lookup(iam, "AKIDUNKNOWN").await));
+    }
+
+    /// Negative — every "no such principal" error IAM can surface is `NotFound`. Answering these
+    /// as `Backend` would keep them out of the negative cache and count every guessed key as a
+    /// backend fault.
+    #[tokio::test]
+    async fn credentials_no_such_principal_errors_are_not_found() {
+        let errors: [fn() -> IamError; 4] = [
+            || IamError::NoSuchUser(ACCESS_KEY.to_owned()),
+            || IamError::NoSuchServiceAccount(ACCESS_KEY.to_owned()),
+            || IamError::NoSuchTempAccount(ACCESS_KEY.to_owned()),
+            || IamError::NoSuchAccount(ACCESS_KEY.to_owned()),
+        ];
+        for error in errors {
+            let iam = fake(true, move || Err(error()));
+            let result = lookup(iam, ACCESS_KEY).await;
+            assert!(is_not_found(&result), "{:?} must be NotFound", error());
+        }
+    }
+
+    /// Negative — any other IAM failure is a backend fault, never `NotFound` and never `Found`.
+    #[tokio::test]
+    async fn credentials_backend_error_is_backend() {
+        let iam = fake(true, || Err(IamError::StringError("storage read failed".to_owned())));
+        assert!(matches!(lookup(iam, ACCESS_KEY).await, Err(ProviderError::Backend)));
+    }
+
+    /// Negative — a `keystone:` key never reaches IAM and never resolves to a secret.
+    ///
+    /// Deliberate change from the legacy s3s path, which answered such a key with an empty
+    /// secret: Keystone authenticates by token, so there is no AWS secret to verify against.
+    #[tokio::test]
+    async fn credentials_keystone_prefix_is_not_found_without_iam() {
+        let iam = fake(true, || found(user("on"), true));
+        let result = lookup(iam.clone(), "keystone:project-user").await;
+        assert!(is_not_found(&result));
+        assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
+    }
+
+    /// Negative — a switched-off or otherwise invalid user resolves, but disabled, and the same
+    /// user enabled does not.
+    #[tokio::test]
+    async fn credentials_disabled_user_is_disabled() {
+        let off = expect_found(lookup(fake(true, || found(user("off"), false)), ACCESS_KEY).await);
+        assert!(off.is_disabled());
+        assert!(off.admit(None, RequestNow::from_unix_seconds(0)).is_err());
+
+        let invalid = expect_found(lookup(fake(true, || found(user("on"), false)), ACCESS_KEY).await);
+        assert!(invalid.is_disabled());
+
+        let on = expect_found(lookup(fake(true, || found(user("on"), true)), ACCESS_KEY).await);
+        assert!(!on.is_disabled());
+    }
+
+    /// Positive — an STS credential carries its token and its lifetime as one session binding.
+    #[tokio::test]
+    async fn credentials_sts_carries_session_binding() {
+        let expires_at = 4_000_000_000_i64;
+        let iam = fake(true, move || {
+            let mut cred = user("on");
+            cred.session_token = "sts-session-token".to_owned();
+            cred.parent_user = "parent".to_owned();
+            cred.expiration = Some(OffsetDateTime::from_unix_timestamp(expires_at).expect("valid timestamp"));
+            found(cred, true)
+        });
+        let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
+        let binding = credentials.session_binding().expect("an STS credential has a session");
+        assert_eq!(binding.expires_at_unix_seconds(), expires_at);
+        assert_eq!(binding.issuer(), STS_SESSION_ISSUER);
+        let now = RequestNow::from_unix_seconds(expires_at - 1);
+        assert!(credentials.admit(Some(b"sts-session-token"), now).is_ok());
+        assert!(credentials.admit(None, now).is_err());
+    }
+
+    /// Negative — an STS credential with no expiry cannot be expressed as a session the gateway
+    /// would ever expire, so it is a backend fault rather than a session that never ends.
+    #[tokio::test]
+    async fn credentials_sts_without_expiry_is_backend() {
+        let iam = fake(true, || {
+            let mut cred = user("on");
+            cred.session_token = "sts-session-token".to_owned();
+            found(cred, true)
+        });
+        assert!(matches!(lookup(iam, ACCESS_KEY).await, Err(ProviderError::Backend)));
+    }
+
+    /// Positive — a service account carries a token internally but authenticates as a long-term
+    /// credential: clients never present that token.
+    #[tokio::test]
+    async fn credentials_service_account_is_long_term() {
+        let iam = fake(true, || {
+            let mut cred = user("on");
+            cred.session_token = "service-account-claims-token".to_owned();
+            cred.parent_user = "parent".to_owned();
+            cred.claims = Some([(rustfs_credentials::IAM_POLICY_CLAIM_NAME_SA.to_owned(), "inherited-policy".into())].into());
+            found(cred, true)
+        });
+        let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
+        assert!(credentials.session_binding().is_none());
+        assert!(credentials.admit(None, RequestNow::from_unix_seconds(0)).is_ok());
+    }
+
+    /// Negative — before IAM is ready a lookup is `Unavailable`, not `NotFound`: a key must not
+    /// be negatively cached because the store had not loaded yet.
+    #[tokio::test]
+    async fn credentials_iam_not_ready_is_unavailable() {
+        let iam = fake(false, || found(user("on"), true));
+        assert!(matches!(lookup(iam.clone(), ACCESS_KEY).await, Err(ProviderError::Unavailable)));
+        assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
+
+        let uninitialized = fake(true, || Err(IamError::IamSysNotInitialized));
+        assert!(matches!(lookup(uninitialized, ACCESS_KEY).await, Err(ProviderError::Unavailable)));
+    }
+
+    /// Positive — the injected root credentials resolve without IAM, even before it is ready, and
+    /// nothing else is answered with the root secret.
+    #[tokio::test]
+    async fn credentials_root_resolves_without_iam() {
+        let iam = fake(false, || found(user("on"), true));
+        let provider = IamCredentials::new(iam.clone(), root_cell());
+        let root = expect_found(provider.lookup("rootadmin").await);
+        assert_eq!(root.secret().expose(), b"root-secret-key");
+        assert!(matches!(provider.lookup("ROOTADMIN").await, Err(ProviderError::Unavailable)));
+        assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
+    }
+
+    /// Negative — a lookup that calls an identity valid but holds an empty or short secret is not
+    /// trusted: such a secret is not a credential, so the key is `NotFound`, never `Found`.
+    #[tokio::test]
+    async fn credentials_short_secret_is_never_found() {
+        for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0"] {
+            let iam = fake(true, move || {
+                let mut cred = user("on");
+                cred.secret_key = weak.to_owned();
+                found(cred, true)
+            });
+            let result = lookup(iam, ACCESS_KEY).await;
+            assert!(is_not_found(&result), "{weak:?} must be NotFound");
+        }
+        for usable in ["12345678", "abc\0defg"] {
+            let iam = fake(true, move || {
+                let mut cred = user("on");
+                cred.secret_key = usable.to_owned();
+                found(cred, true)
+            });
+            let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
+            assert_eq!(credentials.secret().expose(), usable.as_bytes());
+        }
+    }
+
+    /// Negative — a switched-off account is disabled even when the lookup calls it valid.
+    #[tokio::test]
+    async fn credentials_off_status_alone_disables() {
+        let off = expect_found(lookup(fake(true, || found(user("off"), true)), ACCESS_KEY).await);
+        assert!(off.is_disabled());
+        assert!(off.admit(None, RequestNow::from_unix_seconds(0)).is_err());
+    }
+
+    /// Negative — an STS credential admits only its own session's token: the token of another
+    /// session, or none, is refused.
+    #[tokio::test]
+    async fn credentials_sts_refuses_another_sessions_token() {
+        let expires_at = 4_000_000_000_i64;
+        let iam = fake(true, move || {
+            let mut cred = user("on");
+            cred.session_token = "session-token-a".to_owned();
+            cred.parent_user = "parent".to_owned();
+            cred.expiration = Some(OffsetDateTime::from_unix_timestamp(expires_at).expect("valid timestamp"));
+            found(cred, true)
+        });
+        let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
+        let now = RequestNow::from_unix_seconds(expires_at - 1);
+        assert!(credentials.admit(Some(b"session-token-b"), now).is_err());
+        assert!(credentials.admit(Some(b"session-token-a"), now).is_ok());
+    }
+
+    /// Negative — without root credentials no key can be classified, so every lookup is
+    /// `Unavailable` and IAM is not asked.
+    #[tokio::test]
+    async fn credentials_without_root_are_unavailable() {
+        let iam = fake(true, || found(user("on"), true));
+        let provider = IamCredentials::new(iam.clone(), Arc::new(RootCell::new(None)));
+        assert!(matches!(provider.lookup(ACCESS_KEY).await, Err(ProviderError::Unavailable)));
+        assert!(matches!(provider.lookup("rootadmin").await, Err(ProviderError::Unavailable)));
+        assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
+    }
+}

crates/gateway-adapter/src/host.rs

@@ -0,0 +1,306 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! The gateway's [`HostResolver`], built from RustFS's configured domain list.
+//!
+//! Responsible for: turning the `RUSTFS_SERVER_DOMAINS` list into the gateway's legacy RustFS
+//! virtual-host resolver.
+//! NOT responsible for: parsing a `Host` header, matching a domain, validating bucket names (all
+//! done by [`LegacyRustfsVirtualHosts`]), reading the environment, or deciding that a console
+//! deployment serves no domains (the caller passes an empty list then, as `s3_host_domains` in
+//! `rustfs/src/server/http.rs` returns `None`). An empty list needs no branch here:
+//! [`LegacyRustfsVirtualHosts`] built from no domain reads every host path-style, exactly as the
+//! gateway's `PathStyleOnly` does.
+//! Upstream: the RustFS configuration (`Config::server_domains`). Downstream: the gateway's
+//! `ServiceBuilder`, which holds the resolver as `Arc<dyn HostResolver>`.
+//!
+//! # What "the same as `MultiDomain`" means
+//!
+//! RustFS today hands the same list to the s3s framework's multi-domain host, which ignores ports
+//! on both sides when matching a domain, takes the whole prefix in front of a matched domain as
+//! the bucket, compares a domain exactly (so a host is matched on the case it arrives in), and,
+//! for a host no domain matches, falls back to CNAME-style: a host that is itself a bucket name
+//! (lower-cased) is that bucket. [`LegacyRustfsVirtualHosts`] reproduces exactly that.
+//!
+//! What reads **path-style**: an empty domain list; a configured domain itself, with or without a
+//! port; an IP address, or a socket address (`10.0.0.1`, `[::1]:9000`); a host that is a valid
+//! domain but, lower-cased, is no bucket name (it carries a port, is over 63 bytes, or starts
+//! with `xn--`). What reads as a **bucket**: anything under a configured domain, and any other
+//! bucket-shaped host, which is a bucket of its own (`localhost`, `badexample.com`, `EXAMPLE.COM`).
+//! What is **refused** before routing: a host that is no domain (`under_score.org`, a bare
+//! `[::1]`, a trailing dot), and a bucket under a domain that the bucket rules refuse
+//! (`PHOTOS.example.com`, `ab.example.com`). The tests below pin each of those readings.
+
+use std::sync::Arc;
+
+use rustfs_gateway::{HostResolver, LegacyDomainError, LegacyRustfsVirtualHosts};
+
+/// Builds the host resolver for RustFS's configured domain list.
+///
+/// An empty list reads every host path-style. Otherwise the list is read as legacy RustFS reads
+/// it: a later entry whose name without its port repeats an earlier one is dropped, and a list
+/// that legacy RustFS refuses at start-up is refused here.
+///
+/// # Errors
+///
+/// [`LegacyDomainError`] for an entry that is not a domain, or for two entries where one equals or
+/// is a subdomain of the other.
+pub fn rustfs_host_resolver<I, S>(domains: I) -> Result<Arc<dyn HostResolver>, LegacyDomainError>
+where
+    I: IntoIterator<Item = S>,
+    S: AsRef<str>,
+{
+    Ok(Arc::new(LegacyRustfsVirtualHosts::new(domains)?))
+}
+
+#[cfg(test)]
+#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
+mod tests {
+    use super::*;
+    use http::Method;
+    use rustfs_gateway::{Addressing, HostQuery, HostRefusal, TargetKind};
+
+    /// What the resolver reads `GET path` with `Host: host` as: the host-named bucket (`None` for
+    /// path style), the routed target, and the refusal, if any.
+    fn read(resolver: &dyn HostResolver, path: &str, host: &str) -> (Option<String>, TargetKind, Option<HostRefusal>) {
+        let method = Method::GET;
+        let request = http::Request::builder()
+            .method(method.clone())
+            .uri(path)
+            .header("host", host)
+            .body(())
+            .expect("a valid request");
+        let effective = rustfs_gateway_http::effective_host(&request).expect("an acceptable host");
+        let query = HostQuery {
+            host: &effective,
+            path,
+            method: &method,
+        };
+        let resolved = resolver.resolve(&query);
+        let bucket = match &resolved.addressing {
+            Addressing::VirtualHosted { bucket, .. } => Some(bucket.as_str().to_owned()),
+            Addressing::Path => None,
+        };
+        (bucket, resolved.target, resolver.refusal(&query))
+    }
+
+    fn with_domains(domains: &[&str]) -> Arc<dyn HostResolver> {
+        rustfs_host_resolver(domains).expect("a list RustFS accepts")
+    }
+
+    fn bucket(domains: &[&str], host: &str) -> Option<String> {
+        read(&*with_domains(domains), "/key", host).0
+    }
+
+    /// Positive: a host under a configured domain names the bucket in front of it.
+    #[test]
+    fn host_under_domain_names_the_bucket() {
+        assert_eq!(
+            read(&*with_domains(&["example.com"]), "/key", "photos.example.com"),
+            (Some("photos".to_owned()), TargetKind::Object, None)
+        );
+        assert_eq!(read(&*with_domains(&["example.com"]), "/", "photos.example.com").1, TargetKind::Bucket);
+    }
+
+    /// Positive: the port on the host, or on the configured domain, does not take part in matching.
+    #[test]
+    fn ports_are_ignored_on_both_sides() {
+        assert_eq!(bucket(&["example.com"], "photos.example.com:9000").as_deref(), Some("photos"));
+        assert_eq!(bucket(&["example.com:9000"], "photos.example.com").as_deref(), Some("photos"));
+        assert_eq!(bucket(&["example.com:9000"], "photos.example.com:1234").as_deref(), Some("photos"));
+    }
+
+    /// Positive: the whole prefix in front of the domain is the bucket, dots kept.
+    #[test]
+    fn whole_prefix_is_the_bucket() {
+        assert_eq!(bucket(&["example.com"], "my.dotted.bkt.example.com").as_deref(), Some("my.dotted.bkt"));
+    }
+
+    /// Positive: with several domains, each one resolves its own subdomains.
+    #[test]
+    fn every_configured_domain_resolves() {
+        let domains = ["example.com", "s3.local"];
+        assert_eq!(bucket(&domains, "abc.example.com").as_deref(), Some("abc"));
+        assert_eq!(bucket(&domains, "xyz.s3.local").as_deref(), Some("xyz"));
+    }
+
+    /// Positive: the domain itself is the endpoint, so the bucket comes from the path.
+    #[test]
+    fn the_domain_itself_is_path_style() {
+        assert_eq!(bucket(&["example.com"], "example.com"), None);
+        assert_eq!(bucket(&["example.com"], "example.com:9000"), None);
+    }
+
+    /// Negative: no configured domain means path style only, for every host shape.
+    #[test]
+    fn no_domain_is_path_style_only() {
+        for host in ["photos.example.com", "localhost", "localhost:9000", "10.0.0.1"] {
+            assert_eq!(bucket(&[], host), None, "{host}");
+        }
+    }
+
+    /// Negative: an address, and a host carrying a port that no domain matches, are path style.
+    #[test]
+    fn addresses_and_unmatched_ports_are_path_style() {
+        for host in ["10.0.0.1", "10.0.0.1:9000", "localhost:9000", "other.org:9000"] {
+            assert_eq!(bucket(&["example.com"], host), None, "{host}");
+        }
+    }
+
+    /// Negative: a host that only ends in the same letters as a domain is not under it. It is not
+    /// stripped to `bad`; the only reading it gets is the CNAME-style one, the whole host.
+    #[test]
+    fn suffix_lookalike_is_not_under_the_domain() {
+        assert_eq!(bucket(&["example.com"], "badexample.com").as_deref(), Some("badexample.com"));
+        assert_eq!(bucket(&["example.com"], "example.com.evil.org").as_deref(), Some("example.com.evil.org"));
+    }
+
+    /// Negative: matching a configured domain is exact on case, as the legacy multi-domain host does it, so a
+    /// mixed-case host is not stripped to its prefix; it is the lower-cased whole host.
+    #[test]
+    fn mixed_case_host_is_not_stripped_to_a_prefix() {
+        assert_eq!(bucket(&["example.com"], "Photos.Example.COM").as_deref(), Some("photos.example.com"));
+    }
+
+    /// Negative: a host that is no domain at all is refused, not routed.
+    #[test]
+    fn a_host_that_is_no_domain_is_refused() {
+        let resolver = with_domains(&["example.com"]);
+        assert_eq!(read(&*resolver, "/k", "under_score.org").2, Some(HostRefusal::UnusableHost));
+    }
+
+    /// Negative: a bucket the host names that bucket rules refuse is refused.
+    #[test]
+    fn a_host_named_bucket_the_rules_refuse_is_refused() {
+        let resolver = with_domains(&["example.com"]);
+        assert_eq!(read(&*resolver, "/k", "ab.example.com").2, Some(HostRefusal::RefusedBucket));
+        assert_eq!(read(&*resolver, "/k", "bad_host.example.com").2, Some(HostRefusal::RefusedBucket));
+    }
+
+    /// Negative: a list legacy RustFS refuses at start-up is refused.
+    #[test]
+    fn domain_lists_legacy_rustfs_refuses_are_refused() {
+        for list in [
+            vec!["example.com", "a.example.com"],
+            vec!["example.com", "com"],
+            vec!["bad_domain"],
+            vec![""],
+            vec!["example.com:notaport"],
+        ] {
+            assert!(rustfs_host_resolver(&list).is_err(), "{list:?}");
+        }
+    }
+
+    /// Positive: a later repeat of a name, whatever its port, is dropped rather than refused, as
+    /// `s3_host_domains` drops it before the legacy multi-domain host sees the list.
+    #[test]
+    fn a_repeat_of_a_domain_is_dropped() {
+        assert_eq!(
+            bucket(&["example.com:9411", "example.com:9000"], "xyz.example.com").as_deref(),
+            Some("xyz")
+        );
+    }
+
+    /// The refusal alone, for hosts whose bucket reading is not the point.
+    fn refusal(domains: &[&str], host: &str) -> Option<HostRefusal> {
+        read(&*with_domains(domains), "/k", host).2
+    }
+
+    /// Positive: a bucket-shaped host no domain matches is a bucket of its own, `localhost` and
+    /// an upper-case spelling of the domain among them.
+    #[test]
+    fn bucket_shaped_hosts_outside_every_domain_are_their_own_bucket() {
+        assert_eq!(bucket(&["example.com"], "localhost").as_deref(), Some("localhost"));
+        assert_eq!(bucket(&["example.com"], "EXAMPLE.COM").as_deref(), Some("example.com"));
+        assert_eq!(bucket(&["example.com"], "cdn.other.org").as_deref(), Some("cdn.other.org"));
+    }
+
+    /// Negative: a bucket the rules refuse under a domain is refused, and reads no bucket.
+    #[test]
+    fn an_upper_case_bucket_under_the_domain_is_refused() {
+        assert_eq!(bucket(&["example.com"], "PHOTOS.example.com"), None);
+        assert_eq!(refusal(&["example.com"], "PHOTOS.example.com"), Some(HostRefusal::RefusedBucket));
+    }
+
+    /// Negative: an IPv6 literal. The bare form is no domain and is refused; with a port it is a
+    /// socket address and reads path-style.
+    #[test]
+    fn ipv6_literals_follow_the_address_rules() {
+        assert_eq!(refusal(&["example.com"], "[::1]"), Some(HostRefusal::UnusableHost));
+        assert_eq!(bucket(&["example.com"], "[::1]:9000"), None);
+        assert_eq!(refusal(&["example.com"], "[::1]:9000"), None);
+    }
+
+    /// Negative: a trailing root dot makes the host no domain, so it is refused, not stripped.
+    #[test]
+    fn a_trailing_dot_is_refused() {
+        assert_eq!(refusal(&["example.com"], "photos.example.com."), Some(HostRefusal::UnusableHost));
+        assert_eq!(bucket(&["example.com"], "photos.example.com."), None);
+    }
+
+    /// Negative: an empty label never reaches the resolver; the gateway's wire layer refuses the
+    /// host first (legacy answers 400 as well).
+    #[test]
+    fn an_empty_label_is_refused_before_resolution() {
+        for host in ["a..b.example.com", ".example.com"] {
+            let request = http::Request::builder()
+                .uri("/k")
+                .header("host", host)
+                .body(())
+                .expect("a valid request");
+            assert!(rustfs_gateway_http::effective_host(&request).is_err(), "{host}");
+        }
+    }
+
+    /// Negative: a punycode label is no bucket under a domain; outside every domain the host is
+    /// a valid domain that is no bucket name, so it reads path-style.
+    #[test]
+    fn a_punycode_label_is_never_a_bucket() {
+        assert_eq!(refusal(&["example.com"], "xn--bcher-kva.example.com"), Some(HostRefusal::RefusedBucket));
+        assert_eq!(bucket(&["example.com"], "xn--bcher-kva.de"), None);
+        assert_eq!(refusal(&["example.com"], "xn--bcher-kva.de"), None);
+    }
+
+    /// Negative: a label over 63 bytes is no bucket, under a domain or outside one.
+    #[test]
+    fn a_label_over_63_bytes_is_never_a_bucket() {
+        let long = "a".repeat(64);
+        let under = format!("{long}.example.com");
+        assert_eq!(refusal(&["example.com"], &under), Some(HostRefusal::RefusedBucket));
+        let outside = format!("{long}.org");
+        assert_eq!(bucket(&["example.com"], &outside), None);
+        assert_eq!(refusal(&["example.com"], &outside), None);
+        let at_limit = "a".repeat(63);
+        assert_eq!(bucket(&["example.com"], &format!("{at_limit}.example.com")), Some(at_limit));
+    }
+
+    /// Positive and negative: two configured domains where one merely ends in the other's letters
+    /// both work, and each domain itself is path-style.
+    #[test]
+    fn a_domain_and_its_suffix_lookalike_are_both_served() {
+        let domains = ["example.com", "badexample.com"];
+        assert_eq!(bucket(&domains, "photos.example.com").as_deref(), Some("photos"));
+        assert_eq!(bucket(&domains, "photos.badexample.com").as_deref(), Some("photos"));
+        assert_eq!(bucket(&domains, "badexample.com"), None);
+        assert_eq!(bucket(&domains, "bad.example.com").as_deref(), Some("bad"));
+    }
+
+    /// Negative: a configured domain is matched exactly, case included.
+    #[test]
+    fn a_configured_domain_is_matched_on_its_case() {
+        assert_eq!(bucket(&["Example.COM"], "photos.Example.COM").as_deref(), Some("photos"));
+        assert_eq!(bucket(&["Example.COM"], "photos.example.com").as_deref(), Some("photos.example.com"));
+        assert_eq!(bucket(&["Example.COM"], "Example.COM"), None);
+    }
+}

crates/gateway-adapter/src/lib.rs

@@ -0,0 +1,107 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! RustFS implementations of the `rustfs-gateway` extension traits.
+//!
+//! This is the gateway's ring 2: it may depend on RustFS business crates, and nothing in the
+//! gateway depends on it. Each module adapts one extension point to the RustFS subsystem that
+//! owns the answer.
+
+pub mod authz_audit;
+pub mod bucket_owner;
+pub mod credentials;
+pub mod host;
+pub mod policy;
+pub mod root;
+pub mod trace;
+
+use std::sync::Arc;
+
+use rustfs_gateway::{Authenticator, CredentialProvider, ServiceBuilder};
+
+pub use authz_audit::{DenialAdmission, DenialAuditSink, GatewayDenialRecord, RustfsAuthzAudit};
+pub use bucket_owner::{BucketLookup, OwnerLookupError, RustfsBucketOwners};
+pub use credentials::{IamCredentials, IamKeyLookup, STS_SESSION_ISSUER};
+pub use host::rustfs_host_resolver;
+pub use policy::{IamPolicySource, IamPrincipalLookup, PrincipalKind, RustfsPrincipal};
+pub use root::{MIN_SECRET_KEY_LEN, RootCredentials, RootCredentialsError, RootSource};
+pub use trace::{AttachOutcome, RequestIdCarrier, attach_host_request_id, host_request_id};
+
+/// The IAM-backed credential provider and policy source, assembled together.
+///
+/// Both halves read the same IAM handle and the same live [`RootSource`], so a key cannot be root
+/// to one and an ordinary user to the other. Neither half can be taken out on its own: the fields
+/// are private, and [`IamAdapters::install`] consumes the pair and hands both to one builder.
+///
+/// Mixing the halves of two assemblies does not compile:
+///
+/// ```compile_fail
+/// fn mix(a: rustfs_gateway_adapter::IamAdapters, b: rustfs_gateway_adapter::IamAdapters) {
+///     let _mixed = (a.credentials, b.policy);
+/// }
+/// ```
+///
+/// Nor does building a half outside the pair:
+///
+/// ```compile_fail
+/// use std::sync::Arc;
+/// use rustfs_gateway_adapter::{IamPolicySource, IamPrincipalLookup, RootSource};
+/// fn build(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>) {
+///     let _alone = IamPolicySource::new(iam, root);
+/// }
+/// ```
+///
+/// Installing the pair is the one supported path:
+///
+/// ```no_run
+/// use std::sync::Arc;
+/// use rustfs_gateway::{RegionSet, ServiceBuilder, SigV4Authenticator};
+/// use rustfs_gateway_adapter::{IamAdapters, IamPrincipalLookup, RootSource};
+/// fn assemble(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>, regions: RegionSet) -> ServiceBuilder {
+///     IamAdapters::new(iam, root).install(ServiceBuilder::new(), |provider| SigV4Authenticator::new(provider, regions))
+/// }
+/// ```
+pub struct IamAdapters {
+    credentials: IamCredentials,
+    policy: IamPolicySource,
+}
+
+impl IamAdapters {
+    /// Assembles both adapters over one IAM handle and one root source. The binary passes a
+    /// source that reads its live root credentials on every call (see [`RootSource`]).
+    #[must_use]
+    pub fn new(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>) -> Self {
+        Self {
+            credentials: IamCredentials::new(iam.clone(), root.clone()),
+            policy: IamPolicySource::new(iam, root),
+        }
+    }
+
+    /// Installs both halves into `builder`: the credential provider through the authenticator
+    /// `authenticator` builds over it (for example `SigV4Authenticator::new(provider, regions)`
+    /// with the RustFS profile applied), and the policy source.
+    ///
+    /// Consumes the pair and gives both halves to this one builder. The `authenticator` closure
+    /// receives the provider handle and may keep a clone of it; nothing here prevents that, so
+    /// the closure must not hand it to a second builder.
+    #[must_use]
+    pub fn install<A: Authenticator>(
+        self,
+        builder: ServiceBuilder,
+        authenticator: impl FnOnce(Arc<dyn CredentialProvider>) -> A,
+    ) -> ServiceBuilder {
+        let authenticator = authenticator(Arc::new(self.credentials));
+        builder.authenticator(authenticator).policy_source(self.policy)
+    }
+}

crates/gateway-adapter/src/policy.rs

@@ -0,0 +1,779 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! The gateway's [`PolicySource`], answered by RustFS IAM.
+//!
+//! Responsible for: the one reading of the caller's IAM facts a request is judged against — which
+//! principal kind it is, the user it acts for, its groups, its token claims, and whether it owns
+//! the deployment — packed as a [`RustfsPrincipal`] into the gateway's [`PolicySnapshot`].
+//! NOT responsible for: evaluating any policy, choosing which policy documents apply (that stays
+//! in `IamSys::is_allowed`, which owns the STS, OIDC, session-policy and service-account rules), or
+//! verifying the signature or the session token (the gateway did both before calling this).
+//! Upstream: the gateway's verified [`Identity`]. Downstream: the RustFS authorizer, which reads
+//! the payload back with `PolicySnapshot::get::<RustfsPrincipal>()`.
+//!
+//! # Failure classes
+//!
+//! Every refusal is a [`PolicyError`] with a constant label and nothing from IAM or the request:
+//! IAM not ready, not initialized, or failing, or root credentials not initialized, is
+//! [`PolicyError::unavailable`]; a principal IAM no longer knows is [`UNKNOWN_PRINCIPAL`]; a
+//! disabled one, or one whose secret fails the effective-length rule, is
+//! [`DISABLED_PRINCIPAL`]; an identity
+//! whose credential kind disagrees with IAM's record is [`SESSION_MISMATCH`]; a token whose claims
+//! cannot be read is [`PolicyError::malformed`]. None of them is ever an empty snapshot: an empty
+//! snapshot is what an anonymous caller gets, and answering a fault with it would judge an
+//! authenticated caller as anonymous.
+//!
+//! # Secret handling
+//!
+//! [`RustfsPrincipal`] holds no secret key and no session token, and implements no `Debug`, so a
+//! snapshot formatted into a log cannot print its claims either. The root credentials and the STS
+//! token-signing key are read per request from the shared [`RootSource`] and never stored here.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use rustfs_credentials::Credentials as IamCredential;
+use rustfs_gateway::{BoxFuture, Identity, PolicyError, PolicySnapshot, PolicySource};
+use rustfs_iam::error::Error as IamError;
+use rustfs_iam::store::Store;
+use rustfs_iam::sys::{
+    IamSys, SESSION_POLICY_NAME, get_claims_from_token_with_secret, get_claims_from_token_with_secret_allow_missing_exp,
+    is_rustfs_oidc_claims,
+};
+use rustfs_policy::auth::ACCOUNT_OFF;
+use serde_json::Value;
+
+use crate::credentials::IamKeyLookup;
+use crate::root::{RootCredentials, RootSource, is_usable_secret};
+
+/// [`PolicyError`] label: IAM does not know the authenticated principal (any longer).
+pub const UNKNOWN_PRINCIPAL: &str = "unknown-principal";
+/// [`PolicyError`] label: IAM holds the principal disabled, expired, or otherwise invalid.
+pub const DISABLED_PRINCIPAL: &str = "disabled-principal";
+/// [`PolicyError`] label: the identity's credential kind (session or long-term) disagrees with
+/// what IAM holds for its access key.
+pub const SESSION_MISMATCH: &str = "session-mismatch";
+
+/// The narrow slice of IAM a policy reading needs, on top of [`IamKeyLookup`].
+///
+/// Implemented for [`IamSys`]; exists so the source can be tested against a fake.
+pub trait IamPrincipalLookup: IamKeyLookup {
+    /// The built-in IAM groups `user` belongs to now, as `IamSys::user_group_memberships`
+    /// answers. These may include groups granted after a credential was issued.
+    fn group_memberships(&self, user: &str) -> Vec<String>;
+}
+
+impl<T: Store> IamPrincipalLookup for IamSys<T> {
+    fn group_memberships(&self, user: &str) -> Vec<String> {
+        IamSys::user_group_memberships(self, user)
+    }
+}
+
+/// Which kind of credential the principal authenticated with.
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub enum PrincipalKind {
+    /// The deployment's root credentials.
+    Root,
+    /// A long-term IAM user.
+    User,
+    /// A service account, acting for its parent user.
+    ServiceAccount,
+    /// An STS temporary credential, acting for its parent user.
+    Temporary,
+}
+
+/// The caller's IAM facts: the payload of every [`PolicySnapshot`] [`IamPolicySource`] produces.
+///
+/// Carries no secret key and no session token, and deliberately implements no `Debug`.
+pub struct RustfsPrincipal {
+    kind: PrincipalKind,
+    account: String,
+    username: String,
+    parent_user: String,
+    credential_groups: Option<Vec<String>>,
+    group_memberships: Vec<String>,
+    claims: Option<HashMap<String, Value>>,
+    is_owner: bool,
+}
+
+impl RustfsPrincipal {
+    /// The kind of credential the caller authenticated with.
+    #[must_use]
+    pub fn kind(&self) -> PrincipalKind {
+        self.kind
+    }
+
+    /// The access key the caller signed with (IAM `Args::account`).
+    #[must_use]
+    pub fn account(&self) -> &str {
+        &self.account
+    }
+
+    /// The user the caller acts as: the parent user for an STS credential or a service account,
+    /// the access key otherwise.
+    #[must_use]
+    pub fn username(&self) -> &str {
+        &self.username
+    }
+
+    /// The parent user IAM records for the credential; empty for root and long-term users.
+    #[must_use]
+    pub fn parent_user(&self) -> &str {
+        &self.parent_user
+    }
+
+    /// The groups recorded on the credential itself (IAM `Args::groups`).
+    #[must_use]
+    pub fn credential_groups(&self) -> &Option<Vec<String>> {
+        &self.credential_groups
+    }
+
+    /// The groups IAM currently lists for [`Self::username`], read once with the snapshot.
+    #[must_use]
+    pub fn group_memberships(&self) -> &[String] {
+        &self.group_memberships
+    }
+
+    /// The claims the credential's token carries; `None` when it has none.
+    #[must_use]
+    pub fn claims(&self) -> Option<&HashMap<String, Value>> {
+        self.claims.as_ref()
+    }
+
+    /// Whether the caller owns the deployment (IAM `Args::is_owner`): root, or a credential of
+    /// root that no session policy or external OIDC identity narrows.
+    #[must_use]
+    pub fn is_owner(&self) -> bool {
+        self.is_owner
+    }
+}
+
+/// A [`PolicySource`] over RustFS IAM and the live root credentials.
+///
+/// Built only by [`crate::IamAdapters::new`], so it always reads the same [`RootSource`] as the
+/// credential provider it was assembled with.
+pub struct IamPolicySource {
+    iam: Arc<dyn IamPrincipalLookup>,
+    root: Arc<dyn RootSource>,
+}
+
+impl IamPolicySource {
+    pub(crate) fn new(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>) -> Self {
+        Self { iam, root }
+    }
+
+    async fn read(&self, identity: Option<&Identity>) -> Result<PolicySnapshot, PolicyError> {
+        let Some(identity) = identity else {
+            return Ok(PolicySnapshot::empty());
+        };
+        let principal = self.principal(identity).await?;
+        Ok(PolicySnapshot::of(Arc::new(principal)))
+    }
+
+    async fn principal(&self, identity: &Identity) -> Result<RustfsPrincipal, PolicyError> {
+        let access_key = identity.access_key_id();
+        let has_session = identity.session().is_some();
+        // Without root credentials neither ownership nor STS claims can be decided.
+        let Some(root) = self.root.current() else {
+            return Err(PolicyError::unavailable());
+        };
+        let root_access_key = root.access_key();
+
+        if access_key == root_access_key {
+            // Root has no session token, so a session identity for its key is not one IAM issued.
+            if has_session {
+                return Err(PolicyError::because(SESSION_MISMATCH));
+            }
+            return Ok(RustfsPrincipal {
+                kind: PrincipalKind::Root,
+                account: access_key.to_owned(),
+                username: access_key.to_owned(),
+                parent_user: String::new(),
+                credential_groups: None,
+                group_memberships: Vec::new(),
+                claims: None,
+                is_owner: true,
+            });
+        }
+
+        if !self.iam.is_ready() {
+            return Err(PolicyError::unavailable());
+        }
+        let cred = match self.iam.check_key(access_key).await {
+            Ok((Some(identity), true)) => identity.credentials,
+            Ok((Some(_), false)) => return Err(PolicyError::because(DISABLED_PRINCIPAL)),
+            Ok((None, _)) => return Err(PolicyError::because(UNKNOWN_PRINCIPAL)),
+            Err(error) => return Err(classify_iam_error(&error)),
+        };
+        if cred.status == ACCOUNT_OFF || !is_usable_secret(cred.secret_key.as_bytes()) {
+            return Err(PolicyError::because(DISABLED_PRINCIPAL));
+        }
+
+        let kind = if cred.is_service_account() {
+            PrincipalKind::ServiceAccount
+        } else if !cred.session_token.is_empty() {
+            PrincipalKind::Temporary
+        } else {
+            PrincipalKind::User
+        };
+        // The gateway admitted this identity against the credential IAM returned, so the kinds
+        // agree unless IAM's record changed in between; refuse rather than judge a stale reading.
+        if has_session != (kind == PrincipalKind::Temporary) {
+            return Err(PolicyError::because(SESSION_MISMATCH));
+        }
+
+        let claims = read_claims(kind, &cred, &root)?;
+        let username = match kind {
+            PrincipalKind::Temporary | PrincipalKind::ServiceAccount => cred.parent_user.clone(),
+            PrincipalKind::Root | PrincipalKind::User => cred.access_key.clone(),
+        };
+        let membership_subject = if username.is_empty() { access_key } else { username.as_str() };
+        let group_memberships = self.iam.group_memberships(membership_subject);
+        let is_owner = acts_for_root(&cred, claims.as_ref(), root_access_key);
+
+        Ok(RustfsPrincipal {
+            kind,
+            account: access_key.to_owned(),
+            username,
+            parent_user: cred.parent_user,
+            credential_groups: cred.groups,
+            group_memberships,
+            claims,
+            is_owner,
+        })
+    }
+}
+
+/// The claims of the credential's own token. The gateway admitted the presented token only when it
+/// equals the stored one, so reading the stored token reads the presented one.
+fn read_claims(
+    kind: PrincipalKind,
+    cred: &IamCredential,
+    root: &RootCredentials,
+) -> Result<Option<HashMap<String, Value>>, PolicyError> {
+    // As in legacy `check_claims_from_token_with_context`: no token, no claims.
+    if cred.session_token.is_empty() {
+        return Ok(None);
+    }
+    let claims = match kind {
+        PrincipalKind::ServiceAccount => {
+            get_claims_from_token_with_secret_allow_missing_exp(&cred.session_token, &cred.secret_key)
+                .map_err(|_| PolicyError::malformed())?
+        }
+        PrincipalKind::Temporary => get_claims_from_token_with_secret(&cred.session_token, root.sts_signing_key())
+            .map_err(|_| PolicyError::malformed())?,
+        PrincipalKind::Root | PrincipalKind::User => return Ok(None),
+    };
+    Ok(if claims.is_empty() { None } else { Some(claims) })
+}
+
+/// Classifies an IAM failure without keeping any of its text.
+fn classify_iam_error(error: &IamError) -> PolicyError {
+    match error {
+        IamError::NoSuchUser(_)
+        | IamError::NoSuchServiceAccount(_)
+        | IamError::NoSuchTempAccount(_)
+        | IamError::NoSuchAccount(_) => PolicyError::because(UNKNOWN_PRINCIPAL),
+        _ => PolicyError::unavailable(),
+    }
+}
+
+/// Root ownership as the legacy edge decides it (`rustfs/src/auth.rs` `has_root_access`): the
+/// credential is root's or acts for root, and neither a session policy nor an external OIDC
+/// identity narrows it.
+fn acts_for_root(cred: &IamCredential, claims: Option<&HashMap<String, Value>>, root_access_key: &str) -> bool {
+    (cred.access_key == root_access_key || cred.parent_user == root_access_key)
+        && !claims.is_some_and(|claims| claims.contains_key(SESSION_POLICY_NAME) || is_rustfs_oidc_claims(claims))
+}
+
+impl PolicySource for IamPolicySource {
+    fn snapshot<'a>(&'a self, identity: Option<&'a Identity>) -> BoxFuture<'a, Result<PolicySnapshot, PolicyError>> {
+        Box::pin(self.read(identity))
+    }
+}
+
+#[cfg(test)]
+#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
+mod tests {
+    use super::*;
+    use crate::root::RootCell;
+    use rustfs_credentials::{Credentials as IamCredential, IAM_POLICY_CLAIM_NAME_SA};
+    use rustfs_gateway::SessionBinding;
+    use rustfs_iam::error::Error as IamError;
+    use rustfs_iam::sys::SESSION_POLICY_NAME;
+    use rustfs_iam::utils::generate_jwt;
+    use rustfs_policy::auth::UserIdentity;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use time::OffsetDateTime;
+
+    const ROOT_ACCESS_KEY: &str = "rootadmin";
+    const ROOT_SECRET: &str = "root-secret-key-0001";
+    /// Deliberately not the root secret, so every STS test proves which key claims are read with.
+    const SIGNING_KEY: &str = "sts-signing-key-0001";
+    const ACCESS_KEY: &str = "AKIDUSER0001";
+    const SECRET: &str = "user-secret-key-0001";
+    const PARENT: &str = "alice";
+    const EXPIRES_AT: i64 = 4_000_000_000;
+
+    type KeyAnswer = Result<(Option<UserIdentity>, bool), IamError>;
+    type Answer = Box<dyn Fn() -> KeyAnswer + Send + Sync>;
+
+    struct FakeIam {
+        ready: bool,
+        answer: Answer,
+        calls: AtomicUsize,
+    }
+
+    impl IamKeyLookup for FakeIam {
+        fn is_ready(&self) -> bool {
+            self.ready
+        }
+
+        fn check_key<'a>(&'a self, _access_key: &'a str) -> BoxFuture<'a, KeyAnswer> {
+            self.calls.fetch_add(1, Ordering::SeqCst);
+            let answer = (self.answer)();
+            Box::pin(async move { answer })
+        }
+    }
+
+    impl IamPrincipalLookup for FakeIam {
+        /// Distinct groups per name, so a test can tell which name the source asked about.
+        fn group_memberships(&self, user: &str) -> Vec<String> {
+            match user {
+                PARENT => vec!["parent-group".to_owned()],
+                ACCESS_KEY => vec!["key-group".to_owned()],
+                ROOT_ACCESS_KEY => vec!["root-group".to_owned()],
+                _ => Vec::new(),
+            }
+        }
+    }
+
+    fn fake(ready: bool, answer: impl Fn() -> KeyAnswer + Send + Sync + 'static) -> Arc<FakeIam> {
+        Arc::new(FakeIam {
+            ready,
+            answer: Box::new(answer),
+            calls: AtomicUsize::new(0),
+        })
+    }
+
+    fn root() -> RootCredentials {
+        RootCredentials::new(ROOT_ACCESS_KEY, ROOT_SECRET, SIGNING_KEY).expect("valid root credentials")
+    }
+
+    fn source(iam: Arc<FakeIam>) -> IamPolicySource {
+        IamPolicySource::new(iam, Arc::new(RootCell::new(Some(root()))))
+    }
+
+    fn user() -> IamCredential {
+        IamCredential {
+            access_key: ACCESS_KEY.to_owned(),
+            secret_key: SECRET.to_owned(),
+            status: "on".to_owned(),
+            groups: Some(vec!["cred-group".to_owned()]),
+            ..Default::default()
+        }
+    }
+
+    fn sts_token(claims: &[(&str, Value)], secret: &str) -> String {
+        let mut map: HashMap<String, Value> = claims.iter().map(|(k, v)| ((*k).to_owned(), v.clone())).collect();
+        map.insert("exp".to_owned(), Value::from(EXPIRES_AT));
+        generate_jwt(&map, secret).expect("a token")
+    }
+
+    fn sts(parent: &str, token: String) -> IamCredential {
+        IamCredential {
+            session_token: token,
+            parent_user: parent.to_owned(),
+            expiration: Some(OffsetDateTime::from_unix_timestamp(EXPIRES_AT).expect("valid timestamp")),
+            ..user()
+        }
+    }
+
+    fn found(cred: IamCredential) -> KeyAnswer {
+        Ok((Some(UserIdentity::new(cred)), true))
+    }
+
+    fn long_term(access_key: &str) -> Identity {
+        Identity::new(access_key).expect("valid access key")
+    }
+
+    fn session(access_key: &str) -> Identity {
+        let binding = SessionBinding::new(crate::STS_SESSION_ISSUER, EXPIRES_AT).expect("valid binding");
+        long_term(access_key).with_session_binding(binding)
+    }
+
+    async fn principal_of(source: &IamPolicySource, identity: &Identity) -> PolicySnapshot {
+        source.snapshot(Some(identity)).await.expect("a snapshot")
+    }
+
+    fn payload(snapshot: &PolicySnapshot) -> &RustfsPrincipal {
+        snapshot.get::<RustfsPrincipal>().expect("the payload is a RustfsPrincipal")
+    }
+
+    fn refusal(result: Result<PolicySnapshot, PolicyError>) -> PolicyError {
+        match result {
+            Ok(snapshot) => panic!("expected a refusal, got a snapshot (empty: {})", snapshot.is_empty()),
+            Err(error) => error,
+        }
+    }
+
+    /// Compile-time: `RustfsPrincipal` has no `Debug`, so no formatting of a snapshot payload can
+    /// reach its claims. The two blanket impls make the path ambiguous, and so a compile error, as
+    /// soon as `RustfsPrincipal: Debug` holds.
+    #[test]
+    fn policy_principal_has_no_debug() {
+        trait AmbiguousIfDebug<A> {
+            fn some_item() {}
+        }
+        impl<T: ?Sized> AmbiguousIfDebug<()> for T {}
+        impl<T: ?Sized + core::fmt::Debug> AmbiguousIfDebug<u8> for T {}
+        let _ = <RustfsPrincipal as AmbiguousIfDebug<_>>::some_item;
+    }
+
+    /// Positive — a long-term user's facts: itself as username, both group sources, no claims,
+    /// not the owner.
+    #[tokio::test]
+    async fn policy_user_snapshot_carries_iam_facts() {
+        let iam = fake(true, || found(user()));
+        let snapshot = principal_of(&source(iam), &long_term(ACCESS_KEY)).await;
+        let principal = payload(&snapshot);
+        assert!(principal.kind() == PrincipalKind::User);
+        assert_eq!(principal.account(), ACCESS_KEY);
+        assert_eq!(principal.username(), ACCESS_KEY);
+        assert_eq!(principal.parent_user(), "");
+        assert_eq!(principal.credential_groups(), &Some(vec!["cred-group".to_owned()]));
+        assert_eq!(principal.group_memberships(), ["key-group".to_owned()]);
+        assert!(principal.claims().is_none());
+        assert!(!principal.is_owner());
+    }
+
+    /// Positive — an STS credential acts for its parent user, and its claims are the ones its
+    /// token carries, read with the root secret that signed it.
+    #[tokio::test]
+    async fn policy_sts_snapshot_carries_parent_user_and_claims() {
+        let token = sts_token(&[("sub", Value::from("alice-session"))], SIGNING_KEY);
+        let iam = fake(true, move || found(sts(PARENT, token.clone())));
+        let snapshot = principal_of(&source(iam), &session(ACCESS_KEY)).await;
+        let principal = payload(&snapshot);
+        assert!(principal.kind() == PrincipalKind::Temporary);
+        assert_eq!(principal.account(), ACCESS_KEY);
+        assert_eq!(principal.parent_user(), PARENT);
+        assert_eq!(principal.username(), PARENT);
+        let claims = principal.claims().expect("an STS principal has claims");
+        assert_eq!(claims.get("sub"), Some(&Value::from("alice-session")));
+        assert_eq!(principal.group_memberships(), ["parent-group".to_owned()]);
+        assert!(!principal.is_owner());
+    }
+
+    /// Positive — a service account's claims are read with its own secret, and it acts for its
+    /// parent.
+    #[tokio::test]
+    async fn policy_service_account_snapshot_reads_its_own_token() {
+        let mut token_claims: HashMap<String, Value> = HashMap::new();
+        token_claims.insert(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"));
+        token_claims.insert("parent".to_owned(), Value::from(PARENT));
+        let token = generate_jwt(&token_claims, SECRET).expect("a token");
+        let iam = fake(true, move || {
+            let mut cred = user();
+            cred.session_token = token.clone();
+            cred.parent_user = PARENT.to_owned();
+            cred.claims = Some([(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"))].into());
+            found(cred)
+        });
+        let snapshot = principal_of(&source(iam), &long_term(ACCESS_KEY)).await;
+        let principal = payload(&snapshot);
+        assert!(principal.kind() == PrincipalKind::ServiceAccount);
+        assert_eq!(principal.username(), PARENT);
+        assert_eq!(principal.claims().and_then(|c| c.get("parent")), Some(&Value::from(PARENT)));
+        assert_eq!(principal.group_memberships(), ["parent-group".to_owned()]);
+        assert!(!principal.is_owner());
+    }
+
+    /// Positive and negative — root is the owner without asking IAM, even before IAM is ready;
+    /// an STS credential of root is the owner too, but not once a session policy narrows it.
+    #[tokio::test]
+    async fn policy_owner_is_root_or_unrestricted_root_session() {
+        let not_ready = fake(false, || found(user()));
+        let snapshot = principal_of(&source(not_ready.clone()), &long_term(ROOT_ACCESS_KEY)).await;
+        let root = payload(&snapshot);
+        assert!(root.kind() == PrincipalKind::Root);
+        assert!(root.is_owner());
+        assert_eq!(not_ready.calls.load(Ordering::SeqCst), 0);
+
+        let plain = sts_token(&[], SIGNING_KEY);
+        let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, plain.clone())));
+        assert!(payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
+
+        let narrowed = sts_token(&[(SESSION_POLICY_NAME, Value::from("e30"))], SIGNING_KEY);
+        let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, narrowed.clone())));
+        assert!(!payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
+    }
+
+    /// Negative — an anonymous caller is an empty snapshot, and IAM is not asked.
+    #[tokio::test]
+    async fn policy_anonymous_is_empty_snapshot() {
+        let iam = fake(true, || found(user()));
+        let snapshot = source(iam.clone()).snapshot(None).await.expect("an anonymous snapshot");
+        assert!(snapshot.is_empty());
+        assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
+    }
+
+    /// Negative — IAM that is not ready, not initialized, or failing is `unavailable`: never an
+    /// empty snapshot, which an authorizer would judge as an anonymous caller.
+    #[tokio::test]
+    async fn policy_iam_unavailable_is_unavailable() {
+        let not_ready = fake(false, || found(user()));
+        let error = refusal(source(not_ready.clone()).snapshot(Some(&long_term(ACCESS_KEY))).await);
+        assert_eq!(error, PolicyError::unavailable());
+        assert_eq!(not_ready.calls.load(Ordering::SeqCst), 0);
+
+        let failures: [fn() -> IamError; 2] = [
+            || IamError::IamSysNotInitialized,
+            || IamError::StringError("storage read failed".to_owned()),
+        ];
+        for failure in failures {
+            let iam = fake(true, move || Err(failure()));
+            let error = refusal(source(iam).snapshot(Some(&long_term(ACCESS_KEY))).await);
+            assert_eq!(error, PolicyError::unavailable(), "{:?}", failure());
+        }
+    }
+
+    /// Negative — a principal IAM no longer knows is refused as unknown, not as unavailable and
+    /// not as an empty snapshot.
+    #[tokio::test]
+    async fn policy_unknown_principal_is_refused() {
+        let answers: [fn() -> KeyAnswer; 2] = [|| Ok((None, false)), || Err(IamError::NoSuchUser(ACCESS_KEY.to_owned()))];
+        for answer in answers {
+            let error = refusal(source(fake(true, answer)).snapshot(Some(&long_term(ACCESS_KEY))).await);
+            assert_eq!(error, PolicyError::because(UNKNOWN_PRINCIPAL));
+        }
+    }
+
+    /// Negative — a disabled or invalid principal is refused rather than given a snapshot.
+    #[tokio::test]
+    async fn policy_disabled_principal_is_refused() {
+        let invalid = fake(true, || Ok((Some(UserIdentity::new(user())), false)));
+        let error = refusal(source(invalid).snapshot(Some(&long_term(ACCESS_KEY))).await);
+        assert_eq!(error, PolicyError::because(DISABLED_PRINCIPAL));
+
+        let off = fake(true, || {
+            let mut cred = user();
+            cred.status = "off".to_owned();
+            found(cred)
+        });
+        let error = refusal(source(off).snapshot(Some(&long_term(ACCESS_KEY))).await);
+        assert_eq!(error, PolicyError::because(DISABLED_PRINCIPAL));
+    }
+
+    /// Negative — an STS token not signed by root yields no claims at all: it is `malformed`.
+    #[tokio::test]
+    async fn policy_sts_token_with_foreign_signature_is_malformed() {
+        let forged = sts_token(&[("sub", Value::from("alice-session"))], "not-the-root-secret");
+        let iam = fake(true, move || found(sts(PARENT, forged.clone())));
+        let error = refusal(source(iam).snapshot(Some(&session(ACCESS_KEY))).await);
+        assert_eq!(error, PolicyError::malformed());
+    }
+
+    /// Negative — the identity's credential kind must match what IAM holds: a session identity for
+    /// a long-term key, or a long-term identity for an STS key, is refused.
+    #[tokio::test]
+    async fn policy_session_kind_mismatch_is_refused() {
+        let token = sts_token(&[], SIGNING_KEY);
+        let iam = fake(true, move || found(sts(PARENT, token.clone())));
+        let error = refusal(source(iam).snapshot(Some(&long_term(ACCESS_KEY))).await);
+        assert_eq!(error, PolicyError::because(SESSION_MISMATCH));
+
+        let iam = fake(true, || found(user()));
+        let error = refusal(source(iam).snapshot(Some(&session(ACCESS_KEY))).await);
+        assert_eq!(error, PolicyError::because(SESSION_MISMATCH));
+
+        let iam = fake(false, || found(user()));
+        let error = refusal(source(iam).snapshot(Some(&session(ROOT_ACCESS_KEY))).await);
+        assert_eq!(error, PolicyError::because(SESSION_MISMATCH));
+    }
+
+    fn service_account(parent: &str, extra_claims: &[(&str, Value)]) -> KeyAnswer {
+        let mut token_claims: HashMap<String, Value> = extra_claims.iter().map(|(k, v)| ((*k).to_owned(), v.clone())).collect();
+        token_claims.insert(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"));
+        let mut cred = user();
+        cred.session_token = generate_jwt(&token_claims, SECRET).expect("a token");
+        cred.parent_user = parent.to_owned();
+        cred.claims = Some([(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"))].into());
+        found(cred)
+    }
+
+    /// Negative — a valid-flagged identity whose secret is empty or short gets no snapshot.
+    #[tokio::test]
+    async fn policy_short_secret_principal_is_refused() {
+        for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0"] {
+            let iam = fake(true, move || {
+                let mut cred = user();
+                cred.secret_key = weak.to_owned();
+                found(cred)
+            });
+            let error = refusal(source(iam).snapshot(Some(&long_term(ACCESS_KEY))).await);
+            assert_eq!(error, PolicyError::because(DISABLED_PRINCIPAL), "{weak:?}");
+        }
+        for usable in ["12345678", "abc\0defg"] {
+            let iam = fake(true, move || {
+                let mut cred = user();
+                cred.secret_key = usable.to_owned();
+                found(cred)
+            });
+            assert!(payload(&principal_of(&source(iam), &long_term(ACCESS_KEY)).await).kind() == PrincipalKind::User);
+        }
+    }
+
+    /// Negative — a root session issued through external OIDC is not the owner, though its parent
+    /// is root; the same session without the OIDC issuer is.
+    #[tokio::test]
+    async fn policy_oidc_root_session_is_not_owner() {
+        let oidc = sts_token(
+            &[
+                ("iss", Value::from("rustfs-oidc")),
+                ("oidc_provider", Value::from("corp-idp")),
+                ("sub", Value::from("alice@corp")),
+            ],
+            SIGNING_KEY,
+        );
+        let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, oidc.clone())));
+        assert!(!payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
+
+        let local = sts_token(
+            &[
+                ("iss", Value::from("rustfs-sts")),
+                ("oidc_provider", Value::from("corp-idp")),
+                ("sub", Value::from("alice@corp")),
+            ],
+            SIGNING_KEY,
+        );
+        let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, local.clone())));
+        assert!(payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
+    }
+
+    /// Positive and negative — a service account of root is the owner; one narrowed by a session
+    /// policy is not, and neither is a service account of an ordinary user.
+    #[tokio::test]
+    async fn policy_service_account_of_root_is_owner() {
+        let iam = fake(true, || service_account(ROOT_ACCESS_KEY, &[]));
+        let principal = principal_of(&source(iam), &long_term(ACCESS_KEY)).await;
+        assert!(payload(&principal).kind() == PrincipalKind::ServiceAccount);
+        assert!(payload(&principal).is_owner());
+
+        let iam = fake(true, || service_account(ROOT_ACCESS_KEY, &[(SESSION_POLICY_NAME, Value::from("e30"))]));
+        assert!(!payload(&principal_of(&source(iam), &long_term(ACCESS_KEY)).await).is_owner());
+
+        let iam = fake(true, || service_account(PARENT, &[]));
+        assert!(!payload(&principal_of(&source(iam), &long_term(ACCESS_KEY)).await).is_owner());
+    }
+
+    /// Negative — STS claims are read with the STS signing key, not with the root secret: a token
+    /// signed with the root secret while the signing key differs is `malformed`.
+    #[tokio::test]
+    async fn policy_sts_claims_use_the_signing_key() {
+        let root_signed = sts_token(&[("sub", Value::from("alice-session"))], ROOT_SECRET);
+        let iam = fake(true, move || found(sts(PARENT, root_signed.clone())));
+        let error = refusal(source(iam).snapshot(Some(&session(ACCESS_KEY))).await);
+        assert_eq!(error, PolicyError::malformed());
+    }
+
+    /// Negative — without root credentials no principal can be judged: `unavailable`, never an
+    /// empty snapshot, and IAM is not asked.
+    #[tokio::test]
+    async fn policy_without_root_is_unavailable() {
+        let iam = fake(true, || found(user()));
+        let source = IamPolicySource::new(iam.clone(), Arc::new(RootCell::new(None)));
+        for identity in [long_term(ACCESS_KEY), long_term(ROOT_ACCESS_KEY)] {
+            assert_eq!(refusal(source.snapshot(Some(&identity)).await), PolicyError::unavailable());
+        }
+        assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
+    }
+
+    type LiveRoot = Arc<std::sync::Mutex<Option<(String, String)>>>;
+
+    /// The primary path: a source that reads a live accessor on every call, as the binary's does,
+    /// with nothing cached in between.
+    fn live_root_source(live: &LiveRoot) -> Arc<dyn RootSource> {
+        let accessor = live.clone();
+        Arc::new(move || {
+            let guard = accessor.lock().ok()?;
+            let (access_key, secret_key) = guard.as_ref()?;
+            RootCredentials::new(access_key, secret_key, SIGNING_KEY).ok().map(Arc::new)
+        })
+    }
+
+    fn set_live(live: &LiveRoot, value: Option<(&str, &str)>) {
+        *live.lock().expect("live root lock") = value.map(|(ak, sk)| (ak.to_owned(), sk.to_owned()));
+    }
+
+    /// Positive and negative — the two adapters read one live root accessor: a root change is seen
+    /// by both on the next request with no second step, the old root key stops being root to
+    /// both, and an uninitialized or unusable root makes both unavailable.
+    #[tokio::test]
+    async fn adapters_share_one_live_root() {
+        use rustfs_gateway::{CredentialLookup, CredentialProvider, ProviderError};
+
+        let live: LiveRoot = Arc::default();
+        set_live(&live, Some((ROOT_ACCESS_KEY, ROOT_SECRET)));
+        let iam = fake(true, || Ok((None, false)));
+        let adapters = crate::IamAdapters::new(iam, live_root_source(&live));
+
+        let found = adapters.credentials.lookup(ROOT_ACCESS_KEY).await;
+        assert!(matches!(found, Ok(CredentialLookup::Found(_))));
+        let snapshot = principal_of(&adapters.policy, &long_term(ROOT_ACCESS_KEY)).await;
+        assert!(payload(&snapshot).is_owner());
+
+        set_live(&live, Some(("newroot", "new-root-secret-01")));
+        match adapters.credentials.lookup("newroot").await {
+            Ok(CredentialLookup::Found(credentials)) => assert_eq!(credentials.secret().expose(), b"new-root-secret-01"),
+            _ => panic!("the new root key must resolve to the new secret"),
+        }
+        assert!(matches!(
+            adapters.credentials.lookup(ROOT_ACCESS_KEY).await,
+            Ok(CredentialLookup::NotFound)
+        ));
+        assert!(payload(&principal_of(&adapters.policy, &long_term("newroot")).await).is_owner());
+        let old = refusal(adapters.policy.snapshot(Some(&long_term(ROOT_ACCESS_KEY))).await);
+        assert_eq!(old, PolicyError::because(UNKNOWN_PRINCIPAL));
+
+        for unusable in [None, Some(("newroot", "\0\0\0\0\0\0\0\0"))] {
+            set_live(&live, unusable);
+            assert!(matches!(adapters.credentials.lookup("newroot").await, Err(ProviderError::Unavailable)));
+            assert_eq!(
+                refusal(adapters.policy.snapshot(Some(&long_term("newroot"))).await),
+                PolicyError::unavailable()
+            );
+        }
+    }
+
+    /// Positive — `install` builds the authenticator exactly once, over a provider that reads the
+    /// pair's root source.
+    #[tokio::test]
+    async fn adapters_install_hands_over_the_paired_provider_once() {
+        use rustfs_gateway::{CredentialLookup, CredentialProvider, RegionSet, ServiceBuilder, SigV4Authenticator};
+
+        let live: LiveRoot = Arc::default();
+        set_live(&live, Some((ROOT_ACCESS_KEY, ROOT_SECRET)));
+        let adapters = crate::IamAdapters::new(fake(true, || Ok((None, false))), live_root_source(&live));
+        let mut handed = Vec::new();
+        let _builder = adapters.install(ServiceBuilder::new(), |provider| {
+            handed.push(provider.clone());
+            SigV4Authenticator::new(provider, RegionSet::new(["us-east-1"]).expect("a region"))
+        });
+        assert_eq!(handed.len(), 1);
+        let found = handed[0].lookup(ROOT_ACCESS_KEY).await;
+        assert!(matches!(found, Ok(CredentialLookup::Found(_))));
+    }
+}

crates/gateway-adapter/src/root.rs

@@ -0,0 +1,216 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! The deployment's root credentials, as every adapter in this crate reads them.
+//!
+//! Responsible for: [`RootCredentials`], the validated root key pair plus the key STS session
+//! tokens are signed with; [`RootSource`], the one live handle both the credential provider and
+//! the policy source read on every request; and the effective-secret-length rule every secret in
+//! this crate is held to.
+//! NOT responsible for: where the root keys come from or when they change. The binary implements
+//! [`RootSource`] directly over its live root accessor — a closure is enough — so there is no
+//! second copy of root to keep in step.
+//! Upstream: the binary's runtime credentials. Downstream: `crate::credentials`, `crate::policy`.
+//!
+//! # Why one source
+//!
+//! Legacy reads the current root credentials on every request. Two copies taken at assembly could
+//! disagree, or go stale after a root change, and a key one adapter still treats as root would be
+//! answered with the root secret or judged as the owner. Both adapters therefore hold the same
+//! `Arc<dyn RootSource>`, handed to them together by [`crate::IamAdapters::new`].
+
+use std::sync::Arc;
+
+use rustfs_gateway::Credentials;
+
+/// The shortest effective secret key, in bytes, that any credential in this crate may verify a
+/// signature with. The same bound IAM applies to its own credentials; a signature over a shorter
+/// (or empty) key is one anybody can compute.
+pub const MIN_SECRET_KEY_LEN: usize = 8;
+
+/// The number of bytes of `secret` a signature actually depends on: its length after trailing NUL
+/// bytes are removed.
+///
+/// HMAC pads a key shorter than its block with zero bytes, and SigV4's first key is `"AWS4"`
+/// followed by the secret, so trailing NULs add nothing: eight NUL bytes sign exactly like the
+/// empty secret, and `"abc"` followed by five NULs signs like `"abc"`. A secret of NULs only has
+/// an effective length of zero. An interior NUL is followed by real key bytes, which change the
+/// padded key, so it counts like any other byte.
+pub(crate) fn effective_secret_len(secret: &[u8]) -> usize {
+    secret.iter().rposition(|byte| *byte != 0).map_or(0, |last| last + 1)
+}
+
+/// Whether `secret` may verify a signature: its [`effective_secret_len`] is at least
+/// [`MIN_SECRET_KEY_LEN`]. The one rule for root, IAM credentials and the STS signing key.
+pub(crate) fn is_usable_secret(secret: &[u8]) -> bool {
+    effective_secret_len(secret) >= MIN_SECRET_KEY_LEN
+}
+
+/// Why [`RootCredentials::new`] refused its input. Carries no input text.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum RootCredentialsError {
+    /// The access key id is not one the gateway can hold.
+    InvalidAccessKey,
+    /// The root secret key fails the effective-length rule.
+    WeakSecretKey,
+    /// The STS token-signing key fails the effective-length rule.
+    WeakSigningKey,
+}
+
+impl core::fmt::Display for RootCredentialsError {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        f.write_str(match self {
+            Self::InvalidAccessKey => "root access key is not usable",
+            Self::WeakSecretKey => "root secret key is shorter than the minimum length",
+            Self::WeakSigningKey => "STS token-signing key is shorter than the minimum length",
+        })
+    }
+}
+
+impl std::error::Error for RootCredentialsError {}
+
+/// The root key pair and the STS token-signing key, validated together.
+///
+/// Implements no `Debug`; the secrets are readable only inside this crate.
+pub struct RootCredentials {
+    credentials: Credentials,
+    sts_signing_key: Box<str>,
+}
+
+impl RootCredentials {
+    /// Validates the root credentials and the key IAM signs STS session tokens with (today the
+    /// root secret, `rustfs_iam::manager::get_token_signing_key`; pass it separately so a future
+    /// key separation changes no adapter).
+    ///
+    /// # Errors
+    ///
+    /// [`RootCredentialsError`] when either secret fails the effective-length rule (fewer than
+    /// [`MIN_SECRET_KEY_LEN`] bytes once trailing NUL bytes are removed) or the access key id is not
+    /// one the gateway can hold.
+    pub fn new(access_key: &str, secret_key: &str, sts_signing_key: &str) -> Result<Self, RootCredentialsError> {
+        if !is_usable_secret(secret_key.as_bytes()) {
+            return Err(RootCredentialsError::WeakSecretKey);
+        }
+        if !is_usable_secret(sts_signing_key.as_bytes()) {
+            return Err(RootCredentialsError::WeakSigningKey);
+        }
+        let credentials =
+            Credentials::new(access_key, secret_key.as_bytes()).map_err(|_| RootCredentialsError::InvalidAccessKey)?;
+        Ok(Self {
+            credentials,
+            sts_signing_key: sts_signing_key.into(),
+        })
+    }
+
+    /// The root access key id, a public identifier.
+    #[must_use]
+    pub fn access_key(&self) -> &str {
+        self.credentials.identity().access_key_id()
+    }
+
+    pub(crate) fn credentials(&self) -> &Credentials {
+        &self.credentials
+    }
+
+    pub(crate) fn sts_signing_key(&self) -> &str {
+        &self.sts_signing_key
+    }
+}
+
+/// The live root credentials, read on every request by both adapters.
+///
+/// The binary implements it directly over RustFS's live root accessor, so a root change needs no
+/// second step. Any `Fn() -> Option<Arc<RootCredentials>>` is a source; for example, over the
+/// server context:
+///
+/// ```ignore
+/// let root = move || {
+///     let cred = ctx.action_credentials().get()?;
+///     let signing_key = ctx.token_signing_key()?;
+///     RootCredentials::new(&cred.access_key, &cred.secret_key, &signing_key).ok().map(Arc::new)
+/// };
+/// let adapters = IamAdapters::new(iam, Arc::new(root));
+/// ```
+///
+/// Root credentials that fail validation read as `None`, which both adapters answer as
+/// unavailable.
+pub trait RootSource: Send + Sync + 'static {
+    /// The current root credentials; `None` while they are not initialized, which both adapters
+    /// answer as unavailable rather than guessing.
+    fn current(&self) -> Option<Arc<RootCredentials>>;
+}
+
+impl<F> RootSource for F
+where
+    F: Fn() -> Option<Arc<RootCredentials>> + Send + Sync + 'static,
+{
+    fn current(&self) -> Option<Arc<RootCredentials>> {
+        self()
+    }
+}
+
+/// Test helper: a [`RootSource`] holding fixed root credentials. Production code reads the live
+/// accessor instead, so that there is no copy of root to keep in step.
+#[cfg(test)]
+pub(crate) struct RootCell(Option<Arc<RootCredentials>>);
+
+#[cfg(test)]
+impl RootCell {
+    pub(crate) fn new(root: Option<RootCredentials>) -> Self {
+        Self(root.map(Arc::new))
+    }
+}
+
+#[cfg(test)]
+impl RootSource for RootCell {
+    fn current(&self) -> Option<Arc<RootCredentials>> {
+        self.0.clone()
+    }
+}
+
+#[cfg(test)]
+#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
+mod tests {
+    use super::*;
+
+    /// Negative — an empty or short root secret is refused at construction, so no source can ever
+    /// hand out a root key anybody could sign with; eight bytes is accepted.
+    #[test]
+    fn root_short_secret_is_refused() {
+        for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0", "abcdefg\0"] {
+            let refused = RootCredentials::new("rootadmin", weak, "signing-key-0001");
+            assert!(matches!(refused, Err(RootCredentialsError::WeakSecretKey)), "{} bytes", weak.len());
+        }
+        assert!(RootCredentials::new("rootadmin", "12345678", "signing-key-0001").is_ok());
+    }
+
+    /// Positive — an interior NUL is followed by real key bytes and counts: `"abc\0defg"` has an
+    /// effective length of eight and is accepted, as is the same secret with trailing NULs.
+    #[test]
+    fn root_interior_nul_counts_toward_length() {
+        for secret in ["abc\0defg", "abc\0defg\0\0"] {
+            assert!(RootCredentials::new("rootadmin", secret, "signing-key-0001").is_ok(), "{secret:?}");
+            assert!(RootCredentials::new("rootadmin", "root-secret-key-0001", secret).is_ok(), "{secret:?}");
+        }
+    }
+
+    /// Negative — an empty or short STS signing key is refused at construction.
+    #[test]
+    fn root_short_signing_key_is_refused() {
+        for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0"] {
+            let refused = RootCredentials::new("rootadmin", "root-secret-key-0001", weak);
+            assert!(matches!(refused, Err(RootCredentialsError::WeakSigningKey)), "{} bytes", weak.len());
+        }
+    }
+}

crates/gateway-adapter/src/trace.rs

@@ -0,0 +1,230 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! The request id RustFS's outer layer minted, handed to the gateway as its own when that is safe.
+//!
+//! Responsible for: reading the id from the context the outer layer put in the request's
+//! extensions ([`RequestIdCarrier`]) and, when it is inside the gateway's identifier alphabet and
+//! length, inserting it as the gateway's [`HostRequestId`].
+//! NOT responsible for: minting any id, refusing any request (nothing here can fail a request),
+//! choosing the response headers (the gateway writes `x-amz-request-id` from the id it was
+//! handed, and RustFS's outer layer overwrites that header on S3 paths), or the layer that mints
+//! (`rustfs/src/server/layer.rs`). It never reads a header itself.
+//! Upstream: RustFS's `RequestContext`, which the binary makes a [`RequestIdCarrier`]. Downstream:
+//! the gateway service, which reads the [`HostRequestId`] extension once, at the top.
+//!
+//! # Whose id it is
+//!
+//! On S3 paths the outer layer mints a server-owned UUID. On admin, STS and console paths the
+//! context copies the caller's `x-request-id` (or `x-amz-request-id`), as legacy does, so there
+//! the id can be caller-chosen. Either way the id is attached only when it is inert, that is 1 to
+//! 64 bytes of ASCII letters, digits, `-`, `.`, `_` and `~`, which is safe in a header, an XML
+//! document and a log line. Anything else is not attached and the request is **forwarded
+//! unchanged**: the gateway then mints its own id. No request is ever refused because of its id.
+//!
+//! # Known difference when nothing is attached
+//!
+//! The gateway answers with the id it minted (error body `<RequestId>`, its events and audit
+//! records) while, on S3 paths, the outer layer overwrites the response header with RustFS's id.
+//! So for such a request the header and the error body name different ids. That needs a caller-
+//! chosen id outside the alphabet, or a request that never passed the outer layer.
+//!
+//! # Why this is not a `TraceSource`
+//!
+//! The gateway's `TraceSource::mint` takes no request, so a source cannot read the id RustFS
+//! already minted; the gateway's documented path for a host that identifies its own requests is
+//! the [`HostRequestId`] extension, and this module fills it.
+//!
+//! # Why a carrier trait
+//!
+//! `RequestContext` lives in the `rustfs` binary crate, which a library crate cannot depend on.
+//! The binary implements [`RequestIdCarrier`] for it, and this module looks it up in the
+//! extensions by that type.
+
+use rustfs_gateway::HostRequestId;
+
+/// A type in the request extensions that holds the id the outer layer minted or propagated.
+pub trait RequestIdCarrier: Send + Sync + 'static {
+    /// The canonical request id: the value RustFS logs and the response should carry.
+    fn request_id(&self) -> &str;
+}
+
+/// What [`attach_host_request_id`] did. Ignoring it is harmless: every outcome forwards the request.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum AttachOutcome {
+    /// The carrier's id was inserted as the request's [`HostRequestId`].
+    Attached,
+    /// The request carries no context from the outer layer; nothing was inserted.
+    NoContext,
+    /// The carrier's id is empty, too long, or outside the gateway's alphabet; nothing was
+    /// inserted and the id was not repaired.
+    UnusableId,
+}
+
+/// The gateway's [`HostRequestId`] for the id `carrier` holds, or `None` when it is not inert.
+#[must_use]
+pub fn host_request_id<C: RequestIdCarrier + ?Sized>(carrier: &C) -> Option<HostRequestId> {
+    HostRequestId::new(carrier.request_id()).ok()
+}
+
+/// Reads the `C` the outer layer put in `request`'s extensions and, when its id is inert, inserts
+/// it as the request's [`HostRequestId`].
+///
+/// Never refuses the request: the caller forwards it whatever the outcome.
+pub fn attach_host_request_id<C: RequestIdCarrier, B>(request: &mut http::Request<B>) -> AttachOutcome {
+    let Some(carrier) = request.extensions().get::<C>() else {
+        return AttachOutcome::NoContext;
+    };
+    let Some(id) = host_request_id(carrier) else {
+        return AttachOutcome::UnusableId;
+    };
+    request.extensions_mut().insert(id);
+    AttachOutcome::Attached
+}
+
+#[cfg(test)]
+#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
+mod tests {
+    use super::*;
+
+    #[derive(Clone)]
+    struct Ctx(String);
+
+    impl RequestIdCarrier for Ctx {
+        fn request_id(&self) -> &str {
+            &self.0
+        }
+    }
+
+    const UUID: &str = "7c9e6679-7425-40de-944b-e07fc1f90ae7";
+
+    fn request_with(id: &str) -> http::Request<()> {
+        let mut request = http::Request::new(());
+        request.extensions_mut().insert(Ctx(id.to_owned()));
+        request
+    }
+
+    fn handed_over(request: &http::Request<()>) -> Option<String> {
+        request
+            .extensions()
+            .get::<HostRequestId>()
+            .map(|id| id.request_id().as_str().to_owned())
+    }
+
+    fn attach(request: &mut http::Request<()>) -> AttachOutcome {
+        attach_host_request_id::<Ctx, _>(request)
+    }
+
+    /// Positive: the id RustFS minted is the id the gateway is handed, byte for byte.
+    #[test]
+    fn the_minted_id_is_handed_over_unchanged() {
+        let mut request = request_with(UUID);
+        assert_eq!(attach(&mut request), AttachOutcome::Attached);
+        assert_eq!(handed_over(&request).as_deref(), Some(UUID));
+    }
+
+    /// Positive: every shape RustFS mints is accepted, including its two fallbacks.
+    #[test]
+    fn every_shape_rustfs_mints_is_accepted() {
+        for id in [UUID, "trace-0123456789abcdef", "req-0123456789abcdef"] {
+            let mut request = request_with(id);
+            assert_eq!(attach(&mut request), AttachOutcome::Attached, "{id}");
+            assert_eq!(handed_over(&request).as_deref(), Some(id));
+        }
+    }
+
+    /// Positive: a caller-supplied id that is inert (admin and STS paths copy it) is attached, and
+    /// the longest allowed one is accepted.
+    #[test]
+    fn an_inert_caller_supplied_id_is_attached() {
+        let longest = "a".repeat(64);
+        for id in ["abc", "my.trace_id~1-2", longest.as_str()] {
+            let mut request = request_with(id);
+            assert_eq!(attach(&mut request), AttachOutcome::Attached, "{id}");
+            assert_eq!(handed_over(&request).as_deref(), Some(id));
+        }
+    }
+
+    /// Positive: two requests keep their own ids; nothing is shared or minted between them.
+    #[test]
+    fn each_request_keeps_its_own_id() {
+        let mut first = request_with("first-id");
+        let mut second = request_with("second-id");
+        attach(&mut first);
+        attach(&mut second);
+        assert_eq!(handed_over(&first).as_deref(), Some("first-id"));
+        assert_eq!(handed_over(&second).as_deref(), Some("second-id"));
+    }
+
+    /// Positive: with both a carrier and an `x-amz-request-id` header, the carrier's id wins.
+    #[test]
+    fn the_carrier_wins_over_a_request_id_header() {
+        let mut request = http::Request::builder()
+            .header("x-amz-request-id", "caller-chosen")
+            .header("x-request-id", "caller-chosen")
+            .body(())
+            .expect("a request");
+        request.extensions_mut().insert(Ctx(UUID.to_owned()));
+        assert_eq!(attach(&mut request), AttachOutcome::Attached);
+        assert_eq!(handed_over(&request).as_deref(), Some(UUID));
+    }
+
+    /// Negative: a request the outer layer never saw gets no id, and the request is not refused.
+    #[test]
+    fn a_missing_context_hands_over_nothing() {
+        let mut request = http::Request::new(());
+        assert_eq!(attach(&mut request), AttachOutcome::NoContext);
+        assert_eq!(handed_over(&request), None);
+    }
+
+    /// Negative: a header named like the id is not the id; only the context is read.
+    #[test]
+    fn a_request_id_header_alone_is_not_read() {
+        let mut request = http::Request::builder()
+            .header("x-amz-request-id", "caller-chosen")
+            .header("x-request-id", "caller-chosen")
+            .body(())
+            .expect("a request");
+        assert_eq!(attach(&mut request), AttachOutcome::NoContext);
+        assert_eq!(handed_over(&request), None);
+    }
+
+    /// Negative: an id outside the alphabet or the length is never attached or repaired, and the
+    /// outcome says so rather than failing the request.
+    #[test]
+    fn an_id_the_gateway_cannot_carry_is_not_attached() {
+        let too_long = "a".repeat(65);
+        for id in [
+            "",
+            "has space",
+            "line\r\nbreak",
+            "<RequestId>",
+            "quo\"te",
+            "Root=1-5759e988",
+            too_long.as_str(),
+        ] {
+            let mut request = request_with(id);
+            assert_eq!(attach(&mut request), AttachOutcome::UnusableId, "{id:?}");
+            assert_eq!(handed_over(&request), None, "{id:?}");
+        }
+    }
+
+    /// Negative: the unusable id is not repaired into a shorter or cleaned-up one.
+    #[test]
+    fn host_request_id_does_not_repair() {
+        assert!(host_request_id(&Ctx("has space".to_owned())).is_none());
+        assert!(host_request_id(&Ctx(format!("{UUID} "))).is_none());
+        assert!(host_request_id(&Ctx(UUID.to_owned())).is_some());
+    }
+}

crates/scanner/src/scanner_folder.rs

@@ -425,7 +425,10 @@ enum FolderScanSource {
 #[derive(Clone, Copy, PartialEq, Eq)]
 enum FolderScanMode {
     Namespace,
-    ErasureCandidate { reuse_namespace: bool },
+    ErasureCandidate {
+        reuse_namespace: bool,
+        parent_has_metadata: bool,
+    },
 }
 
 #[derive(Debug, PartialEq, Eq)]
@@ -2065,7 +2068,9 @@ impl FolderScanner {
                     if candidate_unclassified {
                         // Apply namespace reuse only after the current listing
                         // proves there is no direct erasure part.
-                        if let FolderScanMode::ErasureCandidate { reuse_namespace: true } = mode
+                        if let FolderScanMode::ErasureCandidate {
+                            reuse_namespace: true, ..
+                        } = mode
                             && !into.compacted
                             && self.old_cache.find(&folder.name).is_some()
                         {
@@ -2079,10 +2084,23 @@ impl FolderScanner {
                             return Ok(FolderScanOutcome::Namespace);
                         }
                         candidate_unclassified = false;
-                        if !self.budget.try_start_directory() {
-                            return Err(ScannerError::Other("Operation cancelled".to_string()));
+                        // An empty UUID directory beside object metadata has
+                        // no namespace work. Its listing was already charged;
+                        // nonempty namespaces still need directory admission.
+                        let empty_object_child = matches!(
+                            mode,
+                            FolderScanMode::ErasureCandidate {
+                                parent_has_metadata: true,
+                                ..
+                            }
+                        ) && deferred_metadata.is_none()
+                            && queued_folders.is_empty();
+                        if !empty_object_child {
+                            if !self.budget.try_start_directory() {
+                                return Err(ScannerError::Other("Operation cancelled".to_string()));
+                            }
+                            wait_for_scanner_io(&ctx, self.sleeper.sleep_folder_preset()).await?;
                         }
-                        wait_for_scanner_io(&ctx, self.sleeper.sleep_folder_preset()).await?;
                     }
                     let Some(entry) = deferred_metadata.take() else { break };
                     (entry, true)
@@ -2552,20 +2570,6 @@ impl FolderScanner {
                 return Err(ScannerError::Other("Operation cancelled".to_string()));
             }
 
-            if found_object_metadata && self.is_erasure_mode {
-                // If we found an object in erasure mode, we skip subdirs (only datadirs)...
-                debug!(
-                    target: "rustfs::scanner::folder",
-                    event = EVENT_SCANNER_FOLDER_STATE,
-                    component = LOG_COMPONENT_SCANNER,
-                    subsystem = LOG_SUBSYSTEM_FOLDER,
-                    folder = %folder.name,
-                    state = "erasure_object_found",
-                    "Scanner folder descent stopped after erasure object"
-                );
-                break;
-            }
-
             // Checkpointed sweeps need child entries to validate their resume
             // frontier (#8292). Outside that, also compact a wide bucket root:
             // excluding it left #8961's ~170k top-level prefixes unbounded until
@@ -2654,6 +2658,7 @@ impl FolderScanner {
                 let mut child_mode = if self.is_erasure_mode && is_erasure_candidate(&folder_item.name) {
                     FolderScanMode::ErasureCandidate {
                         reuse_namespace: reuse_covered,
+                        parent_has_metadata: found_object_metadata,
                     }
                 } else {
                     FolderScanMode::Namespace
@@ -2721,7 +2726,10 @@ impl FolderScanner {
                                 }
                                 // Sampling may skip namespace work, never the
                                 // classification that protects its parent object.
-                                child_mode = FolderScanMode::ErasureCandidate { reuse_namespace: true };
+                                child_mode = FolderScanMode::ErasureCandidate {
+                                    reuse_namespace: true,
+                                    parent_has_metadata: found_object_metadata,
+                                };
                             } else {
                                 folder_item.object_heal_prob_div = data_usage_update_dir_cycles();
                             }
@@ -2816,6 +2824,11 @@ impl FolderScanner {
                         updates.cache.delete_recursive(&h);
                     }
                     into.children.remove(&clone_cache_value(&h.0));
+                    // Object keys can also be prefixes. A classified payload
+                    // directory must not hide their other namespace children.
+                    if found_object_metadata {
+                        continue;
+                    }
                     // Earlier namespace siblings may have completed, but none
                     // can certify coverage of this unresolved parent object.
                     self.coverage_gap |= forward_sweep;

crates/scanner/src/scanner_folder/tests/erasure_probe.rs

@@ -183,6 +183,117 @@ fn charged_ops(root: &std::path::Path) -> u64 {
     status.charged_ops
 }
 
+#[tokio::test]
+#[serial]
+async fn erasure_object_metadata_preserves_child_object_namespaces() {
+    for (corrupt_parent, cached_parent) in [(false, false), (true, false), (false, true)] {
+        let (mut scanner, root) = build_test_scanner().await;
+        let _guard = TestGuard::new(60, 100, &mut scanner, root.clone());
+        scanner.old_cache.info.name = "bucket".into();
+        scanner.new_cache.info.name = "bucket".into();
+        scanner.is_erasure_mode = true;
+        bind_budget(&mut scanner, &root, 1_000_000);
+
+        let namespace = format!("parent/{}/nested", Uuid::new_v4());
+        for object in ["parent", "parent/child", namespace.as_str()] {
+            write_test_object_metadata_bytes(&root, "bucket", object, &metadata_for_object_version("bucket", object, None)).await;
+        }
+        let payload = root.join("bucket/parent").join(Uuid::new_v4().to_string());
+        tokio::fs::create_dir_all(&payload)
+            .await
+            .expect("create parent payload directory");
+        tokio::fs::write(payload.join("part.1"), b"shard")
+            .await
+            .expect("create actual payload evidence");
+        if corrupt_parent {
+            tokio::fs::write(root.join("bucket/parent/xl.meta"), b"")
+                .await
+                .expect("corrupt only the parent metadata");
+        }
+        if cached_parent {
+            scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
+            scanner.old_cache.replace(
+                "bucket/parent",
+                "bucket",
+                DataUsageEntry {
+                    objects: 1,
+                    size: 1,
+                    compacted: true,
+                    ..Default::default()
+                },
+            );
+            let interval = data_usage_update_dir_cycles();
+            scanner.old_cache.info.next_cycle = u64::from(
+                (0..interval)
+                    .find(|cycle| hash_path("bucket/parent").mod_(*cycle, interval))
+                    .expect("one normal sampling cycle revisits the old parent-only cache"),
+            );
+        }
+
+        let mut into = DataUsageEntry::default();
+        scanner
+            .scan_folder(
+                CancellationToken::new(),
+                CachedFolder {
+                    name: "bucket".into(),
+                    parent: None,
+                    object_heal_prob_div: 1,
+                },
+                &mut into,
+            )
+            .await
+            .expect("scan object and child namespaces independently");
+        let total = scanner.new_cache.size_recursive("bucket").expect("retain bucket accounting");
+        assert_eq!(total.objects, if corrupt_parent { 2 } else { 3 });
+        assert_eq!(total.size, if corrupt_parent { 2 } else { 3 });
+        assert_eq!(total.failed_objects, usize::from(corrupt_parent));
+        assert_eq!(
+            scanner.new_cache.info.failed_objects.len(),
+            usize::from(corrupt_parent),
+            "payload evidence must not invent missing parent metadata"
+        );
+    }
+}
+
+#[tokio::test]
+#[serial]
+async fn erasure_object_prefix_children_keep_the_object_budget() {
+    let (mut scanner, root) = build_test_scanner().await;
+    let _guard = TestGuard::new(60, 100, &mut scanner, root.clone());
+    scanner.old_cache.info.name = "bucket".into();
+    scanner.new_cache.info.name = "bucket".into();
+    scanner.is_erasure_mode = true;
+    bind_budget(&mut scanner, &root, 1_000_000);
+    for object in ["parent", "parent/child", "parent/other"] {
+        write_test_object_metadata(&root, "bucket", object).await;
+    }
+    let visits = metadata_visits::observe(&mut scanner);
+    let parent = CancellationToken::new();
+    let budget = ScannerCycleBudget::new_with_progress_tracking(
+        &parent,
+        crate::scanner_budget::ScannerCycleBudgetConfig {
+            max_objects: Some(2),
+            ..Default::default()
+        },
+    );
+    scanner.budget = budget.clone();
+    let result = scanner
+        .scan_folder(
+            budget.token(),
+            CachedFolder {
+                name: "bucket".into(),
+                parent: None,
+                object_heal_prob_div: 1,
+            },
+            &mut DataUsageEntry::default(),
+        )
+        .await;
+    assert!(result.is_err(), "parent metadata cannot hide children or bypass their budget");
+    assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects));
+    assert_eq!(budget.progress().0, 2);
+    assert_eq!(visits.load(Ordering::Relaxed), 2);
+}
+
 #[tokio::test]
 #[serial]
 async fn erasure_directory_probe_reaches_eof_across_budgeted_batches() {
@@ -204,7 +315,10 @@ async fn erasure_directory_probe_reaches_eof_across_budgeted_batches() {
                 ctx,
                 folder,
                 &mut DataUsageEntry::default(),
-                FolderScanMode::ErasureCandidate { reuse_namespace: false },
+                FolderScanMode::ErasureCandidate {
+                    reuse_namespace: false,
+                    parent_has_metadata: false,
+                },
                 false,
             )
             .await
@@ -271,7 +385,10 @@ async fn scanner_directory_probe_budget_wait_cancels_without_rate_delay() {
             ctx.clone(),
             folder,
             &mut DataUsageEntry::default(),
-            FolderScanMode::ErasureCandidate { reuse_namespace: false },
+            FolderScanMode::ErasureCandidate {
+                reuse_namespace: false,
+                parent_has_metadata: false,
+            },
             false,
         )
         .await;
@@ -311,7 +428,10 @@ async fn scanner_directory_probe_cancels_during_entry_budget_wait() {
             ctx.clone(),
             folder,
             &mut DataUsageEntry::default(),
-            FolderScanMode::ErasureCandidate { reuse_namespace: false },
+            FolderScanMode::ErasureCandidate {
+                reuse_namespace: false,
+                parent_has_metadata: false,
+            },
             false,
         )
         .await;
@@ -382,7 +502,16 @@ async fn erasure_candidate_defers_metadata_until_part_classification_finishes()
     scanner.budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, Default::default());
     let mut into = DataUsageEntry::default();
     let outcome = scanner
-        .scan_folder_inner(ctx, folder, &mut into, FolderScanMode::ErasureCandidate { reuse_namespace: false }, false)
+        .scan_folder_inner(
+            ctx,
+            folder,
+            &mut into,
+            FolderScanMode::ErasureCandidate {
+                reuse_namespace: false,
+                parent_has_metadata: false,
+            },
+            false,
+        )
         .await
         .expect("classify candidate");
     assert_eq!(outcome, FolderScanOutcome::ErasureData);
@@ -594,7 +723,10 @@ async fn erasure_candidate_part_symlink_is_not_data_evidence() {
             CancellationToken::new(),
             folder,
             &mut DataUsageEntry::default(),
-            FolderScanMode::ErasureCandidate { reuse_namespace: false },
+            FolderScanMode::ErasureCandidate {
+                reuse_namespace: false,
+                parent_has_metadata: false,
+            },
             false,
         )
         .await

docs/architecture/s3-compatibility-matrix.md

@@ -130,7 +130,7 @@ These behaviors are client-visible and differ from AWS S3 or MinIO. They bound h
 | Bucket tag key or value outside the AWS tag character class (`[\p{L}\p{N}\p{Z}+\-=._:/@]`) | `400 InvalidTag` on `PutBucketTagging` | AWS enforces the same class; MinIO is more permissive | AWS parity. Legacy stored tags still load. |
 | `CopyObject` with the default `x-amz-tagging-directive: COPY` from a source that carries tags | `403 AccessDenied` unless the caller also holds `s3:GetObjectTagging` (or `s3:GetObjectVersionTagging`) on the source; the destination's `s3:RequestObjectTag/*` conditions are evaluated against the copied tag set | AWS and MinIO copy the tags with `s3:GetObject` alone | A copy would otherwise disclose source tags to a principal that may not read them, and source tags could satisfy destination conditions the caller cannot assert (rustfs/backlog#3034). Use `REPLACE` with an explicit `x-amz-tagging` to copy without the tagging grant. |
 | `ListObjectVersions` / `ListObjectsV2` with the include-deleted extension | Requires `s3:ListBucketVersions` | Vendor extension | Delete markers and non-current versions are version-listing data. |
-| `DeleteObject` of a missing key that names a prefix | `204 NoContent` when the recovery probe of the prefix fits the request budget (`DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES`, 4096 entries per drive, one entry per 64 plain children); `503 SlowDown` beyond that, deterministically, and for the duration of a rolling upgrade while a drive in the set still runs a release without the bounded recovery RPC | `204 NoContent` | The probe looks for interrupted delete transactions under the prefix and is bounded so a hostile prefix cannot pin the request. Prefixes with more than 262,144 direct children per drive, or roughly 1,400 to 4,096 UUID-named children, exceed the bound (rustfs/backlog#3038 residual). |
+| `DeleteObject` of a missing key that names a prefix | `204 NoContent` when the recovery probe of the prefix fits its per-drive allowance (`DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES`, 262,144 entries: one per UUID-named direct child, one per 64 other names), its per-call time limit (a third of the drive timeout, at most 10 seconds) and the request budget (`DeleteRecoveryWorkBudget`, shared by every key and drive of the request); `503 SlowDown` beyond that, and for the duration of a rolling upgrade while a drive in the set still runs a release without the bounded recovery RPC | `204 NoContent` | The probe looks for interrupted delete transactions of the key and is bounded so a hostile prefix cannot pin the request. A folder completes with up to 262,144 UUID-named direct children or about 16.7 million other direct children per drive when the drive can walk them within the time limit (a warm cache walks roughly 25,000 UUID-named children per second; a cold HDD far fewer); inside a UUID-named child only journal files and data-directory markers are examined, so a child key's own `xl.meta`, data directories or children cost 1/64 each. |
 | `PutObject` / `UploadPart` body delivering less than 64 KiB per 60 seconds of server wait | `400 RequestTimeout` (the minimum-progress floor, logged as `upload_body_progress_stalled`); independent of `RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT`, including when that timeout is 0 | No minimum upload rate | A trickle upload otherwise pins upload admission and its buffers indefinitely; roughly 1.1 KiB/s is below any practical client. |
 | First write after `SetBucketQuota`, or after the quota changes, on a bucket without an authoritative usage ledger | `503 SlowDown` (retryable) until a background scan of the bucket publishes the exact ledger; a failed scan retries after 30 seconds | AWS has no bucket quota; MinIO admits with unknown usage | Quota admission uses only the authoritative ledger (scanner reporting is informational, see `reporting_retirement` tests), and the request must not hold its bucket metadata guard across a whole-bucket listing. The window is one bucket listing long. |
 

docs/testing/ci-gates.md

@@ -101,6 +101,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run. The v
 | `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes (disabled) | `make coverage` |
 | `runner-hygiene.yml` (monthly) | `check-ephemerality` | runner ephemerality | yes | dispatch |
 | `e2e-upgrade.yml` (weekly, and on version tags) | `upgrade` (serial cases sharing one harness build) | upgrade and rollback gate; server logs | yes | `python3 scripts/e2e_binary.py build`, then the job's `python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test` commands with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
+| `reference-parity.yml` (nightly, and on PRs that change the lane's own files) | `plan`, `build-candidate`, `s3tests` (one job per target in `.config/reference-parity-targets.toml`), `compare` | report-only parity gate: fails on any per-node-id s3-tests difference between a branch candidate and the release pinned in `.config/reference-binary.toml` that `.config/reference-parity-tolerated.txt` does not list, and on a stale tolerated entry; per-target JUnit, logs and `diff.md`/`diff.json` | yes | `DEPLOY_MODE=binary RUSTFS_BINARY=<binary> MAXFAIL=0 XDIST=4 ./scripts/s3-tests/run.sh` once per target, then `python3 scripts/s3-tests/reference_parity.py compare` as in the `compare` job |
 | `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | yes | see the PR row |
 | `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | yes (disabled) | see the PR row |
 | `gateway-integration-sync.yml` (weekdays) | `sync` | merges `main` into `gateway/integration` as a `--no-ff` merge commit; a conflicting merge is aborted and opens a `sync/<source>-<date>` pull request listing the conflicting files, which keeps the run green; a rejected push or missing pull request fails it | yes | dispatch, optionally with `source_ref` |

rustfs/src/app/bucket_list_through.rs

@@ -843,7 +843,7 @@ mod tests {
         };
         tokio::time::timeout(
             Duration::from_secs(10),
-            DefaultBucketUsecase::from_global().execute_list_objects_v2(request),
+            crate::app::test_seams::list_objects_v2(&DefaultBucketUsecase::from_global(), request),
         )
         .await
         .expect("listing must complete within its bounded source budget")
@@ -1241,19 +1241,23 @@ mod tests {
                             let mut list = get.clone().map_input(|_| input.clone());
                             list.uri = http::Uri::from_static("/?list-type=2");
                             let usecase = crate::app::object::DefaultObjectUsecase::from_global();
-                            let get_error = tokio::time::timeout(Duration::from_secs(10), usecase.execute_get_object(get))
-                                .await
-                                .expect("GET stays local")
-                                .expect_err("local object is absent");
+                            let get_error =
+                                tokio::time::timeout(Duration::from_secs(10), crate::app::test_seams::get_object(&usecase, get))
+                                    .await
+                                    .expect("GET stays local")
+                                    .expect_err("local object is absent");
                             assert_eq!(*get_error.code(), S3ErrorCode::NoSuchKey);
-                            let head_error = tokio::time::timeout(Duration::from_secs(10), usecase.execute_head_object(head))
-                                .await
-                                .expect("HEAD stays local")
-                                .expect_err("local object is absent");
+                            let head_error = tokio::time::timeout(
+                                Duration::from_secs(10),
+                                crate::app::test_seams::head_object(&usecase, head),
+                            )
+                            .await
+                            .expect("HEAD stays local")
+                            .expect_err("local object is absent");
                             assert_eq!(*head_error.code(), S3ErrorCode::NoSuchKey);
                             let listing = tokio::time::timeout(
                                 Duration::from_secs(10),
-                                DefaultBucketUsecase::from_global().execute_list_objects_v2(list),
+                                crate::app::test_seams::list_objects_v2(&DefaultBucketUsecase::from_global(), list),
                             )
                             .await
                             .expect("LIST stays local")

rustfs/src/app/lifecycle_transition_api_test/common.rs

@@ -0,0 +1,384 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Shared fixtures for the lifecycle integration tests; owned by no operation batch.
+
+use super::*;
+
+static GLOBAL_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>)> = OnceLock::new();
+
+pub(super) const TRANSITION_WAIT_TIMEOUT: Duration = Duration::from_secs(15);
+
+pub(super) const RUSTFS_EXPECTED_CURRENT_VERSION_ID: &str = "x-rustfs-expected-current-version-id";
+
+fn init_tracing() {
+    rustfs_test_utils::init_tracing();
+}
+
+pub(super) async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
+    init_tracing();
+
+    if let Some((paths, ecstore)) = GLOBAL_ENV.get() {
+        return (paths.clone(), ecstore.clone());
+    }
+
+    // Production startup installs this proof through the notification-system
+    // fleet probe. This isolated single-node harness constructs ECStore
+    // directly, so model that completed all-v6 probe before any journal permit
+    // or background expiry worker can exist.
+    install_all_v6_fleet_capability_proof();
+
+    let test_base_dir = format!("/tmp/rustfs_app_lifecycle_test_{}", Uuid::new_v4());
+    let temp_dir = PathBuf::from(&test_base_dir);
+    if temp_dir.exists() {
+        fs::remove_dir_all(&temp_dir).await.ok();
+    }
+    fs::create_dir_all(&temp_dir).await.unwrap();
+
+    let disk_paths = vec![
+        temp_dir.join("disk1"),
+        temp_dir.join("disk2"),
+        temp_dir.join("disk3"),
+        temp_dir.join("disk4"),
+    ];
+
+    for disk_path in &disk_paths {
+        fs::create_dir_all(disk_path).await.unwrap();
+    }
+
+    let mut endpoints = Vec::new();
+    for (i, disk_path) in disk_paths.iter().enumerate() {
+        let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
+        endpoint.set_pool_index(0);
+        endpoint.set_set_index(0);
+        endpoint.set_disk_index(i);
+        endpoints.push(endpoint);
+    }
+
+    let pool_endpoints = PoolEndpoints {
+        legacy: false,
+        set_count: 1,
+        drives_per_set: 4,
+        endpoints: Endpoints::from(endpoints),
+        cmd_line: "test".to_string(),
+        platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
+    };
+
+    let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
+
+    super::super::storage_api::test::runtime::init_local_disks(endpoint_pools.clone())
+        .await
+        .unwrap();
+
+    let server_addr: std::net::SocketAddr = "127.0.0.1:9003".parse().unwrap();
+    let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
+        .await
+        .unwrap();
+
+    let buckets_list = ecstore
+        .list_bucket(&BucketOptions {
+            no_metadata: true,
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+    let buckets = buckets_list.into_iter().map(|v| v.name).collect();
+    metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
+
+    lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
+
+    // The usecases resolve the object store through the ambient AppContext
+    // since the backlog#1052 InstanceContext migration; without one installed
+    // every store-touching usecase call fails with InternalError "Not init".
+    // Publish a default-interface context wrapping this test store.
+    crate::app::runtime_sources::install_test_app_context(ecstore.clone()).await;
+
+    let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
+
+    (disk_paths, ecstore)
+}
+
+pub(super) async fn create_test_bucket(ecstore: &Arc<ECStore>, bucket_name: &str) {
+    (**ecstore)
+        .make_bucket(
+            bucket_name,
+            &MakeBucketOptions {
+                versioning_enabled: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("Failed to create test bucket");
+}
+
+pub(super) async fn upload_test_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str, data: &[u8]) -> ObjectInfo {
+    let mut reader = PutObjReader::from_vec(data.to_vec());
+    (**ecstore)
+        .put_object(bucket, object, &mut reader, &ObjectOptions::default())
+        .await
+        .expect("Failed to upload test object")
+}
+
+pub(super) async fn set_bucket_lifecycle_transition_with_tier(
+    bucket_name: &str,
+    storage_class: &str,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let lifecycle_xml = format!(
+        r#"<?xml version="1.0" encoding="UTF-8"?>
+<LifecycleConfiguration>
+    <Rule>
+        <ID>test-rule</ID>
+        <Status>Enabled</Status>
+        <Filter>
+            <Prefix>test/</Prefix>
+        </Filter>
+        <Transition>
+          <Days>0</Days>
+          <StorageClass>{storage_class}</StorageClass>
+        </Transition>
+    </Rule>
+</LifecycleConfiguration>"#
+    );
+
+    metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
+    Ok(())
+}
+
+pub(super) async fn set_bucket_object_lock_enabled(bucket_name: &str) -> Result<(), Box<dyn std::error::Error>> {
+    let object_lock_xml = r#"<?xml version="1.0" encoding="UTF-8"?>
+<ObjectLockConfiguration>
+    <ObjectLockEnabled>Enabled</ObjectLockEnabled>
+</ObjectLockConfiguration>"#;
+    metadata_sys::update(bucket_name, OBJECT_LOCK_CONFIG, object_lock_xml.as_bytes().to_vec()).await?;
+    Ok(())
+}
+
+/// Register the shared [`MockWarmBackend`] into this instance's tier config
+/// manager. Thin wrapper over the shared `register_mock_tier` helper
+/// (rustfs/backlog#1148 ilm-6).
+pub(super) async fn register_mock_tier(tier_name: &str) -> MockWarmBackend {
+    register_mock_tier_util(&current_tier_config_handle(), tier_name).await
+}
+
+pub(super) async fn wait_for_transition(
+    ecstore: &Arc<ECStore>,
+    bucket: &str,
+    object: &str,
+    timeout: Duration,
+) -> Result<ObjectInfo, String> {
+    let deadline = tokio::time::Instant::now() + timeout;
+
+    loop {
+        let observed = (**ecstore).get_object_info(bucket, object, &ObjectOptions::default()).await;
+        let observed = match observed {
+            Ok(info) if info.transitioned_object.status == "complete" => return Ok(info),
+            other => other,
+        };
+
+        if tokio::time::Instant::now() >= deadline {
+            let last_observation = match observed {
+                Ok(info) => format!("transitioned_status={:?}", info.transitioned_object.status),
+                Err(error) => format!("metadata_error={:?}", error.code()),
+            };
+            let state = get_global_transition_state();
+            return Err(format!(
+                "transition did not complete within {timeout:?}; {last_observation}; pending={}, active={}, compensation_pending={}, compensation_running={}",
+                state.pending_tasks(),
+                state.active_tasks(),
+                state.compensation_pending_tasks(),
+                state.compensation_running_tasks(),
+            ));
+        }
+
+        tokio::time::sleep(Duration::from_millis(50)).await;
+    }
+}
+
+// SAFETY: this helper is used only by `#[serial]` tests and runs under the single-threaded Tokio
+// runtime (`worker_threads = 1`), so no concurrent test can mutate process environment during the
+// `env::set_var` / `env::remove_var` window.
+#[allow(unsafe_code)]
+pub(super) async fn with_forced_immediate_enqueue_timeout<F, Fut>(backend: &MockWarmBackend, test_fn: F)
+where
+    F: FnOnce() -> Fut,
+    Fut: std::future::Future<Output = ()>,
+{
+    let transition_state = get_global_transition_state();
+    let missed_before = transition_state.missed_immediate_tasks();
+    let compensation_before = transition_state.compensation_scheduled_tasks();
+    let put_barrier = backend.arm_put_barrier().await;
+    let original = env::var_os(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT);
+    unsafe {
+        env::set_var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, "1");
+    }
+    let result = std::panic::AssertUnwindSafe(test_fn()).catch_unwind().await;
+    match original {
+        Some(value) => unsafe {
+            env::set_var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, value);
+        },
+        None => unsafe {
+            env::remove_var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT);
+        },
+    }
+    if let Err(err) = result {
+        std::panic::resume_unwind(err);
+    }
+
+    assert!(
+        transition_state.missed_immediate_tasks() > missed_before,
+        "the operation should exercise the forced immediate enqueue failure"
+    );
+    assert!(
+        transition_state.compensation_scheduled_tasks() > compensation_before,
+        "the operation should schedule automatic compensation backfill"
+    );
+
+    // Separate automatic discovery and queue latency from the metadata commit deadline.
+    put_barrier.wait_until_paused().await;
+    assert_eq!(backend.object_count().await, 1, "compensation should store the body before committing");
+    put_barrier.release();
+}
+
+pub(super) async fn wait_for_delete_marker(ecstore: &Arc<ECStore>, bucket: &str, object: &str, timeout: Duration) -> bool {
+    let deadline = tokio::time::Instant::now() + timeout;
+
+    loop {
+        if let Ok(info) = ecstore.get_object_info(bucket, object, &ObjectOptions::default()).await
+            && info.delete_marker
+        {
+            return true;
+        }
+
+        if tokio::time::Instant::now() >= deadline {
+            return false;
+        }
+
+        tokio::time::sleep(Duration::from_millis(50)).await;
+    }
+}
+
+pub(super) fn build_request<T>(input: T, method: Method) -> S3Request<T> {
+    S3Request {
+        input,
+        method,
+        uri: Uri::from_static("/"),
+        headers: HeaderMap::new(),
+        extensions: Extensions::new(),
+        credentials: None,
+        region: None,
+        service: None,
+        trailing_headers: None,
+    }
+}
+
+pub(super) fn streaming_blob_from_bytes(data: &[u8]) -> StreamingBlob {
+    let body = Bytes::copy_from_slice(data);
+    StreamingBlob::wrap::<_, Infallible>(stream::once(async move { Ok(body) }))
+}
+
+pub(super) async fn read_object_bytes(ecstore: &Arc<ECStore>, bucket: &str, object: &str) -> Vec<u8> {
+    let mut reader = (**ecstore)
+        .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+        .await
+        .expect("Failed to read object");
+    let mut buf = Vec::new();
+    reader
+        .stream
+        .read_to_end(&mut buf)
+        .await
+        .expect("Failed to drain object reader");
+    buf
+}
+
+pub(super) async fn live_object_version_count(ecstore: &Arc<ECStore>, bucket: &str, object: &str) -> usize {
+    let versions = ecstore
+        .clone()
+        .list_object_versions(bucket, object, None, None, None, 1000)
+        .await
+        .expect("Failed to list object versions");
+
+    versions
+        .objects
+        .iter()
+        .filter(|info| info.name == object && !info.delete_marker)
+        .count()
+}
+
+pub(super) async fn install_delete_replication_config(
+    bucket: &str,
+    delete_enabled: bool,
+    replica_modifications: bool,
+    required_tag: Option<(&str, &str)>,
+) {
+    use super::super::storage_api::test::{bucket::utils::serialize, get_global_bucket_metadata_sys, set_bucket_metadata};
+
+    let sys = get_global_bucket_metadata_sys().expect("bucket metadata system should be initialized");
+    let metadata = {
+        let sys = sys.read().await;
+        sys.get(bucket)
+            .await
+            .expect("bucket metadata should be cached before replication config injection")
+    };
+    let mut metadata = (*metadata).clone();
+    let target = "arn:aws:s3:::target-bucket";
+    let delete_status = if delete_enabled {
+        Status::ENABLED.to_string()
+    } else {
+        Status::DISABLED.to_string()
+    };
+
+    metadata.versioning_config_xml = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec();
+    metadata.versioning_config = Some(rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
+        status: Some("Enabled".to_owned()),
+        ..Default::default()
+    });
+    let filter = required_tag.map(|(key, value)| PersistedReplicationFilter {
+        tag: Some(PersistedReplicationTag {
+            key: Some(key.to_string()),
+            value: Some(value.to_string()),
+        }),
+        ..Default::default()
+    });
+    let source_selection_criteria = replica_modifications.then(|| PersistedSourceSelectionCriteria {
+        replica_modifications: Some(PersistedReplicationStatus {
+            status: Status::ENABLED.to_string(),
+        }),
+        sse_kms_encrypted_objects: None,
+    });
+    let replication_config = PersistedReplicationConfiguration {
+        role: String::new(),
+        rules: vec![PersistedReplicationRule {
+            delete_marker_replication: Some(PersistedOptionalReplicationStatus {
+                status: Some(Status::DISABLED.to_string()),
+            }),
+            delete_replication: Some(PersistedReplicationStatus { status: delete_status }),
+            destination: PersistedReplicationDestination {
+                bucket: target.to_string(),
+                ..Default::default()
+            },
+            existing_object_replication: None,
+            filter,
+            id: Some("delete".to_string()),
+            prefix: Some(String::new()),
+            priority: Some(1),
+            source_selection_criteria,
+            status: Status::ENABLED.to_string(),
+        }],
+    };
+    metadata.replication_config_xml = serialize(&replication_config).expect("replication test config should serialize");
+    metadata.replication_config = Some(replication_config);
+    set_bucket_metadata(bucket.to_string(), metadata)
+        .await
+        .expect("delete replication metadata should be installed for the request test");
+}

rustfs/src/app/lifecycle_transition_api_test/copy.rs

@@ -0,0 +1,654 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! CopyObject tests, including tests that also PUT (batch E owns CopyObject).
+
+use super::common::*;
+use super::*;
+use crate::app::test_seams;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn copy_object_if_none_match_existing_destination_returns_precondition_failed() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+
+    let src_bucket = format!("test-copy-if-none-match-src-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let dst_bucket = format!("test-copy-if-none-match-dst-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let src_object = "test/source.txt";
+    let dst_object = "test/destination.txt";
+    let src_payload = b"conditional copy source payload";
+    let dst_payload = b"conditional copy destination payload";
+
+    create_test_bucket(&ecstore, src_bucket.as_str()).await;
+    create_test_bucket(&ecstore, dst_bucket.as_str()).await;
+    let _ = upload_test_object(&ecstore, src_bucket.as_str(), src_object, src_payload).await;
+    let _ = upload_test_object(&ecstore, dst_bucket.as_str(), dst_object, dst_payload).await;
+
+    let dst_info = ecstore
+        .get_object_info(dst_bucket.as_str(), dst_object, &ObjectOptions::default())
+        .await
+        .expect("Failed to fetch destination object info");
+    let dst_etag = dst_info.etag.expect("destination object should have an ETag");
+    let initial_versions = live_object_version_count(&ecstore, dst_bucket.as_str(), dst_object).await;
+
+    for (condition, expected_code) in [
+        (ETagCondition::Any, S3ErrorCode::PreconditionFailed),
+        (ETagCondition::ETag(ETag::Strong(dst_etag)), S3ErrorCode::NotImplemented),
+        (
+            ETagCondition::ETag(ETag::Strong("different-etag".to_string())),
+            S3ErrorCode::NotImplemented,
+        ),
+    ] {
+        let header = condition.to_http_header().expect("conditional COPY header should encode");
+        // Cover header-only callers and the DTO-plus-header shape of parsed HTTP requests.
+        for if_none_match in [None, Some(condition)] {
+            let has_dto = if_none_match.is_some();
+            let copy_input = CopyObjectInput::builder()
+                .copy_source(CopySource::Bucket {
+                    bucket: src_bucket.clone().into(),
+                    key: src_object.to_string().into(),
+                    version_id: None,
+                })
+                .bucket(dst_bucket.clone())
+                .key(dst_object.to_string())
+                .if_none_match(if_none_match)
+                .build()
+                .expect("conditional COPY input should build");
+            let mut req = build_request(copy_input, Method::PUT);
+            req.headers.insert(IF_NONE_MATCH, header.clone());
+
+            let err = Box::pin(usecase.execute_copy_object(req))
+                .await
+                .expect_err("conditional COPY must reject the replacement");
+            assert_eq!(err.code(), &expected_code, "header={header:?}, has_dto={has_dto}");
+            assert_eq!(
+                read_object_bytes(&ecstore, dst_bucket.as_str(), dst_object).await,
+                dst_payload,
+                "failed conditional CopyObject must not overwrite the current destination object"
+            );
+            assert_eq!(
+                live_object_version_count(&ecstore, dst_bucket.as_str(), dst_object).await,
+                initial_versions,
+                "failed conditional CopyObject must not create a version"
+            );
+        }
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state object-version integration test: runs serialized in the CI ILM Integration (serial) lane"]
+async fn undo_copy_requires_the_observed_current_version() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let fs = FS::new();
+    let usecase = DefaultObjectUsecase::from_global();
+    let bucket = format!("test-undo-copy-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/object.txt";
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+
+    let put = |payload: &'static [u8]| {
+        PutObjectInput::builder()
+            .bucket(bucket.clone())
+            .key(object.to_string())
+            .body(Some(streaming_blob_from_bytes(payload)))
+            .content_length(Some(payload.len() as i64))
+            .build()
+            .unwrap()
+    };
+    let original = Box::pin(test_seams::put_object(&usecase, &fs, build_request(put(b"original"), Method::PUT)))
+        .await
+        .expect("initial version should be written")
+        .output
+        .version_id
+        .expect("versioned PUT should return a version ID");
+    let observed_current = Box::pin(test_seams::put_object(&usecase, &fs, build_request(put(b"replacement"), Method::PUT)))
+        .await
+        .expect("replacement version should be written")
+        .output
+        .version_id
+        .expect("versioned PUT should return a version ID");
+
+    let copy = || {
+        CopyObjectInput::builder()
+            .copy_source(CopySource::Bucket {
+                bucket: bucket.clone().into(),
+                key: object.to_string().into(),
+                version_id: Some(original.clone().into()),
+            })
+            .bucket(bucket.clone())
+            .key(object.to_string())
+            .build()
+            .unwrap()
+    };
+    let mut stale_req = build_request(copy(), Method::PUT);
+    stale_req.headers.insert(
+        RUSTFS_EXPECTED_CURRENT_VERSION_ID,
+        HeaderValue::from_str(&Uuid::new_v4().to_string()).unwrap(),
+    );
+    let versions_before = live_object_version_count(&ecstore, bucket.as_str(), object).await;
+    let err = Box::pin(usecase.execute_copy_object(stale_req))
+        .await
+        .expect_err("stale destination must fail");
+    assert_eq!(err.code(), &S3ErrorCode::PreconditionFailed);
+    assert_eq!(live_object_version_count(&ecstore, bucket.as_str(), object).await, versions_before);
+    assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), object).await, b"replacement");
+
+    let mut matching_req = build_request(copy(), Method::PUT);
+    matching_req
+        .headers
+        .insert(RUSTFS_EXPECTED_CURRENT_VERSION_ID, HeaderValue::from_str(&observed_current).unwrap());
+    Box::pin(usecase.execute_copy_object(matching_req))
+        .await
+        .expect("matching destination version should permit undo copy");
+    assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), object).await, b"original");
+    assert_eq!(live_object_version_count(&ecstore, bucket.as_str(), object).await, versions_before + 1);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+#[ignore = "global-state object-version integration test: runs serialized in the CI ILM Integration (serial) lane"]
+async fn undo_copy_serializes_a_concurrent_put_until_after_commit() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let bucket = format!("test-undo-race-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/object.txt";
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+
+    let put = |payload: &'static [u8]| {
+        PutObjectInput::builder()
+            .bucket(bucket.clone())
+            .key(object.to_string())
+            .body(Some(streaming_blob_from_bytes(payload)))
+            .content_length(Some(payload.len() as i64))
+            .build()
+            .unwrap()
+    };
+    let fs = FS::new();
+    let usecase = DefaultObjectUsecase::from_global();
+    let original = Box::pin(test_seams::put_object(&usecase, &fs, build_request(put(b"original"), Method::PUT)))
+        .await
+        .expect("original should be written")
+        .output
+        .version_id
+        .expect("original should be versioned");
+    let observed = Box::pin(test_seams::put_object(&usecase, &fs, build_request(put(b"observed"), Method::PUT)))
+        .await
+        .expect("observed should be written")
+        .output
+        .version_id
+        .expect("observed should be versioned");
+    let encoded_object = encode_dir_object(object);
+    let lock = ecstore
+        .new_ns_lock(bucket.as_str(), &encoded_object)
+        .await
+        .expect("namespace lock should be available");
+    let blocker = lock
+        .get_write_lock(Duration::from_secs(10))
+        .await
+        .expect("test should acquire blocker");
+
+    let mut undo_req = build_request(
+        CopyObjectInput::builder()
+            .copy_source(CopySource::Bucket {
+                bucket: bucket.clone().into(),
+                key: object.to_string().into(),
+                version_id: Some(original.into()),
+            })
+            .bucket(bucket.clone())
+            .key(object.to_string())
+            .build()
+            .unwrap(),
+        Method::PUT,
+    );
+    undo_req
+        .headers
+        .insert(RUSTFS_EXPECTED_CURRENT_VERSION_ID, HeaderValue::from_str(&observed).unwrap());
+
+    let undo = tokio::spawn(async move {
+        let usecase = DefaultObjectUsecase::from_global();
+        Box::pin(usecase.execute_copy_object(undo_req)).await
+    });
+    tokio::task::yield_now().await;
+    let put = PutObjectInput::builder()
+        .bucket(bucket.clone())
+        .key(object.to_string())
+        .body(Some(streaming_blob_from_bytes(b"concurrent")))
+        .content_length(Some(10))
+        .build()
+        .unwrap();
+    let concurrent_put = tokio::spawn(async move {
+        let fs = FS::new();
+        let usecase = DefaultObjectUsecase::from_global();
+        Box::pin(test_seams::put_object(&usecase, &fs, build_request(put, Method::PUT))).await
+    });
+
+    drop(blocker);
+    let undo_result = undo.await.expect("undo task should not panic");
+    let put_result = concurrent_put.await.expect("PUT task should not panic");
+    assert!(put_result.is_ok(), "ordinary PUT should eventually succeed: {put_result:?}");
+    if let Err(err) = undo_result {
+        assert_eq!(err.code(), &S3ErrorCode::PreconditionFailed);
+    }
+    assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), object).await, b"concurrent");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn copy_object_allows_new_version_for_locked_destination_but_blocks_explicit_overwrite() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let fs = FS::new();
+    let usecase = DefaultObjectUsecase::from_global();
+
+    let bucket = format!("test-copy-object-lock-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let src_object = "test/source.txt";
+    let dst_object = "test/destination.txt";
+    let locked_payload = b"locked destination payload";
+    let source_payload = b"copy source payload";
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+    set_bucket_object_lock_enabled(bucket.as_str())
+        .await
+        .expect("Failed to enable object lock for bucket");
+    let _ = upload_test_object(&ecstore, bucket.as_str(), src_object, source_payload).await;
+
+    let retain_until = Timestamp::from(time::OffsetDateTime::now_utc().saturating_add(time::Duration::days(1)));
+    let locked_input = PutObjectInput::builder()
+        .bucket(bucket.clone())
+        .key(dst_object.to_string())
+        .body(Some(streaming_blob_from_bytes(locked_payload)))
+        .content_length(Some(locked_payload.len() as i64))
+        .object_lock_mode(Some(ObjectLockMode::from_static(ObjectLockMode::COMPLIANCE)))
+        .object_lock_retain_until_date(Some(retain_until))
+        .build()
+        .unwrap();
+
+    Box::pin(test_seams::put_object(&usecase, &fs, build_request(locked_input, Method::PUT)))
+        .await
+        .expect("Failed to upload locked destination object");
+
+    let locked_info = ecstore
+        .get_object_info(bucket.as_str(), dst_object, &ObjectOptions::default())
+        .await
+        .expect("Failed to fetch locked destination object info");
+    let locked_version_id = locked_info
+        .version_id
+        .expect("locked destination should have a version ID")
+        .to_string();
+
+    let copy_input = CopyObjectInput::builder()
+        .copy_source(CopySource::Bucket {
+            bucket: bucket.clone().into(),
+            key: src_object.to_string().into(),
+            version_id: None,
+        })
+        .bucket(bucket.clone())
+        .key(dst_object.to_string())
+        .build()
+        .unwrap();
+
+    let copy_output = Box::pin(usecase.execute_copy_object(build_request(copy_input, Method::PUT)))
+        .await
+        .expect("CopyObject should create a new version over a locked current version");
+    let copied_version_id = copy_output
+        .output
+        .version_id
+        .expect("versioned CopyObject should return the created version ID");
+
+    assert_ne!(copied_version_id, locked_version_id);
+    assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), dst_object).await, source_payload);
+    assert_eq!(live_object_version_count(&ecstore, bucket.as_str(), dst_object).await, 2);
+
+    let explicit_overwrite_input = CopyObjectInput::builder()
+        .copy_source(CopySource::Bucket {
+            bucket: bucket.clone().into(),
+            key: src_object.to_string().into(),
+            version_id: None,
+        })
+        .bucket(bucket.clone())
+        .key(dst_object.to_string())
+        .version_id(Some(locked_version_id))
+        .build()
+        .unwrap();
+
+    let err = Box::pin(usecase.execute_copy_object(build_request(explicit_overwrite_input, Method::PUT)))
+        .await
+        .expect_err("explicit CopyObject overwrite of a locked version should be blocked");
+
+    assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
+    assert_eq!(
+        err.message(),
+        Some("Invalid argument: A destination versionId requires replication authorization")
+    );
+    assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), dst_object).await, source_payload);
+    assert_eq!(live_object_version_count(&ecstore, bucket.as_str(), dst_object).await, 2);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn concurrent_reverse_copy_object_does_not_deadlock_with_reader_locks() {
+    temp_env::async_with_vars([(ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
+        let (_disk_paths, ecstore) = setup_test_env().await;
+
+        let bucket = format!("test-reverse-copy-{}", &Uuid::new_v4().simple().to_string()[..8]);
+        let object_a = "test/a.bin";
+        let object_b = "test/b.bin";
+        let payload_a = vec![b'a'; 4 * 1024 * 1024];
+        let payload_b = vec![b'b'; 4 * 1024 * 1024];
+
+        create_test_bucket(&ecstore, bucket.as_str()).await;
+        let _ = upload_test_object(&ecstore, bucket.as_str(), object_a, &payload_a).await;
+        let _ = upload_test_object(&ecstore, bucket.as_str(), object_b, &payload_b).await;
+
+        let a_to_b_input = CopyObjectInput::builder()
+            .copy_source(CopySource::Bucket {
+                bucket: bucket.clone().into(),
+                key: object_a.to_string().into(),
+                version_id: None,
+            })
+            .bucket(bucket.clone())
+            .key(object_b.to_string())
+            .build()
+            .unwrap();
+        let b_to_a_input = CopyObjectInput::builder()
+            .copy_source(CopySource::Bucket {
+                bucket: bucket.clone().into(),
+                key: object_b.to_string().into(),
+                version_id: None,
+            })
+            .bucket(bucket.clone())
+            .key(object_a.to_string())
+            .build()
+            .unwrap();
+
+        let start = Arc::new(Barrier::new(3));
+        let start_a = start.clone();
+        let a_to_b = tokio::spawn(async move {
+            start_a.wait().await;
+            let usecase = DefaultObjectUsecase::from_global();
+            Box::pin(usecase.execute_copy_object(build_request(a_to_b_input, Method::PUT))).await
+        });
+        let start_b = start.clone();
+        let b_to_a = tokio::spawn(async move {
+            start_b.wait().await;
+            let usecase = DefaultObjectUsecase::from_global();
+            Box::pin(usecase.execute_copy_object(build_request(b_to_a_input, Method::PUT))).await
+        });
+
+        start.wait().await;
+
+        let (a_to_b_result, b_to_a_result) = tokio::time::timeout(Duration::from_secs(10), async {
+            let (a_to_b_result, b_to_a_result) = tokio::join!(a_to_b, b_to_a);
+            (
+                a_to_b_result.expect("A-to-B CopyObject task should not panic"),
+                b_to_a_result.expect("B-to-A CopyObject task should not panic"),
+            )
+        })
+        .await
+        .expect("reverse CopyObject operations should not deadlock");
+
+        a_to_b_result.expect("A-to-B CopyObject should succeed");
+        b_to_a_result.expect("B-to-A CopyObject should succeed");
+
+        let final_a = read_object_bytes(&ecstore, bucket.as_str(), object_a).await;
+        let final_b = read_object_bytes(&ecstore, bucket.as_str(), object_b).await;
+        assert!(
+            final_a == payload_a || final_a == payload_b,
+            "object A must contain a complete copied payload"
+        );
+        assert!(
+            final_b == payload_a || final_b == payload_b,
+            "object B must contain a complete copied payload"
+        );
+    })
+    .await;
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn put_and_copy_object_transition_immediately_via_usecases() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let fs = FS::new();
+    let usecase = DefaultObjectUsecase::from_global();
+
+    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+    let backend = register_mock_tier(&tier_name).await;
+
+    let put_bucket = format!("test-api-put-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let put_object = "test/object.txt";
+    let put_payload = b"Hello, immediate transition through put API!";
+
+    create_test_bucket(&ecstore, put_bucket.as_str()).await;
+    set_bucket_lifecycle_transition_with_tier(put_bucket.as_str(), &tier_name)
+        .await
+        .expect("Failed to set lifecycle configuration");
+
+    let put_input = PutObjectInput::builder()
+        .bucket(put_bucket.clone())
+        .key(put_object.to_string())
+        .body(Some(streaming_blob_from_bytes(put_payload)))
+        .content_length(Some(put_payload.len() as i64))
+        .build()
+        .unwrap();
+
+    Box::pin(test_seams::put_object(&usecase, &fs, build_request(put_input, Method::PUT)))
+        .await
+        .expect("Failed to put object through usecase");
+
+    let put_info = wait_for_transition(&ecstore, put_bucket.as_str(), put_object, TRANSITION_WAIT_TIMEOUT)
+        .await
+        .expect("object should transition immediately after put usecase");
+
+    assert_eq!(put_info.transitioned_object.status, "complete");
+    assert_eq!(put_info.transitioned_object.tier, tier_name);
+    assert!(backend.contains(&put_info.transitioned_object.name).await);
+
+    let src_bucket = format!("test-api-copy-src-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let dst_bucket = format!("test-api-copy-dst-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let src_object = "test/source.txt";
+    let dst_object = "test/copied.txt";
+    let copy_payload = b"copy object immediate transition through copy API";
+
+    create_test_bucket(&ecstore, src_bucket.as_str()).await;
+    create_test_bucket(&ecstore, dst_bucket.as_str()).await;
+    set_bucket_lifecycle_transition_with_tier(dst_bucket.as_str(), &tier_name)
+        .await
+        .expect("Failed to set destination lifecycle configuration");
+    let _ = upload_test_object(&ecstore, src_bucket.as_str(), src_object, copy_payload).await;
+
+    let copy_input = CopyObjectInput::builder()
+        .copy_source(CopySource::Bucket {
+            bucket: src_bucket.clone().into(),
+            key: src_object.to_string().into(),
+            version_id: None,
+        })
+        .bucket(dst_bucket.clone())
+        .key(dst_object.to_string())
+        .build()
+        .unwrap();
+
+    Box::pin(usecase.execute_copy_object(build_request(copy_input, Method::PUT)))
+        .await
+        .expect("Failed to copy object through usecase");
+
+    let copy_info = wait_for_transition(&ecstore, dst_bucket.as_str(), dst_object, TRANSITION_WAIT_TIMEOUT)
+        .await
+        .expect("copied object should transition immediately after copy usecase");
+
+    assert_eq!(copy_info.transitioned_object.status, "complete");
+    assert_eq!(copy_info.transitioned_object.tier, tier_name);
+    assert!(backend.contains(&copy_info.transitioned_object.name).await);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn compensation_driven_copy_still_completes_transition() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+
+    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+    let backend = register_mock_tier(&tier_name).await;
+
+    let src_bucket = format!("test-comp-copy-src-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let dst_bucket = format!("test-comp-copy-dst-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let src_object = "test/source.txt";
+    let dst_object = "test/copied.txt";
+    let payload = b"copy object should still transition after compensation";
+
+    create_test_bucket(&ecstore, src_bucket.as_str()).await;
+    create_test_bucket(&ecstore, dst_bucket.as_str()).await;
+    set_bucket_lifecycle_transition_with_tier(dst_bucket.as_str(), &tier_name)
+        .await
+        .expect("Failed to set destination lifecycle configuration");
+    let _ = upload_test_object(&ecstore, src_bucket.as_str(), src_object, payload).await;
+
+    let copy_input = CopyObjectInput::builder()
+        .copy_source(CopySource::Bucket {
+            bucket: src_bucket.clone().into(),
+            key: src_object.to_string().into(),
+            version_id: None,
+        })
+        .bucket(dst_bucket.clone())
+        .key(dst_object.to_string())
+        .build()
+        .unwrap();
+
+    with_forced_immediate_enqueue_timeout(&backend, || async {
+        Box::pin(usecase.execute_copy_object(build_request(copy_input, Method::PUT)))
+            .await
+            .expect("Failed to copy object through usecase");
+    })
+    .await;
+
+    let info = wait_for_transition(&ecstore, dst_bucket.as_str(), dst_object, TRANSITION_WAIT_TIMEOUT)
+        .await
+        .expect("copied object should eventually transition after compensation backfill");
+
+    assert_eq!(backend.put_count().await, 1, "compensation should upload the copied version once");
+    assert_eq!(info.transitioned_object.status, "complete");
+    assert_eq!(info.transitioned_object.tier, tier_name);
+    assert!(backend.contains(&info.transitioned_object.name).await);
+    assert_eq!(read_object_bytes(&ecstore, dst_bucket.as_str(), dst_object).await, payload);
+}
+
+/// CopyObject creates a new object on the destination key, so it must join
+/// bucket replication exactly like PutObject: compute one replication decision
+/// that drives both the persisted pending marker and the post-commit schedule
+/// (MinIO CopyObjectHandler parity). A count of 0 means a copied object is
+/// never scheduled for replication and — because the copy path clones the
+/// source metadata wholesale — the destination silently inherits the source's
+/// replication bookkeeping, faking a COMPLETED/REPLICA state for an object
+/// that never replicated. The second half of this test pins the metadata
+/// cleanup (MinIO filterReplicationStatusMetadata parity).
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state usecase integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn copy_object_computes_replication_decision_and_strips_stale_status() {
+    use super::super::storage_api::object_usecase::bucket::replication::MUST_REPLICATE_OBJECT_CALLS;
+    use rustfs_utils::http::{
+        SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, get_str,
+        insert_str,
+    };
+    use std::collections::HashMap;
+    use std::sync::atomic::Ordering;
+
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+
+    let bucket = format!("test-copy-repl-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let src_object = "copy/source.txt";
+    let dst_object = "copy/destination.txt";
+    let payload = b"copy replication decision payload";
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+
+    // Seed the source object with stale replication bookkeeping, as if it had
+    // already replicated elsewhere (internal status/timestamp under both
+    // compatibility prefixes, replica state, and the surfaced x-amz header).
+    let mut stale_metadata = HashMap::new();
+    insert_str(
+        &mut stale_metadata,
+        SUFFIX_REPLICATION_STATUS,
+        "arn:minio:replication::stale:dst=COMPLETED;".to_string(),
+    );
+    insert_str(&mut stale_metadata, SUFFIX_REPLICATION_TIMESTAMP, "2024-01-01T00:00:00Z".to_string());
+    insert_str(&mut stale_metadata, SUFFIX_REPLICA_STATUS, "REPLICA".to_string());
+    insert_str(&mut stale_metadata, SUFFIX_REPLICA_TIMESTAMP, "2024-01-01T00:00:00Z".to_string());
+    stale_metadata.insert(AMZ_BUCKET_REPLICATION_STATUS.to_string(), "COMPLETED".to_string());
+    let mut reader = PutObjReader::from_vec(payload.to_vec());
+    (*ecstore)
+        .put_object(
+            bucket.as_str(),
+            src_object,
+            &mut reader,
+            &ObjectOptions {
+                user_defined: stale_metadata,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("Failed to upload source object with stale replication metadata");
+
+    MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
+
+    let copy_input = CopyObjectInput::builder()
+        .copy_source(CopySource::Bucket {
+            bucket: bucket.clone().into(),
+            key: src_object.into(),
+            version_id: None,
+        })
+        .bucket(bucket.clone())
+        .key(dst_object.to_string())
+        .build()
+        .unwrap();
+    Box::pin(usecase.execute_copy_object(build_request(copy_input, Method::PUT)))
+        .await
+        .expect("Failed to copy object through usecase");
+
+    assert_eq!(
+        MUST_REPLICATE_OBJECT_CALLS.load(Ordering::SeqCst),
+        1,
+        "CopyObject must compute the replication decision exactly once; 0 means the copied \
+         object never enters bucket replication (P0-6, MinIO CopyObjectHandler parity)"
+    );
+
+    let copied = ecstore
+        .get_object_info(bucket.as_str(), dst_object, &ObjectOptions::default())
+        .await
+        .expect("Failed to read copied destination object info");
+    assert!(
+        get_str(&copied.user_defined, SUFFIX_REPLICATION_STATUS).is_none(),
+        "destination must not inherit the source's replication status; got {:?}",
+        get_str(&copied.user_defined, SUFFIX_REPLICATION_STATUS)
+    );
+    assert!(
+        get_str(&copied.user_defined, SUFFIX_REPLICATION_TIMESTAMP).is_none(),
+        "destination must not inherit the source's replication timestamp"
+    );
+    assert!(
+        get_str(&copied.user_defined, SUFFIX_REPLICA_STATUS).is_none(),
+        "destination must not inherit the source's replica status"
+    );
+    assert!(
+        !copied
+            .user_defined
+            .keys()
+            .any(|k| k.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS)),
+        "destination must not inherit the surfaced x-amz-replication-status header key"
+    );
+}

rustfs/src/app/lifecycle_transition_api_test/delete.rs

@@ -0,0 +1,923 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! DeleteObject and DeleteObjects tests (batch E owns both).
+
+use super::common::*;
+use super::*;
+use crate::app::test_seams;
+
+async fn wait_for_object_absence(ecstore: &Arc<ECStore>, bucket: &str, object: &str, timeout: Duration) -> bool {
+    let deadline = tokio::time::Instant::now() + timeout;
+
+    loop {
+        if ecstore
+            .get_object_info(bucket, object, &ObjectOptions::default())
+            .await
+            .is_err()
+        {
+            return true;
+        }
+
+        if tokio::time::Instant::now() >= deadline {
+            return false;
+        }
+
+        tokio::time::sleep(Duration::from_millis(50)).await;
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state object-version integration test: runs serialized in the CI ILM Integration (serial) lane"]
+async fn undo_delete_only_removes_the_current_matching_delete_marker() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let fs = FS::new();
+    let usecase = DefaultObjectUsecase::from_global();
+    let bucket = format!("test-undo-delete-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/object.txt";
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+
+    let put = PutObjectInput::builder()
+        .bucket(bucket.clone())
+        .key(object.to_string())
+        .body(Some(streaming_blob_from_bytes(b"visible again")))
+        .content_length(Some(13))
+        .build()
+        .unwrap();
+    Box::pin(test_seams::put_object(&usecase, &fs, build_request(put, Method::PUT)))
+        .await
+        .expect("object version should be written");
+    let marker = Box::pin(
+        usecase.execute_delete_object(build_request(
+            DeleteObjectInput::builder()
+                .bucket(bucket.clone())
+                .key(object.to_string())
+                .build()
+                .unwrap(),
+            Method::DELETE,
+        )),
+    )
+    .await
+    .expect("delete marker should be created")
+    .output
+    .version_id
+    .expect("versioned delete should return the marker version");
+
+    let delete_marker = || {
+        DeleteObjectInput::builder()
+            .bucket(bucket.clone())
+            .key(object.to_string())
+            .version_id(Some(marker.clone()))
+            .build()
+            .unwrap()
+    };
+    let mut stale_req = build_request(delete_marker(), Method::DELETE);
+    stale_req.headers.insert(
+        RUSTFS_EXPECTED_CURRENT_VERSION_ID,
+        HeaderValue::from_str(&Uuid::new_v4().to_string()).unwrap(),
+    );
+    let err = Box::pin(usecase.execute_delete_object(stale_req))
+        .await
+        .expect_err("mismatched delete marker must fail");
+    assert_eq!(err.code(), &S3ErrorCode::PreconditionFailed);
+
+    let mut matching_req = build_request(delete_marker(), Method::DELETE);
+    matching_req
+        .headers
+        .insert(RUSTFS_EXPECTED_CURRENT_VERSION_ID, HeaderValue::from_str(&marker).unwrap());
+    Box::pin(usecase.execute_delete_object(matching_req))
+        .await
+        .expect("current matching delete marker should be removed");
+    assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), object).await, b"visible again");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn delete_transitioned_object_removes_remote_tier_copy_via_usecase() {
+    use super::super::storage_api::test::ReqInfo;
+
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+
+    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+    let backend = register_mock_tier(&tier_name).await;
+
+    let bucket = format!("test-api-delete-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/object.txt";
+    let payload = b"delete transitioned object through delete API";
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+    set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
+        .await
+        .expect("Failed to set lifecycle configuration");
+    let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
+
+    lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(ecstore.clone(), bucket.as_str())
+        .await
+        .expect("Failed to enqueue transitioned object");
+
+    let transitioned = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
+        .await
+        .expect("object should transition before delete usecase runs");
+    let remote_object = transitioned.transitioned_object.name.clone();
+
+    assert!(backend.contains(&remote_object).await);
+
+    let mut req = build_request(
+        DeleteObjectInput::builder()
+            .bucket(bucket.clone())
+            .key(object.to_string())
+            .build()
+            .unwrap(),
+        Method::DELETE,
+    );
+    insert_header(&mut req.headers, SUFFIX_FORCE_DELETE, "true");
+    req.extensions.insert(ReqInfo {
+        cred: Some(rustfs_credentials::Credentials::default()),
+        is_owner: true,
+        ..Default::default()
+    });
+
+    Box::pin(usecase.execute_delete_object(req))
+        .await
+        .expect("Failed to delete object through usecase");
+
+    assert!(
+        wait_for_object_absence(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
+        "object should be removed from hot tier after delete usecase"
+    );
+
+    assert!(
+        backend.wait_for_remote_absence(&remote_object, TRANSITION_WAIT_TIMEOUT).await,
+        "transitioned object should be removed from remote tier after delete usecase"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn compensation_driven_transition_still_cleans_remote_tier_on_delete() {
+    use super::super::storage_api::test::ReqInfo;
+
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+
+    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+    let backend = register_mock_tier(&tier_name).await;
+
+    let bucket = format!("test-compensation-delete-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/object.txt";
+    let payload = b"compensation should still preserve delete cleanup";
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+    set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
+        .await
+        .expect("Failed to set lifecycle configuration");
+
+    with_forced_immediate_enqueue_timeout(&backend, || async {
+        let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
+    })
+    .await;
+
+    let transitioned = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
+        .await
+        .expect("object should eventually transition after compensation backfill");
+    assert_eq!(backend.put_count().await, 1, "compensation should upload the version once");
+    let remote_object = transitioned.transitioned_object.name.clone();
+
+    assert!(backend.contains(&remote_object).await);
+
+    let mut req = build_request(
+        DeleteObjectInput::builder()
+            .bucket(bucket.clone())
+            .key(object.to_string())
+            .build()
+            .unwrap(),
+        Method::DELETE,
+    );
+    insert_header(&mut req.headers, SUFFIX_FORCE_DELETE, "true");
+    req.extensions.insert(ReqInfo {
+        cred: Some(rustfs_credentials::Credentials::default()),
+        is_owner: true,
+        ..Default::default()
+    });
+
+    Box::pin(usecase.execute_delete_object(req))
+        .await
+        .expect("Failed to delete object through usecase after compensation-driven transition");
+
+    assert!(
+        wait_for_object_absence(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
+        "object should be removed from hot tier after delete usecase"
+    );
+
+    assert!(
+        backend.wait_for_remote_absence(&remote_object, TRANSITION_WAIT_TIMEOUT).await,
+        "transitioned object should be removed from remote tier after delete usecase"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn compensation_driven_versioned_delete_still_creates_delete_marker() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+
+    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+    let backend = register_mock_tier(&tier_name).await;
+
+    let bucket = format!("test-comp-versioned-delete-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/object.txt";
+    let payload = b"versioned delete should preserve transitioned remote version behind delete marker";
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+    set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
+        .await
+        .expect("Failed to set lifecycle configuration");
+
+    with_forced_immediate_enqueue_timeout(&backend, || async {
+        let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
+    })
+    .await;
+
+    let transitioned = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
+        .await
+        .expect("object should eventually transition after compensation backfill");
+    assert_eq!(backend.put_count().await, 1, "compensation should upload the version once");
+    let remote_object = transitioned.transitioned_object.name.clone();
+
+    assert!(backend.contains(&remote_object).await);
+
+    let req = build_request(
+        DeleteObjectInput::builder()
+            .bucket(bucket.clone())
+            .key(object.to_string())
+            .build()
+            .unwrap(),
+        Method::DELETE,
+    );
+
+    Box::pin(usecase.execute_delete_object(req))
+        .await
+        .expect("Failed to issue versioned delete after compensation-driven transition");
+
+    assert!(
+        wait_for_delete_marker(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
+        "versioned delete should create a delete marker after compensation-driven transition"
+    );
+    assert!(
+        backend.contains(&remote_object).await,
+        "creating a delete marker should not remove the transitioned remote object version"
+    );
+}
+
+/// Regression pin for the DeleteObjects hot path: the bucket's versioning
+/// configuration must be resolved exactly once per request — not once per
+/// key, which used to cost up to 1000 identical metadata lookups per call.
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
+async fn delete_objects_resolves_bucket_versioning_once_per_request() {
+    use super::super::storage_api::test::bucket::replication::DELETE_CONFIG_SNAPSHOT_LOADS;
+    use super::super::storage_api::test::{ReqInfo, VERSIONING_CONFIG_LOOKUPS};
+    use std::sync::atomic::Ordering;
+
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+
+    let bucket = format!("test-delobjs-vers-once-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+
+    let keys = ["batch/a.txt", "batch/b.txt", "batch/c.txt"];
+    for key in keys {
+        upload_test_object(&ecstore, bucket.as_str(), key, b"delete objects versioning payload").await;
+    }
+
+    let input = DeleteObjectsInput::builder()
+        .bucket(bucket.clone())
+        .delete(Delete {
+            objects: keys
+                .iter()
+                .map(|key| ObjectIdentifier {
+                    key: key.to_string(),
+                    version_id: None,
+                    ..Default::default()
+                })
+                .collect(),
+            quiet: None,
+        })
+        .build()
+        .expect("delete objects input should build");
+
+    let mut req = build_request(input, Method::POST);
+    req.extensions.insert(ReqInfo {
+        cred: Some(rustfs_credentials::Credentials::default()),
+        is_owner: true,
+        ..Default::default()
+    });
+
+    let lookups_before = VERSIONING_CONFIG_LOOKUPS.load(Ordering::SeqCst);
+    let snapshots_before = DELETE_CONFIG_SNAPSHOT_LOADS.load(Ordering::SeqCst);
+    let response = usecase
+        .execute_delete_objects(req)
+        .await
+        .expect("delete objects should succeed");
+
+    let output = response.output;
+    let errors = output.errors.unwrap_or_default();
+    assert!(errors.is_empty(), "DeleteObjects reported per-key errors: {errors:?}");
+    assert_eq!(
+        output.deleted.unwrap_or_default().len(),
+        keys.len(),
+        "every requested key should be deleted"
+    );
+
+    let lookups = VERSIONING_CONFIG_LOOKUPS.load(Ordering::SeqCst) - lookups_before;
+    let snapshots = DELETE_CONFIG_SNAPSHOT_LOADS.load(Ordering::SeqCst) - snapshots_before;
+    assert_eq!(
+        snapshots,
+        1,
+        "DeleteObjects must resolve delete admission config exactly once per request; got {snapshots} snapshots for {} keys",
+        keys.len()
+    );
+    assert_eq!(lookups, 0, "DeleteObjects must not fall back to legacy per-key versioning lookups");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
+async fn delete_objects_treats_empty_directory_version_as_absent() {
+    use super::super::storage_api::test::ReqInfo;
+
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+    let bucket = format!("test-delobjs-empty-version-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "directory/";
+    create_test_bucket(&ecstore, &bucket).await;
+    upload_test_object(&ecstore, &bucket, object, b"").await;
+
+    let input = DeleteObjectsInput::builder()
+        .bucket(bucket)
+        .delete(Delete {
+            objects: vec![ObjectIdentifier {
+                key: object.to_string(),
+                version_id: Some(" \t ".to_string()),
+                ..Default::default()
+            }],
+            quiet: None,
+        })
+        .build()
+        .expect("delete objects input should build");
+    let mut req = build_request(input, Method::POST);
+    req.extensions.insert(ReqInfo {
+        cred: Some(rustfs_credentials::Credentials::default()),
+        is_owner: true,
+        ..Default::default()
+    });
+
+    let response = usecase
+        .execute_delete_objects(req)
+        .await
+        .expect("empty VersionId should be treated as an absent version");
+    let deleted = response
+        .output
+        .deleted
+        .expect("delete response should contain the directory entry");
+
+    assert_eq!(deleted.len(), 1);
+    assert_eq!(
+        deleted[0].delete_marker, None,
+        "the synthetic null version must be purged, not hidden by a new marker"
+    );
+    assert_eq!(deleted[0].version_id, None, "the synthetic null sentinel must stay off the wire");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
+async fn delete_objects_preserves_explicit_null_directory_version() {
+    use super::super::storage_api::test::ReqInfo;
+
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+    let bucket = format!("test-delobjs-null-version-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "directory/";
+    create_test_bucket(&ecstore, &bucket).await;
+    upload_test_object(&ecstore, &bucket, object, b"").await;
+
+    let input = DeleteObjectsInput::builder()
+        .bucket(bucket)
+        .delete(Delete {
+            objects: vec![ObjectIdentifier {
+                key: object.to_string(),
+                version_id: Some("null".to_string()),
+                ..Default::default()
+            }],
+            quiet: None,
+        })
+        .build()
+        .expect("delete objects input should build");
+    let mut req = build_request(input, Method::POST);
+    req.extensions.insert(ReqInfo {
+        cred: Some(rustfs_credentials::Credentials::default()),
+        is_owner: true,
+        ..Default::default()
+    });
+
+    let response = usecase
+        .execute_delete_objects(req)
+        .await
+        .expect("explicit null VersionId should delete the directory's null version");
+    let deleted = response
+        .output
+        .deleted
+        .expect("delete response should contain the directory entry");
+
+    assert_eq!(deleted.len(), 1);
+    assert_eq!(deleted[0].delete_marker, None);
+    assert_eq!(deleted[0].version_id.as_deref(), Some("null"));
+}
+
+async fn install_malformed_versioning_config(bucket: &str) {
+    use super::super::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata};
+
+    let sys = get_global_bucket_metadata_sys().expect("bucket metadata system should be initialized");
+    let metadata = {
+        let sys = sys.read().await;
+        sys.get(bucket)
+            .await
+            .expect("bucket metadata should be cached before corruption injection")
+    };
+    let mut metadata = (*metadata).clone();
+    metadata.versioning_config_xml = b"<VersioningConfiguration>".to_vec();
+    metadata.versioning_config = None;
+    set_bucket_metadata(bucket.to_string(), metadata)
+        .await
+        .expect("malformed versioning metadata should be installed for the request test");
+}
+
+async fn install_malformed_replication_config(bucket: &str) {
+    use super::super::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata};
+
+    let sys = get_global_bucket_metadata_sys().expect("bucket metadata system should be initialized");
+    let metadata = {
+        let sys = sys.read().await;
+        sys.get(bucket)
+            .await
+            .expect("bucket metadata should be cached before corruption injection")
+    };
+    let mut metadata = (*metadata).clone();
+    metadata.replication_config_xml = b"<ReplicationConfiguration>".to_vec();
+    metadata.replication_config = None;
+    set_bucket_metadata(bucket.to_string(), metadata)
+        .await
+        .expect("malformed replication metadata should be installed for the request test");
+}
+
+#[tokio::test]
+#[serial]
+async fn delete_object_versioning_config_failure_leaves_latest_object_intact() {
+    use super::super::storage_api::test::ReqInfo;
+
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+    let bucket = format!("test-delete-versioning-fail-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "object.txt";
+    let payload = b"local delete must not commit before versioning admission";
+    create_test_bucket(&ecstore, &bucket).await;
+    upload_test_object(&ecstore, &bucket, object, payload).await;
+    install_malformed_versioning_config(&bucket).await;
+
+    let input = DeleteObjectInput::builder()
+        .bucket(bucket.clone())
+        .key(object.to_string())
+        .build()
+        .expect("delete input should build");
+    let mut req = build_request(input, Method::DELETE);
+    req.extensions.insert(ReqInfo {
+        cred: Some(rustfs_credentials::Credentials::default()),
+        is_owner: true,
+        ..Default::default()
+    });
+
+    let err = usecase
+        .execute_delete_object(req)
+        .await
+        .expect_err("versioning config failure must reject DeleteObject");
+
+    assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
+    assert_eq!(read_object_bytes(&ecstore, &bucket, object).await, payload);
+}
+
+#[tokio::test]
+#[serial]
+async fn delete_object_replication_config_failure_leaves_latest_object_intact() {
+    use super::super::storage_api::test::ReqInfo;
+
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+    let bucket = format!("test-delete-repl-fail-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "object.txt";
+    let payload = b"local delete must not commit before replication admission";
+    create_test_bucket(&ecstore, &bucket).await;
+    upload_test_object(&ecstore, &bucket, object, payload).await;
+    install_malformed_replication_config(&bucket).await;
+
+    let input = DeleteObjectInput::builder()
+        .bucket(bucket.clone())
+        .key(object.to_string())
+        .build()
+        .expect("delete input should build");
+    let mut req = build_request(input, Method::DELETE);
+    req.extensions.insert(ReqInfo {
+        cred: Some(rustfs_credentials::Credentials::default()),
+        is_owner: true,
+        ..Default::default()
+    });
+
+    let err = usecase
+        .execute_delete_object(req)
+        .await
+        .expect_err("replication config failure must reject DeleteObject");
+
+    assert_eq!(err.code(), &S3ErrorCode::InternalError);
+    assert_eq!(read_object_bytes(&ecstore, &bucket, object).await, payload);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+async fn delete_object_uses_one_replication_config_generation() {
+    use super::super::object_usecase::install_delete_snapshot_test_hook;
+    use super::super::storage_api::test::ReqInfo;
+    use super::super::storage_api::test::bucket::replication::take_scheduled_replication_deletes;
+
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+    let bucket = format!("test-delete-config-generation-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "object.txt";
+    create_test_bucket(&ecstore, &bucket).await;
+
+    let mut reader = PutObjReader::from_vec(b"generation-bound delete".to_vec());
+    let uploaded = ecstore
+        .put_object(
+            &bucket,
+            object,
+            &mut reader,
+            &ObjectOptions {
+                versioned: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("versioned object should be uploaded");
+    let version_id = uploaded.version_id.expect("versioned upload should return a version ID");
+    let version_id_string = version_id.to_string();
+
+    install_delete_replication_config(&bucket, true, false, None).await;
+    assert!(take_scheduled_replication_deletes().is_empty());
+
+    let loaded = Arc::new(Barrier::new(2));
+    let resume = Arc::new(Barrier::new(2));
+    install_delete_snapshot_test_hook(bucket.clone(), Arc::clone(&loaded), Arc::clone(&resume));
+
+    let input = DeleteObjectInput::builder()
+        .bucket(bucket.clone())
+        .key(object.to_string())
+        .version_id(Some(version_id_string.clone()))
+        .build()
+        .expect("version delete input should build");
+    let mut req = build_request(input, Method::DELETE);
+    req.extensions.insert(ReqInfo {
+        cred: Some(rustfs_credentials::Credentials::default()),
+        is_owner: true,
+        ..Default::default()
+    });
+
+    let update_config = async {
+        loaded.wait().await;
+        install_delete_replication_config(&bucket, false, false, None).await;
+        resume.wait().await;
+    };
+    let (response, ()) = tokio::join!(usecase.execute_delete_object(req), update_config);
+    let response = response.expect("delete admitted by the first config generation should succeed");
+
+    assert_eq!(response.output.version_id.as_deref(), Some(version_id_string.as_str()));
+    let scheduled = take_scheduled_replication_deletes();
+    assert_eq!(scheduled.len(), 1, "the first config generation must control post-delete queueing");
+    assert_eq!(scheduled[0].version_id, Some(version_id));
+    let state = scheduled[0]
+        .replication_state
+        .as_ref()
+        .expect("admitted version purge should carry replication state");
+    assert_eq!(
+        state.version_purge_status_internal.as_deref(),
+        Some("arn:aws:s3:::target-bucket=PENDING;")
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn replica_delete_uses_authorized_source_version_and_its_tags() {
+    use super::super::object_usecase::install_delete_source_test_hook;
+    use super::super::storage_api::test::ReqInfo;
+    use super::super::storage_api::test::bucket::replication::{ReplicationStatusType, take_scheduled_replication_deletes};
+    use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
+    use rustfs_utils::http::{SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID};
+    use std::collections::HashMap;
+
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+    let bucket = format!("test-replica-source-version-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "object.txt";
+    create_test_bucket(&ecstore, &bucket).await;
+    install_delete_replication_config(&bucket, true, true, Some(("generation", "updated"))).await;
+
+    let mut old_opts = ObjectOptions {
+        versioned: true,
+        user_defined: HashMap::from([(AMZ_OBJECT_TAGGING.to_string(), "generation=old".to_string())]),
+        ..Default::default()
+    };
+    old_opts.set_replica_status(ReplicationStatusType::Replica);
+    let mut old_reader = PutObjReader::from_vec(b"old replica version".to_vec());
+    let old = ecstore
+        .put_object(&bucket, object, &mut old_reader, &old_opts)
+        .await
+        .expect("old replica version should be uploaded");
+    let old_version_id = old.version_id.expect("old replica version should have a version ID");
+    let old_version_id_string = old_version_id.to_string();
+
+    let mut latest_opts = ObjectOptions {
+        versioned: true,
+        user_defined: HashMap::from([(AMZ_OBJECT_TAGGING.to_string(), "generation=latest".to_string())]),
+        ..Default::default()
+    };
+    latest_opts.set_replica_status(ReplicationStatusType::Replica);
+    let mut latest_reader = PutObjReader::from_vec(b"latest replica version".to_vec());
+    let latest = ecstore
+        .put_object(&bucket, object, &mut latest_reader, &latest_opts)
+        .await
+        .expect("latest replica version should be uploaded");
+    assert_ne!(latest.version_id, Some(old_version_id));
+    assert!(take_scheduled_replication_deletes().is_empty());
+
+    let source_loaded = Arc::new(Barrier::new(2));
+    let resume_delete = Arc::new(Barrier::new(2));
+    install_delete_source_test_hook(bucket.clone(), Arc::clone(&source_loaded), Arc::clone(&resume_delete));
+
+    let input = DeleteObjectInput::builder()
+        .bucket(bucket.clone())
+        .key(object.to_string())
+        .build()
+        .expect("replica delete input should build");
+    let mut req = build_request(input, Method::DELETE);
+    req.headers
+        .insert(AMZ_BUCKET_REPLICATION_STATUS, HeaderValue::from_static("REPLICA"));
+    insert_header(&mut req.headers, SUFFIX_SOURCE_VERSION_ID, old_version_id_string.clone());
+    insert_header(&mut req.headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
+    req.extensions.insert(ReqInfo {
+        cred: Some(rustfs_credentials::Credentials::default()),
+        is_owner: true,
+        ..Default::default()
+    });
+
+    let update_tags = async {
+        source_loaded.wait().await;
+        ecstore
+            .put_object_tags(
+                &bucket,
+                object,
+                "generation=updated",
+                &ObjectOptions {
+                    versioned: true,
+                    version_id: Some(old_version_id_string.clone()),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("tag update ordered before the locked delete should succeed");
+        resume_delete.wait().await;
+    };
+    let (response, ()) = tokio::join!(usecase.execute_delete_object(req), update_tags);
+    let response = response.expect("authorized replica version delete should succeed");
+
+    assert_eq!(response.output.version_id.as_deref(), Some(old_version_id_string.as_str()));
+    let scheduled = take_scheduled_replication_deletes();
+    assert_eq!(
+        scheduled.len(),
+        1,
+        "the tag committed after the advisory pre-read must control downstream fanout"
+    );
+    assert_eq!(scheduled[0].version_id, Some(old_version_id));
+    assert_eq!(
+        scheduled[0]
+            .replication_state
+            .as_ref()
+            .and_then(|state| state.version_purge_status_internal.as_deref()),
+        Some("arn:aws:s3:::target-bucket=PENDING;")
+    );
+    assert_eq!(read_object_bytes(&ecstore, &bucket, object).await, b"latest replica version");
+}
+
+#[tokio::test]
+#[serial]
+async fn force_delete_replication_config_failure_leaves_prefix_intact() {
+    use super::super::storage_api::test::ReqInfo;
+
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+    let bucket = format!("test-force-delete-repl-fail-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let prefix = "prefix/";
+    let object = "prefix/object.txt";
+    let payload = b"force delete must validate replication metadata before deleting descendants";
+    create_test_bucket(&ecstore, &bucket).await;
+    upload_test_object(&ecstore, &bucket, object, payload).await;
+    install_malformed_replication_config(&bucket).await;
+
+    let input = DeleteObjectInput::builder()
+        .bucket(bucket.clone())
+        .key(prefix.to_string())
+        .build()
+        .expect("delete input should build");
+    let mut req = build_request(input, Method::DELETE);
+    insert_header(&mut req.headers, SUFFIX_FORCE_DELETE, "true");
+    req.extensions.insert(ReqInfo {
+        cred: Some(rustfs_credentials::Credentials::default()),
+        is_owner: true,
+        ..Default::default()
+    });
+
+    let err = usecase
+        .execute_delete_object(req)
+        .await
+        .expect_err("replication config failure must reject force delete");
+
+    assert_eq!(err.code(), &S3ErrorCode::InternalError);
+    assert_eq!(read_object_bytes(&ecstore, &bucket, object).await, payload);
+}
+
+#[tokio::test]
+#[serial]
+async fn replica_delete_replication_config_failure_leaves_object_intact() {
+    use super::super::storage_api::test::ReqInfo;
+
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+    let bucket = format!("test-replica-delete-repl-fail-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "object.txt";
+    let payload = b"incoming replica delete must use the admitted request snapshot";
+    create_test_bucket(&ecstore, &bucket).await;
+    upload_test_object(&ecstore, &bucket, object, payload).await;
+    install_malformed_replication_config(&bucket).await;
+
+    let input = DeleteObjectInput::builder()
+        .bucket(bucket.clone())
+        .key(object.to_string())
+        .build()
+        .expect("delete input should build");
+    let mut req = build_request(input, Method::DELETE);
+    req.headers
+        .insert(AMZ_BUCKET_REPLICATION_STATUS, HeaderValue::from_static("REPLICA"));
+    req.extensions.insert(ReqInfo {
+        cred: Some(rustfs_credentials::Credentials::default()),
+        is_owner: true,
+        ..Default::default()
+    });
+
+    let err = usecase
+        .execute_delete_object(req)
+        .await
+        .expect_err("replication config failure must reject incoming replica delete");
+
+    assert_eq!(err.code(), &S3ErrorCode::InternalError);
+    assert_eq!(read_object_bytes(&ecstore, &bucket, object).await, payload);
+}
+
+#[tokio::test]
+#[serial]
+async fn delete_objects_replication_config_failure_does_not_call_storage_delete() {
+    use super::super::storage_api::test::ReqInfo;
+
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+    let bucket = format!("test-delete-repls-fail-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "object.txt";
+    let payload = b"batch delete must fail closed on replication metadata errors";
+    create_test_bucket(&ecstore, &bucket).await;
+    upload_test_object(&ecstore, &bucket, object, payload).await;
+    install_malformed_replication_config(&bucket).await;
+
+    let input = DeleteObjectsInput::builder()
+        .bucket(bucket.clone())
+        .delete(Delete {
+            objects: vec![ObjectIdentifier {
+                key: object.to_string(),
+                version_id: None,
+                ..Default::default()
+            }],
+            quiet: None,
+        })
+        .build()
+        .expect("delete objects input should build");
+    let mut req = build_request(input, Method::POST);
+    req.extensions.insert(ReqInfo {
+        cred: Some(rustfs_credentials::Credentials::default()),
+        is_owner: true,
+        ..Default::default()
+    });
+
+    let err = usecase
+        .execute_delete_objects(req)
+        .await
+        .expect_err("replication config failure must reject DeleteObjects");
+
+    assert_eq!(err.code(), &S3ErrorCode::InternalError);
+    assert_eq!(read_object_bytes(&ecstore, &bucket, object).await, payload);
+}
+
+/// Regression pin for the nonexistent-bucket fast path: GET/HEAD/DeleteObject
+/// must decide NoSuchBucket from the bucket-existence check, before any bucket
+/// versioning lookup runs — and cheap request-shape validation still wins over
+/// bucket existence for GET.
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
+async fn nonexistent_bucket_fails_before_versioning_lookup_on_get_head_delete() {
+    use super::super::storage_api::test::VERSIONING_CONFIG_LOOKUPS;
+    use std::sync::atomic::Ordering;
+
+    let (_disk_paths, _ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+
+    // Never created.
+    let bucket = format!("test-missing-{}", &Uuid::new_v4().simple().to_string()[..8]);
+
+    // Request-shape validation keeps precedence over bucket existence: GET with
+    // both range and partNumber is InvalidArgument even on a missing bucket.
+    let get_invalid = GetObjectInput::builder()
+        .bucket(bucket.clone())
+        .key("k.txt".to_string())
+        .range(Some(Range::Int { first: 0, last: Some(1) }))
+        .part_number(Some(1))
+        .build()
+        .expect("get input should build");
+    let err = Box::pin(test_seams::get_object(&usecase, build_request(get_invalid, Method::GET)))
+        .await
+        .expect_err("range+partNumber must be rejected");
+    assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
+
+    let lookups_before = VERSIONING_CONFIG_LOOKUPS.load(Ordering::SeqCst);
+
+    let get = GetObjectInput::builder()
+        .bucket(bucket.clone())
+        .key("k.txt".to_string())
+        .build()
+        .expect("get input should build");
+    let err = Box::pin(test_seams::get_object(&usecase, build_request(get, Method::GET)))
+        .await
+        .expect_err("GET on a missing bucket must fail");
+    assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket);
+
+    // Bucket existence beats versionId-format validation (which lives in the
+    // opts builders) for HEAD and DeleteObject.
+    let head = HeadObjectInput::builder()
+        .bucket(bucket.clone())
+        .key("k.txt".to_string())
+        .version_id(Some("not-a-uuid".to_string()))
+        .build()
+        .expect("head input should build");
+    let err = Box::pin(test_seams::head_object(&usecase, build_request(head, Method::HEAD)))
+        .await
+        .expect_err("HEAD on a missing bucket must fail");
+    assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket);
+
+    let del = DeleteObjectInput::builder()
+        .bucket(bucket.clone())
+        .key("k.txt".to_string())
+        .version_id(Some("not-a-uuid".to_string()))
+        .build()
+        .expect("delete input should build");
+    let err = Box::pin(usecase.execute_delete_object(build_request(del, Method::DELETE)))
+        .await
+        .expect_err("DeleteObject on a missing bucket must fail");
+    assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket);
+
+    let lookups = VERSIONING_CONFIG_LOOKUPS.load(Ordering::SeqCst) - lookups_before;
+    assert_eq!(
+        lookups, 0,
+        "requests naming a nonexistent bucket must fail before any bucket versioning lookup; got {lookups}"
+    );
+}

rustfs/src/app/lifecycle_transition_api_test/lifecycle_config.rs

@@ -0,0 +1,403 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! PutBucketLifecycleConfiguration tests (batch A3 owns the lifecycle operations).
+
+use super::common::*;
+use super::*;
+
+fn expiration_lifecycle_configuration(prefix: &str) -> BucketLifecycleConfiguration {
+    BucketLifecycleConfiguration {
+        expiry_updated_at: None,
+        rules: vec![LifecycleRule {
+            status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+            abort_incomplete_multipart_upload: None,
+            del_marker_expiration: None,
+            expiration: Some(LifecycleExpiration {
+                date: Some(Timestamp::from(
+                    time::OffsetDateTime::now_utc()
+                        .replace_time(time::Time::MIDNIGHT)
+                        .saturating_sub(time::Duration::days(1)),
+                )),
+                days: None,
+                expired_object_delete_marker: None,
+                ..Default::default()
+            }),
+            filter: Some(LifecycleRuleFilter {
+                and: None,
+                object_size_greater_than: None,
+                object_size_less_than: None,
+                prefix: Some(prefix.to_string()),
+                tag: None,
+                ..Default::default()
+            }),
+            id: Some("expire-existing".to_string()),
+            noncurrent_version_expiration: None,
+            noncurrent_version_transitions: None,
+            prefix: None,
+            transitions: None,
+        }],
+    }
+}
+
+fn expired_object_delete_marker_lifecycle_configuration() -> BucketLifecycleConfiguration {
+    BucketLifecycleConfiguration {
+        expiry_updated_at: None,
+        rules: vec![LifecycleRule {
+            status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+            abort_incomplete_multipart_upload: None,
+            del_marker_expiration: None,
+            expiration: Some(LifecycleExpiration {
+                expired_object_delete_marker: Some(true),
+                ..Default::default()
+            }),
+            filter: Some(LifecycleRuleFilter {
+                prefix: Some("test/".to_string()),
+                ..Default::default()
+            }),
+            id: Some("expired-delete-marker".to_string()),
+            noncurrent_version_expiration: None,
+            noncurrent_version_transitions: None,
+            prefix: None,
+            transitions: None,
+        }],
+    }
+}
+
+fn del_marker_expiration_lifecycle_configuration(days: i32) -> BucketLifecycleConfiguration {
+    BucketLifecycleConfiguration {
+        expiry_updated_at: None,
+        rules: vec![LifecycleRule {
+            status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+            abort_incomplete_multipart_upload: None,
+            del_marker_expiration: Some(DelMarkerExpiration { days: Some(days) }),
+            expiration: Some(LifecycleExpiration {
+                days: Some(30),
+                ..Default::default()
+            }),
+            filter: Some(LifecycleRuleFilter {
+                prefix: Some("test/".to_string()),
+                ..Default::default()
+            }),
+            id: Some("del-marker-expiration".to_string()),
+            noncurrent_version_expiration: None,
+            noncurrent_version_transitions: None,
+            prefix: None,
+            transitions: None,
+        }],
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn compensation_driven_delete_marker_still_honors_lifecycle_cleanup() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+    let bucket_usecase = DefaultBucketUsecase::from_global();
+
+    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+    let backend = register_mock_tier(&tier_name).await;
+
+    let bucket = format!("test-comp-del-marker-cleanup-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/object.txt";
+    let payload = b"delete marker lifecycle should still clean up after compensation-driven transition";
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+    set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
+        .await
+        .expect("Failed to set transition lifecycle configuration");
+
+    with_forced_immediate_enqueue_timeout(&backend, || async {
+        let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
+    })
+    .await;
+
+    let transitioned = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
+        .await
+        .expect("object should eventually transition after compensation backfill");
+    assert_eq!(backend.put_count().await, 1, "compensation should upload the version once");
+    let remote_object = transitioned.transitioned_object.name.clone();
+
+    assert!(backend.contains(&remote_object).await);
+
+    let req = build_request(
+        DeleteObjectInput::builder()
+            .bucket(bucket.clone())
+            .key(object.to_string())
+            .build()
+            .unwrap(),
+        Method::DELETE,
+    );
+
+    Box::pin(usecase.execute_delete_object(req))
+        .await
+        .expect("Failed to issue versioned delete after compensation-driven transition");
+
+    assert!(
+        wait_for_delete_marker(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
+        "versioned delete should create a delete marker before lifecycle cleanup"
+    );
+    assert!(
+        backend.contains(&remote_object).await,
+        "delete marker creation should keep the transitioned remote object version"
+    );
+
+    let req = build_request(
+        PutBucketLifecycleConfigurationInput::builder()
+            .bucket(bucket.clone())
+            .lifecycle_configuration(Some(expiration_lifecycle_configuration("test/")))
+            .build()
+            .unwrap(),
+        Method::PUT,
+    );
+    let prepared = req
+        .input
+        .lifecycle_configuration
+        .clone()
+        .map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
+        .transpose();
+    bucket_usecase
+        .execute_put_bucket_lifecycle_configuration(req, prepared)
+        .await
+        .expect("Failed to update lifecycle configuration for delete marker cleanup");
+
+    assert!(
+        wait_for_delete_marker(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
+        "delete marker should remain visible after lifecycle update until cleanup completes"
+    );
+    assert!(
+        backend.contains(&remote_object).await,
+        "delete marker lifecycle cleanup should not remove the transitioned remote object version"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn put_bucket_lifecycle_configuration_expires_existing_objects() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultBucketUsecase::from_global();
+
+    let bucket = format!("test-api-expire-existing-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/existing.txt";
+    let payload = b"expire existing object after lifecycle update";
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+    let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
+
+    let req = build_request(
+        PutBucketLifecycleConfigurationInput::builder()
+            .bucket(bucket.clone())
+            .lifecycle_configuration(Some(expiration_lifecycle_configuration("test/")))
+            .build()
+            .unwrap(),
+        Method::PUT,
+    );
+
+    let prepared = req
+        .input
+        .lifecycle_configuration
+        .clone()
+        .map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
+        .transpose();
+    usecase
+        .execute_put_bucket_lifecycle_configuration(req, prepared)
+        .await
+        .expect("Failed to update lifecycle configuration");
+
+    assert!(
+        wait_for_delete_marker(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
+        "existing object should be lifecycle-deleted after lifecycle update"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn put_bucket_lifecycle_configuration_allows_expired_delete_marker_on_object_lock_bucket() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultBucketUsecase::from_global();
+
+    let bucket = format!("test-lock-expired-del-marker-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+    set_bucket_object_lock_enabled(bucket.as_str())
+        .await
+        .expect("Failed to enable object lock for bucket");
+
+    let req = build_request(
+        PutBucketLifecycleConfigurationInput::builder()
+            .bucket(bucket.clone())
+            .lifecycle_configuration(Some(expired_object_delete_marker_lifecycle_configuration()))
+            .build()
+            .unwrap(),
+        Method::PUT,
+    );
+
+    let prepared = req
+        .input
+        .lifecycle_configuration
+        .clone()
+        .map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
+        .transpose();
+    usecase
+        .execute_put_bucket_lifecycle_configuration(req, prepared)
+        .await
+        .expect("ExpiredObjectDeleteMarker should be accepted on object-lock bucket");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn put_bucket_lifecycle_configuration_rejects_del_marker_expiration_on_object_lock_bucket() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultBucketUsecase::from_global();
+
+    let bucket = format!("test-lock-del-marker-exp-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+    set_bucket_object_lock_enabled(bucket.as_str())
+        .await
+        .expect("Failed to enable object lock for bucket");
+
+    let req = build_request(
+        PutBucketLifecycleConfigurationInput::builder()
+            .bucket(bucket.clone())
+            .lifecycle_configuration(Some(del_marker_expiration_lifecycle_configuration(1)))
+            .build()
+            .unwrap(),
+        Method::PUT,
+    );
+
+    let prepared = req
+        .input
+        .lifecycle_configuration
+        .clone()
+        .map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
+        .transpose();
+    let err = usecase
+        .execute_put_bucket_lifecycle_configuration(req, prepared)
+        .await
+        .expect_err("DelMarkerExpiration must be rejected on object-lock bucket");
+    assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
+    let message = err.message().unwrap_or_default();
+    assert!(
+        message.contains(
+            "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket"
+        ),
+        "unexpected error message: {message}"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn put_bucket_lifecycle_configuration_rejects_zero_day_del_marker_expiration_on_object_lock_bucket() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultBucketUsecase::from_global();
+
+    let bucket = format!("test-lock-del-marker-exp0-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+    set_bucket_object_lock_enabled(bucket.as_str())
+        .await
+        .expect("Failed to enable object lock for bucket");
+
+    let req = build_request(
+        PutBucketLifecycleConfigurationInput::builder()
+            .bucket(bucket.clone())
+            .lifecycle_configuration(Some(del_marker_expiration_lifecycle_configuration(0)))
+            .build()
+            .unwrap(),
+        Method::PUT,
+    );
+
+    let prepared = req
+        .input
+        .lifecycle_configuration
+        .clone()
+        .map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
+        .transpose();
+    let err = usecase
+        .execute_put_bucket_lifecycle_configuration(req, prepared)
+        .await
+        .expect_err("DelMarkerExpiration with zero days must be rejected on object-lock bucket");
+    assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
+    let message = err.message().unwrap_or_default();
+    assert!(
+        message.contains("Days must be a positive integer with DelMarkerExpiration"),
+        "unexpected error message: {message}"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn put_bucket_lifecycle_configuration_rejects_zero_day_expiration() {
+    // S3 compatibility (ceph s3-tests `test_lifecycle_expiration_days0`): a rule with
+    // Expiration{Days:0} must be rejected with InvalidArgument through the real
+    // PutBucketLifecycleConfiguration handler, not merely at the crates/lifecycle
+    // io::Error layer. No object lock required.
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultBucketUsecase::from_global();
+
+    let bucket = format!("test-zero-day-exp-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+
+    let lifecycle = BucketLifecycleConfiguration {
+        expiry_updated_at: None,
+        rules: vec![LifecycleRule {
+            status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+            abort_incomplete_multipart_upload: None,
+            del_marker_expiration: None,
+            expiration: Some(LifecycleExpiration {
+                days: Some(0),
+                ..Default::default()
+            }),
+            filter: Some(LifecycleRuleFilter {
+                prefix: Some("days0/".to_string()),
+                ..Default::default()
+            }),
+            id: Some("expire-zero-days".to_string()),
+            noncurrent_version_expiration: None,
+            noncurrent_version_transitions: None,
+            prefix: None,
+            transitions: None,
+        }],
+    };
+
+    let req = build_request(
+        PutBucketLifecycleConfigurationInput::builder()
+            .bucket(bucket.clone())
+            .lifecycle_configuration(Some(lifecycle))
+            .build()
+            .unwrap(),
+        Method::PUT,
+    );
+
+    let prepared = req
+        .input
+        .lifecycle_configuration
+        .clone()
+        .map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
+        .transpose();
+    let err = usecase
+        .execute_put_bucket_lifecycle_configuration(req, prepared)
+        .await
+        .expect_err("zero-day expiration must be rejected with InvalidArgument");
+    assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
+    let message = err.message().unwrap_or_default();
+    assert!(
+        message.contains("'Days' for Expiration action must be a positive integer"),
+        "unexpected error message: {message}"
+    );
+}

rustfs/src/app/lifecycle_transition_api_test/multipart.rs

@@ -0,0 +1,157 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! CompleteMultipartUpload tests (batch E owns CompleteMultipartUpload).
+
+use super::common::*;
+use super::*;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn complete_multipart_upload_transitions_immediately_via_usecase() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultMultipartUsecase::from_global();
+
+    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+    let backend = register_mock_tier(&tier_name).await;
+
+    let bucket = format!("test-api-mpu-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/multipart.txt";
+    let payload = b"multipart immediate transition through complete API";
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+    set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
+        .await
+        .expect("Failed to set lifecycle configuration");
+
+    let upload = ecstore
+        .new_multipart_upload(bucket.as_str(), object, &ObjectOptions::default())
+        .await
+        .expect("Failed to create multipart upload");
+
+    let mut reader = PutObjReader::from_vec(payload.to_vec());
+    let uploaded_part = ecstore
+        .put_object_part(bucket.as_str(), object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
+        .await
+        .expect("Failed to upload multipart part");
+
+    let complete_input = CompleteMultipartUploadInput::builder()
+        .bucket(bucket.clone())
+        .key(object.to_string())
+        .upload_id(upload.upload_id.clone())
+        .multipart_upload(Some(CompletedMultipartUpload {
+            parts: Some(vec![CompletedPart {
+                part_number: Some(1),
+                e_tag: uploaded_part.etag.clone().map(|etag| to_s3s_etag(&etag)),
+                ..Default::default()
+            }]),
+        }))
+        .build()
+        .unwrap();
+
+    let complete_output = Box::pin(usecase.execute_complete_multipart_upload(build_request(complete_input, Method::POST)))
+        .await
+        .expect("Failed to complete multipart upload through usecase");
+    assert!(
+        complete_output.output.version_id.is_some(),
+        "versioned CompleteMultipartUpload should return the created version ID"
+    );
+
+    let info = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
+        .await
+        .expect("multipart object should transition immediately after complete usecase");
+
+    assert_eq!(info.transitioned_object.status, "complete");
+    assert_eq!(info.transitioned_object.tier, tier_name);
+    assert!(backend.contains(&info.transitioned_object.name).await);
+}
+
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+#[test]
+fn compensation_driven_complete_multipart_upload_still_transitions() {
+    std::thread::Builder::new()
+        .name("lifecycle-compensation-complete-multipart".to_string())
+        .stack_size(32 * 1024 * 1024)
+        .spawn(|| {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("compensation multipart test runtime should build");
+
+            runtime.block_on(compensation_driven_complete_multipart_upload_still_transitions_inner());
+        })
+        .expect("compensation multipart test thread should spawn")
+        .join()
+        .expect("compensation multipart test thread should finish");
+}
+
+async fn compensation_driven_complete_multipart_upload_still_transitions_inner() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultMultipartUsecase::from_global();
+
+    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+    let backend = register_mock_tier(&tier_name).await;
+
+    let bucket = format!("test-comp-mpu-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/multipart.txt";
+    let payload = b"multipart should still transition after compensation";
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+    set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
+        .await
+        .expect("Failed to set lifecycle configuration");
+
+    let upload = ecstore
+        .new_multipart_upload(bucket.as_str(), object, &ObjectOptions::default())
+        .await
+        .expect("Failed to create multipart upload");
+
+    let mut reader = PutObjReader::from_vec(payload.to_vec());
+    let uploaded_part = ecstore
+        .put_object_part(bucket.as_str(), object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
+        .await
+        .expect("Failed to upload multipart part");
+
+    let complete_input = CompleteMultipartUploadInput::builder()
+        .bucket(bucket.clone())
+        .key(object.to_string())
+        .upload_id(upload.upload_id.clone())
+        .multipart_upload(Some(CompletedMultipartUpload {
+            parts: Some(vec![CompletedPart {
+                part_number: Some(1),
+                e_tag: uploaded_part.etag.clone().map(|etag| to_s3s_etag(&etag)),
+                ..Default::default()
+            }]),
+        }))
+        .build()
+        .unwrap();
+
+    with_forced_immediate_enqueue_timeout(&backend, || async {
+        Box::pin(usecase.execute_complete_multipart_upload(build_request(complete_input, Method::POST)))
+            .await
+            .expect("Failed to complete multipart upload through usecase");
+    })
+    .await;
+
+    let info = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
+        .await
+        .expect("multipart object should eventually transition after compensation backfill");
+
+    assert_eq!(backend.put_count().await, 1, "compensation should upload the version once");
+    assert_eq!(info.transitioned_object.status, "complete");
+    assert_eq!(info.transitioned_object.tier, tier_name);
+    assert!(backend.contains(&info.transitioned_object.name).await);
+}

rustfs/src/app/lifecycle_transition_api_test/object_lock.rs

@@ -0,0 +1,113 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! PutObjectRetention and PutObjectLegalHold tests (batch B1 owns both).
+
+use super::common::*;
+use super::*;
+
+/// The object-lock handlers do not go through the object PUT path, so they must schedule
+/// replication themselves. Without it a retention or legal hold applied after upload stays
+/// local and the replica keeps its previous, unprotected state (a WORM object that is still
+/// deletable on the peer). This observes the shared decision counter: each handler must
+/// compute a replication decision exactly once when replication is configured, which is
+/// the step that drives both the persisted pending marker and the schedule.
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state usecase integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn object_lock_handlers_schedule_replication() {
+    use super::super::storage_api::object_usecase::bucket::replication::MUST_REPLICATE_OBJECT_CALLS;
+    use crate::app::storage_api::s3::S3;
+    use std::sync::atomic::Ordering;
+
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let fs = FS::new();
+
+    let bucket = format!("test-lock-repl-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "locked/object.txt";
+
+    (*ecstore)
+        .make_bucket(
+            bucket.as_str(),
+            &MakeBucketOptions {
+                lock_enabled: true,
+                versioning_enabled: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("Failed to create object-lock bucket");
+    upload_test_object(&ecstore, bucket.as_str(), object, b"worm payload").await;
+
+    // PutObjectRetention must compute a replication decision.
+    MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
+    let retention_input = PutObjectRetentionInput::builder()
+        .bucket(bucket.clone())
+        .key(object.to_string())
+        .retention(Some(ObjectLockRetention {
+            mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
+            retain_until_date: Some(time::macros::datetime!(2030-01-01 00:00:00 UTC).into()),
+        }))
+        .build()
+        .unwrap();
+    fs.put_object_retention(build_request(retention_input, Method::PUT))
+        .await
+        .expect("put_object_retention should succeed");
+    assert_eq!(
+        MUST_REPLICATE_OBJECT_CALLS.load(Ordering::SeqCst),
+        1,
+        "PutObjectRetention must compute the replication decision exactly once; 0 means the \
+         retention change never replicates and the peer copy stays unprotected"
+    );
+
+    // A bucket without a replication configuration can skip the decision.
+    MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
+    let no_replication_input = PutObjectLegalHoldInput::builder()
+        .bucket(bucket.clone())
+        .key(object.to_string())
+        .legal_hold(Some(ObjectLockLegalHold {
+            status: Some(ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF)),
+        }))
+        .build()
+        .unwrap();
+    fs.put_object_legal_hold(build_request(no_replication_input, Method::PUT))
+        .await
+        .expect("legal hold without replication should succeed");
+    assert_eq!(
+        MUST_REPLICATE_OBJECT_CALLS.load(Ordering::SeqCst),
+        0,
+        "a bucket without replication must skip the decision"
+    );
+
+    // A configured bucket must still evaluate the metadata replication decision.
+    install_delete_replication_config(&bucket, true, false, None).await;
+    MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
+    let legal_hold_input = PutObjectLegalHoldInput::builder()
+        .bucket(bucket.clone())
+        .key(object.to_string())
+        .legal_hold(Some(ObjectLockLegalHold {
+            status: Some(ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::ON)),
+        }))
+        .build()
+        .unwrap();
+    fs.put_object_legal_hold(build_request(legal_hold_input, Method::PUT))
+        .await
+        .expect("put_object_legal_hold should succeed");
+    assert_eq!(
+        MUST_REPLICATE_OBJECT_CALLS.load(Ordering::SeqCst),
+        1,
+        "PutObjectLegalHold must compute the replication decision exactly once; 0 means the \
+         legal hold never replicates and the peer copy stays deletable"
+    );
+}

rustfs/src/app/lifecycle_transition_api_test/put.rs

@@ -0,0 +1,221 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! PutObject tests (batch D owns PutObject).
+
+use super::common::*;
+use super::*;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn put_object_if_none_match_existing_object_returns_precondition_failed() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let fs = FS::new();
+    let usecase = DefaultObjectUsecase::from_global();
+
+    let bucket = format!("test-put-if-none-match-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/object.txt";
+    let initial_payload = b"initial conditional put payload";
+    let replacement_payload = b"replacement conditional put payload";
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+
+    let initial_input = PutObjectInput::builder()
+        .bucket(bucket.clone())
+        .key(object.to_string())
+        .body(Some(streaming_blob_from_bytes(initial_payload)))
+        .content_length(Some(initial_payload.len() as i64))
+        .build()
+        .unwrap();
+    Box::pin(usecase.execute_put_object(&fs, build_request(initial_input, Method::PUT)))
+        .await
+        .expect("Failed to upload initial object through usecase");
+
+    let existing_info = ecstore
+        .get_object_info(bucket.as_str(), object, &ObjectOptions::default())
+        .await
+        .expect("Failed to fetch existing object info");
+    let existing_etag = existing_info.etag.expect("existing object should have an ETag");
+    let initial_versions = live_object_version_count(&ecstore, bucket.as_str(), object).await;
+
+    for (condition, expected_code) in [
+        (ETagCondition::Any, S3ErrorCode::PreconditionFailed),
+        (ETagCondition::ETag(ETag::Strong(existing_etag)), S3ErrorCode::NotImplemented),
+        (
+            ETagCondition::ETag(ETag::Strong("different-etag".to_string())),
+            S3ErrorCode::NotImplemented,
+        ),
+    ] {
+        let header = condition.to_http_header().expect("conditional PUT header should encode");
+        // Cover header-only callers and the DTO-plus-header shape of parsed HTTP requests.
+        for if_none_match in [None, Some(condition)] {
+            let has_dto = if_none_match.is_some();
+            let replacement_input = PutObjectInput::builder()
+                .bucket(bucket.clone())
+                .key(object.to_string())
+                .body(Some(streaming_blob_from_bytes(replacement_payload)))
+                .content_length(Some(replacement_payload.len() as i64))
+                .if_none_match(if_none_match)
+                .build()
+                .expect("conditional replacement PUT input should build");
+            let mut req = build_request(replacement_input, Method::PUT);
+            req.headers.insert(IF_NONE_MATCH, header.clone());
+
+            let err = Box::pin(usecase.execute_put_object(&fs, req))
+                .await
+                .expect_err("conditional PUT must reject the replacement");
+            assert_eq!(err.code(), &expected_code, "header={header:?}, has_dto={has_dto}");
+            assert_eq!(
+                read_object_bytes(&ecstore, bucket.as_str(), object).await,
+                initial_payload,
+                "failed conditional PutObject must not overwrite the current object"
+            );
+            assert_eq!(
+                live_object_version_count(&ecstore, bucket.as_str(), object).await,
+                initial_versions,
+                "failed conditional PutObject must not create a version"
+            );
+        }
+    }
+}
+
+/// rustfs/backlog#1320: a single PUT must compute the replication decision
+/// exactly once. Historically the PUT path computed `must_replicate_object`
+/// twice with the same inputs — once before commit to persist the pending
+/// replication metadata, and again after commit to drive the schedule. Besides
+/// repeating the versioning/config/target traversal on the hot path, a
+/// replication-config hot update between the two computations could split the
+/// two phases (pending-without-schedule or schedule-without-pending). The fix
+/// reuses one immutable decision for both phases; this test observes the
+/// computation counter and fails if a second computation is reintroduced.
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state usecase integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn put_object_computes_replication_decision_exactly_once() {
+    use super::super::storage_api::object_usecase::bucket::replication::MUST_REPLICATE_OBJECT_CALLS;
+    use std::sync::atomic::Ordering;
+
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let fs = FS::new();
+    let usecase = DefaultObjectUsecase::from_global();
+
+    let bucket = format!("test-repl-decision-once-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/object.txt";
+    let payload = b"replication decision single-compute payload";
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+
+    // Reset immediately before the single PUT so the counter reflects only this
+    // request. `#[serial]` guarantees no other usecase PUT runs concurrently, so
+    // no unrelated `must_replicate_object` call can perturb the count.
+    MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
+
+    let input = PutObjectInput::builder()
+        .bucket(bucket.clone())
+        .key(object.to_string())
+        .body(Some(streaming_blob_from_bytes(payload)))
+        .content_length(Some(payload.len() as i64))
+        .build()
+        .unwrap();
+    Box::pin(usecase.execute_put_object(&fs, build_request(input, Method::PUT)))
+        .await
+        .expect("Failed to upload object through usecase");
+
+    let calls = MUST_REPLICATE_OBJECT_CALLS.load(Ordering::SeqCst);
+    assert_eq!(
+        calls, 1,
+        "a single PUT must compute the replication decision exactly once; reverting to a \
+         pre-commit + post-commit recompute makes this observe 2 (rustfs/backlog#1320)"
+    );
+}
+
+/// Snowball auto-extract writes each archive member as an independent object,
+/// so each member must join bucket replication like a regular PUT (MinIO
+/// PutObjectExtract parity). The archive holds two file entries; a count other
+/// than 2 means extracted objects bypass the replication decision entirely and
+/// stay local forever.
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state usecase integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn put_object_extract_computes_replication_decision_per_entry() {
+    use super::super::storage_api::object_usecase::bucket::replication::MUST_REPLICATE_OBJECT_CALLS;
+    use super::super::storage_api::test::ReqInfo;
+    use super::super::storage_api::test::bucket::metadata::BUCKET_POLICY_CONFIG;
+    use std::io::Cursor;
+    use std::sync::atomic::Ordering;
+
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+
+    let bucket = format!("test-extract-repl-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+
+    // The extract path re-authorizes every extracted entry internally. The test
+    // harness has no IAM system, so allow anonymous PutObject via bucket policy.
+    let policy_json = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Principal": "*",
+                "Action": ["s3:PutObject"],
+                "Resource": [format!("arn:aws:s3:::{bucket}/*")]
+            }
+        ]
+    })
+    .to_string();
+    metadata_sys::update(bucket.as_str(), BUCKET_POLICY_CONFIG, policy_json.into_bytes())
+        .await
+        .expect("Failed to install anonymous put bucket policy");
+
+    let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
+    for (path, data) in [
+        ("member-one.txt", b"first member payload".as_slice()),
+        ("nested/member-two.txt", b"second member payload".as_slice()),
+    ] {
+        let mut header = tokio_tar::Header::new_gnu();
+        header.set_size(data.len() as u64);
+        header.set_mode(0o644);
+        header.set_cksum();
+        builder
+            .append_data(&mut header, path, Cursor::new(data))
+            .await
+            .expect("Failed to append archive member");
+    }
+    let archive = builder.into_inner().await.expect("Failed to finish archive").into_inner();
+
+    MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
+
+    let input = PutObjectInput::builder()
+        .bucket(bucket.clone())
+        .key("archive.tar".to_string())
+        .body(Some(streaming_blob_from_bytes(&archive)))
+        .content_length(Some(archive.len() as i64))
+        .build()
+        .unwrap();
+    let mut req = build_request(input, Method::PUT);
+    req.extensions.insert(ReqInfo::default());
+    Box::pin(usecase.execute_put_object_extract(req))
+        .await
+        .expect("Failed to extract archive through usecase");
+
+    assert_eq!(
+        MUST_REPLICATE_OBJECT_CALLS.load(Ordering::SeqCst),
+        2,
+        "snowball auto-extract must compute one replication decision per extracted member \
+         object; 0 means extracted objects bypass bucket replication (P0-6 companion, MinIO \
+         PutObjectExtract parity)"
+    );
+}

rustfs/src/app/lifecycle_transition_api_test/restore.rs

@@ -0,0 +1,492 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! RestoreObject tests (batch E owns RestoreObject).
+
+use super::common::*;
+use super::*;
+
+const RESTORE_SUSPENDED_WAIT_TIMEOUT: Duration = Duration::from_secs(15);
+
+async fn suspend_test_bucket(bucket: &str) {
+    DefaultBucketUsecase::from_global()
+        .execute_put_bucket_versioning(
+            build_request(
+                PutBucketVersioningInput::builder()
+                    .bucket(bucket.to_string())
+                    .versioning_configuration(VersioningConfiguration {
+                        status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
+                        ..Default::default()
+                    })
+                    .build()
+                    .expect("suspended versioning request should build"),
+                Method::PUT,
+            ),
+            Ok(rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
+                status: Some("Suspended".to_owned()),
+                ..Default::default()
+            }),
+        )
+        .await
+        .expect("bucket versioning should be suspended");
+}
+
+async fn transition_uploaded_object_directly(
+    ecstore: &Arc<ECStore>,
+    bucket: &str,
+    object: &str,
+    tier_name: &str,
+    uploaded: &ObjectInfo,
+) -> ObjectInfo {
+    let transition_opts = ObjectOptions {
+        transition: lifecycle::lifecycle_contract::TransitionOptions {
+            status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(),
+            tier: tier_name.to_string(),
+            etag: uploaded.etag.clone().unwrap_or_default(),
+            ..Default::default()
+        },
+        version_id: uploaded.version_id.map(|version| version.to_string()),
+        versioned: uploaded.version_id.is_some(),
+        mod_time: uploaded.mod_time,
+        ..Default::default()
+    };
+    ecstore
+        .transition_object(bucket, object, &transition_opts)
+        .await
+        .expect("Failed to transition object directly");
+    wait_for_transition(ecstore, bucket, object, TRANSITION_WAIT_TIMEOUT)
+        .await
+        .expect("object should transition before restore assertions")
+}
+
+async fn wait_for_restore_completion(
+    ecstore: &Arc<ECStore>,
+    backend: &MockWarmBackend,
+    bucket: &str,
+    object: &str,
+    timeout: Duration,
+) -> Result<ObjectInfo, String> {
+    let deadline = tokio::time::Instant::now() + timeout;
+    let mut last_state = None;
+
+    loop {
+        if tokio::time::Instant::now() >= deadline {
+            let tier_gets = backend.get_count().await;
+            let op_log = backend.op_log().await;
+            return Err(format!(
+                "restore copy-back should complete within {timeout:?}; tier_gets={tier_gets}, op_log={op_log:?}; last observed state: {}",
+                last_state.unwrap_or_else(|| "no object info observed".to_string())
+            ));
+        }
+
+        match (**ecstore).get_object_info(bucket, object, &ObjectOptions::default()).await {
+            Ok(info) => {
+                if !info.restore_ongoing && info.restore_expires.is_some() {
+                    return Ok(info);
+                }
+                last_state = Some(format!(
+                    "restore_ongoing={}, restore_expires={:?}, transitioned_status={}",
+                    info.restore_ongoing, info.restore_expires, info.transitioned_object.status
+                ));
+            }
+            Err(err) => {
+                last_state = Some(format!("get_object_info failed: {err}"));
+            }
+        }
+
+        tokio::time::sleep(Duration::from_millis(500)).await;
+    }
+}
+
+/// backlog#1148 ilm-8: the RestoreObject API surface on a transitioned object.
+///
+/// POST restore(days=1) is accepted and flips the object to
+/// `x-amz-restore: ongoing-request="true"` while the mock tier GET barrier
+/// proves the background copy-back has reached the remote read; a second POST
+/// during that window is rejected with 409 `RestoreAlreadyInProgress`.
+/// Synchronous SetDisks transition tests cover copy-back completion, restore
+/// metadata, and local byte-identical reads.
+///
+/// Re-enabled in the serial lane by backlog#1304: the accept path now flips
+/// the ongoing flag under a short compare-and-set guard and the copy-back
+/// runs without the #4877 whole-copy-back lock, so the mid-restore
+/// `ongoing-request="true"` read and the fast 409 rejection asserted here are
+/// the implemented contract.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-8)"]
+async fn restore_object_usecase_reports_ongoing_conflict() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+
+    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+    let backend = register_mock_tier(&tier_name).await;
+
+    let bucket = format!("test-api-restore-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    // Keep the object under the shared ILM test prefix even though this setup
+    // transitions it directly; it keeps diagnostics aligned with sibling tests.
+    let object = "test/restore/api-object.bin";
+    let payload: Vec<u8> = (0..128 * 1024).map(|i| (i % 251) as u8).collect();
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+    let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
+    assert!(uploaded.version_id.is_none(), "fixture must create the null version");
+    let _ = transition_uploaded_object_directly(&ecstore, bucket.as_str(), object, &tier_name, &uploaded).await;
+    backend.clear_op_log().await;
+    let get_barrier = backend.arm_get_barrier().await;
+
+    let restore_request = || RestoreRequest {
+        days: Some(1),
+        description: None,
+        glacier_job_parameters: None,
+        output_location: None,
+        select_parameters: None,
+        tier: None,
+        type_: None,
+    };
+    let restore_input = || {
+        RestoreObjectInput::builder()
+            .bucket(bucket.clone())
+            .key(object.to_string())
+            .restore_request(Some(restore_request()))
+            .build()
+            .unwrap()
+    };
+
+    Box::pin(usecase.execute_restore_object(build_request(restore_input(), Method::POST)))
+        .await
+        .expect("restore request should be accepted");
+
+    get_barrier.wait_until_paused().await;
+
+    // The barrier proves the detached copy-back reached the tier GET and is
+    // still paused, so the ongoing state and conflict rejection are not timing
+    // assumptions about task scheduling.
+    let ongoing = ecstore
+        .get_object_info(bucket.as_str(), object, &ObjectOptions::default())
+        .await
+        .expect("Failed to load object info during restore");
+    assert!(
+        ongoing.restore_ongoing,
+        "x-amz-restore must report ongoing-request=true while the copy-back tier GET is paused"
+    );
+
+    // A second restore while one is in flight is rejected.
+    let err = Box::pin(usecase.execute_restore_object(build_request(restore_input(), Method::POST)))
+        .await
+        .expect_err("a second restore while one is ongoing must be rejected");
+    assert_eq!(
+        err.code(),
+        &S3ErrorCode::RestoreAlreadyInProgress,
+        "unexpected rejection for a repeated restore: {err:?}"
+    );
+
+    get_barrier.release();
+}
+
+/// rustfs/backlog#1337: cancellation after the ongoing metadata commit but
+/// before detached worker creation must not strand the object forever. The
+/// replacement POST proves the abandoned v1 generation has no live worker,
+/// atomically supersedes it, and performs exactly one remote copy-back.
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1337"]
+#[test]
+fn restore_object_usecase_recovers_cancelled_post_commit_generation() {
+    std::thread::Builder::new()
+        .name("lifecycle-restore-orphan-recovery".to_string())
+        .stack_size(32 * 1024 * 1024)
+        .spawn(|| {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("orphan restore recovery test runtime should build");
+            runtime.block_on(restore_object_usecase_recovers_cancelled_post_commit_generation_inner());
+        })
+        .expect("orphan restore recovery test thread should spawn")
+        .join()
+        .expect("orphan restore recovery test thread should finish");
+}
+
+async fn restore_object_usecase_recovers_cancelled_post_commit_generation_inner() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+    let backend = register_mock_tier(&tier_name).await;
+    let bucket = format!("test-api-restore-orphan-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/restore/orphaned-generation.bin";
+    let payload: Vec<u8> = (0..128 * 1024).map(|i| (i % 251) as u8).collect();
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+    let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
+    let _ = transition_uploaded_object_directly(&ecstore, bucket.as_str(), object, &tier_name, &uploaded).await;
+    backend.clear_op_log().await;
+    let tier_gets_before_restore = backend.get_count().await;
+
+    let restore_input = || {
+        RestoreObjectInput::builder()
+            .bucket(bucket.clone())
+            .key(object.to_string())
+            .restore_request(Some(RestoreRequest {
+                days: Some(1),
+                description: None,
+                glacier_job_parameters: None,
+                output_location: None,
+                select_parameters: None,
+                tier: None,
+                type_: None,
+            }))
+            .build()
+            .expect("restore request should build")
+    };
+
+    let commit_barrier = crate::app::object::RestoreStatusCommitBarrier::install(bucket.as_str(), object);
+    let first_input = restore_input();
+    let first = tokio::spawn(async move {
+        DefaultObjectUsecase::from_global()
+            .execute_restore_object(build_request(first_input, Method::POST))
+            .await
+    });
+    commit_barrier.wait_until_paused().await;
+
+    first.abort();
+    assert!(first.await.expect_err("the first request must be cancelled").is_cancelled());
+    drop(commit_barrier);
+
+    let orphaned = ecstore
+        .get_object_info(bucket.as_str(), object, &ObjectOptions::default())
+        .await
+        .expect("the committed orphan generation should remain readable");
+    assert!(orphaned.restore_ongoing, "the interrupted request must have committed ongoing=true");
+    assert_eq!(
+        rustfs_utils::http::get_consistent_str(orphaned.user_defined.as_ref(), rustfs_utils::http::SUFFIX_RESTORE_WORKER_LOCK,),
+        Some(rustfs_utils::http::RESTORE_WORKER_LOCK_PROTOCOL_V1),
+        "recoverable generations must advertise the worker-lock protocol"
+    );
+    let orphaned_operation_id =
+        rustfs_utils::http::get_consistent_str(orphaned.user_defined.as_ref(), rustfs_utils::http::SUFFIX_RESTORE_OPERATION_ID)
+            .and_then(|value| Uuid::parse_str(value).ok())
+            .expect("recoverable generations must persist a non-nil operation id");
+    assert!(!orphaned_operation_id.is_nil());
+
+    DefaultObjectUsecase::from_global()
+        .execute_restore_object(build_request(restore_input(), Method::POST))
+        .await
+        .expect("a new POST must supersede the committed generation whose worker lock was released");
+
+    let completed = wait_for_restore_completion(&ecstore, &backend, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
+        .await
+        .unwrap_or_else(|err| panic!("{err}"));
+    assert!(!completed.restore_ongoing);
+    assert!(completed.restore_expires.is_some());
+    assert!(
+        rustfs_utils::http::get_str(completed.user_defined.as_ref(), rustfs_utils::http::SUFFIX_RESTORE_OPERATION_ID,).is_none(),
+        "completed replacement restore must consume its operation id"
+    );
+    assert!(
+        rustfs_utils::http::get_str(completed.user_defined.as_ref(), rustfs_utils::http::SUFFIX_RESTORE_WORKER_LOCK,).is_none(),
+        "completed replacement restore must consume its liveness marker"
+    );
+    assert_eq!(
+        backend.get_count().await - tier_gets_before_restore,
+        1,
+        "the cancelled pre-spawn generation must not issue a tier GET"
+    );
+}
+
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#4879"]
+#[test]
+fn restore_object_usecase_completes_suspended_null_version_in_place() {
+    std::thread::Builder::new()
+        .name("lifecycle-restore-suspended-null".to_string())
+        .stack_size(32 * 1024 * 1024)
+        .spawn(|| {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("suspended null-version restore test runtime should build");
+
+            runtime.block_on(restore_object_usecase_completes_suspended_null_version_in_place_inner());
+        })
+        .expect("suspended null-version restore test thread should spawn")
+        .join()
+        .expect("suspended null-version restore test thread should finish");
+}
+
+async fn restore_object_usecase_completes_suspended_null_version_in_place_inner() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+    let backend = register_mock_tier(&tier_name).await;
+    let bucket = format!("test-api-restore-suspended-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/restore/suspended-null.bin";
+    let payload: Vec<u8> = (0..128 * 1024).map(|i| (i % 251) as u8).collect();
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+    let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
+    assert!(uploaded.version_id.is_none(), "fixture must create the null version");
+    let _ = transition_uploaded_object_directly(&ecstore, bucket.as_str(), object, &tier_name, &uploaded).await;
+    suspend_test_bucket(bucket.as_str()).await;
+    backend.clear_op_log().await;
+    let tier_gets_before_restore = backend.get_count().await;
+    let get_barrier = backend.arm_get_barrier().await;
+
+    Box::pin(
+        usecase.execute_restore_object(build_request(
+            RestoreObjectInput::builder()
+                .bucket(bucket.clone())
+                .key(object.to_string())
+                .restore_request(Some(RestoreRequest {
+                    days: Some(1),
+                    description: None,
+                    glacier_job_parameters: None,
+                    output_location: None,
+                    select_parameters: None,
+                    tier: None,
+                    type_: None,
+                }))
+                .build()
+                .expect("restore request should build"),
+            Method::POST,
+        )),
+    )
+    .await
+    .expect("suspended null-version restore should be accepted");
+
+    get_barrier.wait_until_paused().await;
+    get_barrier.release();
+    let completed = wait_for_restore_completion(&ecstore, &backend, bucket.as_str(), object, RESTORE_SUSPENDED_WAIT_TIMEOUT)
+        .await
+        .unwrap_or_else(|err| panic!("{err}"));
+
+    assert!(!completed.restore_ongoing, "the original null version must complete in place");
+    assert!(completed.restore_expires.is_some(), "completed restore must carry an expiry");
+    assert!(
+        completed.version_id.is_none() || completed.version_id.is_some_and(|version_id| version_id.is_nil()),
+        "suspended restore must remain on the null version"
+    );
+    assert_eq!(
+        live_object_version_count(&ecstore, bucket.as_str(), object).await,
+        1,
+        "suspended restore must not create a UUID version"
+    );
+    assert_eq!(
+        backend.get_count().await - tier_gets_before_restore,
+        1,
+        "suspended restore copy-back must fetch the tier exactly once"
+    );
+}
+
+/// backlog#1304: the restore-accept compare-and-set itself, under real
+/// concurrency. Two POST ?restore for the same transitioned object race each
+/// other from separate tasks; the accept guard must let exactly one through —
+/// the winner flips `ongoing-request="true"`, the loser must read that flag
+/// and be rejected with 409 `RestoreAlreadyInProgress` — and the tier must
+/// see exactly one copy-back GET, never two. Without the accept guard both
+/// requests can observe ongoing=false and double-start the copy-back (the
+/// #4859 race), which this test catches via the tier GET count.
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-8)"]
+async fn restore_object_usecase_accepts_exactly_one_of_two_concurrent_restores() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+
+    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+    let backend = register_mock_tier(&tier_name).await;
+
+    let bucket = format!("test-api-restore-cas-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    // Keep the object under the shared ILM test prefix for diagnostics parity.
+    let object = "test/restore/cas-object.bin";
+    let payload: Vec<u8> = (0..128 * 1024).map(|i| (i % 251) as u8).collect();
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+    let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
+    let _ = transition_uploaded_object_directly(&ecstore, bucket.as_str(), object, &tier_name, &uploaded).await;
+    backend.clear_op_log().await;
+
+    // Hold the accepted copy-back at the tier GET until both accept attempts
+    // return, so the loser cannot observe an already-restored object.
+    let get_barrier = backend.arm_get_barrier().await;
+
+    let tier_gets_before_restore = backend.get_count().await;
+
+    let restore_input = || {
+        RestoreObjectInput::builder()
+            .bucket(bucket.clone())
+            .key(object.to_string())
+            .restore_request(Some(RestoreRequest {
+                days: Some(1),
+                description: None,
+                glacier_job_parameters: None,
+                output_location: None,
+                select_parameters: None,
+                tier: None,
+                type_: None,
+            }))
+            .build()
+            .unwrap()
+    };
+    let spawn_restore = |input: RestoreObjectInput| {
+        tokio::spawn(async move {
+            let usecase = DefaultObjectUsecase::from_global();
+            Box::pin(usecase.execute_restore_object(build_request(input, Method::POST))).await
+        })
+    };
+
+    let (first, second) = tokio::join!(spawn_restore(restore_input()), spawn_restore(restore_input()));
+    let results = [
+        first.expect("first concurrent restore task panicked"),
+        second.expect("second concurrent restore task panicked"),
+    ];
+
+    let accepted = results.iter().filter(|result| result.is_ok()).count();
+    assert_eq!(accepted, 1, "exactly one of two concurrent restores must be accepted, got {accepted}");
+    let rejection = results
+        .iter()
+        .find_map(|result| result.as_ref().err())
+        .expect("the losing concurrent restore must surface an error");
+    assert_eq!(
+        rejection.code(),
+        &S3ErrorCode::RestoreAlreadyInProgress,
+        "the losing concurrent restore must be rejected as already in progress: {rejection:?}"
+    );
+
+    get_barrier.wait_until_paused().await;
+    get_barrier.release();
+
+    // This test is scoped to the accept CAS: completion, expiry metadata, and
+    // local restored GET service are covered by the single-request restore test
+    // above. Here it is enough to prove exactly one copy-back was admitted.
+    let expected_tier_gets = tier_gets_before_restore + 1;
+    let deadline = tokio::time::Instant::now() + TRANSITION_WAIT_TIMEOUT;
+    loop {
+        let actual_tier_gets = backend.get_count().await;
+        if actual_tier_gets >= expected_tier_gets {
+            assert_eq!(
+                actual_tier_gets - tier_gets_before_restore,
+                1,
+                "two concurrent restore requests must trigger exactly one tier copy-back GET"
+            );
+            break;
+        }
+        if tokio::time::Instant::now() >= deadline {
+            let op_log = backend.op_log().await;
+            panic!(
+                "mock tier should record exactly one restore GET within {TRANSITION_WAIT_TIMEOUT:?}; \
+                 tier_gets_before_restore={tier_gets_before_restore}, actual_tier_gets={actual_tier_gets}, op_log={op_log:?}"
+            );
+        }
+
+        tokio::time::sleep(Duration::from_millis(50)).await;
+    }
+}

rustfs/src/app/lifecycle_transition_api_test/transition.rs

@@ -0,0 +1,498 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! ILM transition-engine tests that call no S3 use-case; owned by no operation batch.
+
+use super::common::*;
+use super::*;
+
+const ENV_GET_CODEC_STREAMING_ENABLE: &str = "RUSTFS_GET_CODEC_STREAMING_ENABLE";
+
+const ENV_GET_CODEC_STREAMING_ROLLOUT: &str = "RUSTFS_GET_CODEC_STREAMING_ROLLOUT";
+
+const ENV_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED: &str = "RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED";
+
+const ENV_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED: &str = "RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED";
+
+const ENV_GET_CODEC_STREAMING_MIN_SIZE: &str = "RUSTFS_GET_CODEC_STREAMING_MIN_SIZE";
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn transition_wait_timeout_reports_metadata_state_and_read_failure() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let bucket = format!("test-transition-wait-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    create_test_bucket(&ecstore, &bucket).await;
+    upload_test_object(&ecstore, &bucket, "plain.txt", b"not transitioned").await;
+
+    let pending = wait_for_transition(&ecstore, &bucket, "plain.txt", Duration::ZERO)
+        .await
+        .expect_err("an untransitioned object must still fail the observation deadline");
+    assert!(pending.contains("transitioned_status=\"\""), "{pending}");
+    assert!(pending.contains("compensation_pending="), "{pending}");
+    assert!(pending.contains("compensation_running="), "{pending}");
+
+    let missing = wait_for_transition(&ecstore, &bucket, "missing.txt", Duration::ZERO)
+        .await
+        .expect_err("metadata read failures must not become a completed transition");
+    assert!(missing.contains("metadata_error=ObjectNotFound"), "{missing}");
+}
+
+async fn with_get_codec_streaming_remote_probe_env<F, Fut>(test_fn: F)
+where
+    F: FnOnce() -> Fut,
+    Fut: std::future::Future<Output = ()>,
+{
+    let metrics_was_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
+    rustfs_io_metrics::set_get_stage_metrics_enabled(true);
+    let result = std::panic::AssertUnwindSafe(temp_env::async_with_vars(
+        [
+            (ENV_GET_CODEC_STREAMING_ENABLE, Some("true")),
+            (ENV_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")),
+            (ENV_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
+            (ENV_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
+            (ENV_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
+        ],
+        test_fn(),
+    ))
+    .catch_unwind()
+    .await;
+    rustfs_io_metrics::set_get_stage_metrics_enabled(metrics_was_enabled);
+    if let Err(err) = result {
+        std::panic::resume_unwind(err);
+    }
+}
+
+#[test]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+fn get_transitioned_object_uses_remote_codec_fallback_path() {
+    // CI's default test stack overflows this deep async path; run it on a dedicated large-stack thread.
+    std::thread::Builder::new()
+        .name("lifecycle-transition-remote-codec-fallback".to_string())
+        .stack_size(16 * 1024 * 1024)
+        .spawn(|| {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("remote codec fallback test runtime should build");
+
+            runtime.block_on(async {
+                with_get_codec_streaming_remote_probe_env(|| async {
+                    let (_disk_paths, ecstore) = setup_test_env().await;
+
+                    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+                    let backend = register_mock_tier(&tier_name).await;
+
+                    let bucket = format!("test-api-get-remote-{}", &Uuid::new_v4().simple().to_string()[..8]);
+                    let object = "test/remote-codec-fallback.txt";
+                    let payload: Vec<u8> = (0..(1024 * 1024))
+                        .map(|index| u8::try_from(index % 251).expect("payload byte fits in u8"))
+                        .collect();
+
+                    create_test_bucket(&ecstore, bucket.as_str()).await;
+
+                    let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
+                    let transition_opts = ObjectOptions {
+                        transition: lifecycle::lifecycle_contract::TransitionOptions {
+                            status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(),
+                            tier: tier_name.clone(),
+                            etag: uploaded.etag.clone().unwrap_or_default(),
+                            ..Default::default()
+                        },
+                        version_id: uploaded.version_id.map(|version| version.to_string()),
+                        versioned: true,
+                        mod_time: uploaded.mod_time,
+                        ..Default::default()
+                    };
+                    ecstore
+                        .transition_object(bucket.as_str(), object, &transition_opts)
+                        .await
+                        .expect("Failed to transition object directly");
+
+                    let transitioned = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
+                        .await
+                        .expect("object should transition before remote fallback GET");
+
+                    assert_eq!(transitioned.transitioned_object.status, "complete");
+                    assert_eq!(transitioned.transitioned_object.tier, tier_name);
+                    assert!(!transitioned.transitioned_object.name.is_empty());
+                    assert!(backend.contains(&transitioned.transitioned_object.name).await);
+
+                    let actual = read_object_bytes(&ecstore, bucket.as_str(), object).await;
+                    assert_eq!(actual, payload);
+                })
+                .await;
+            });
+        })
+        .expect("remote codec fallback test thread should spawn")
+        .join()
+        .expect("remote codec fallback test thread should not panic");
+}
+
+/// Regression test for rustfs/rustfs#4827: a duplicate transition task that runs
+/// after the winning transition completed must return early without a second
+/// tier upload and without clobbering the winner's remote object / metadata.
+///
+/// The duplicate models the residual race left after the in-flight enqueue dedup
+/// (#4839): a stale-snapshot task admitted after the winner released its claim.
+/// On an unversioned bucket the duplicate's metadata re-read is eligible for the
+/// GET metadata cache; before the fix the cache still held the pre-transition
+/// entry (transition_object never invalidated it after persisting), so the
+/// TRANSITION_COMPLETE early-return never fired and the duplicate streamed the
+/// already-deleted local data to the tier again (NotFound reader errors +
+/// rejected duplicate tier PUT).
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn duplicate_transition_task_skips_already_transitioned_version() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+
+    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+    let backend = register_mock_tier(&tier_name).await;
+
+    let bucket = format!("test-api-dup-transition-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/duplicate-transition.txt";
+    let payload = b"duplicate transition tasks must not re-upload";
+
+    // Unversioned bucket: the duplicate's re-read goes through the GET metadata
+    // cache (versioned reads bypass it), which is the stale-read window under test.
+    (*ecstore)
+        .make_bucket(bucket.as_str(), &MakeBucketOptions::default())
+        .await
+        .expect("Failed to create unversioned test bucket");
+
+    let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
+    let transition_opts = ObjectOptions {
+        transition: lifecycle::lifecycle_contract::TransitionOptions {
+            status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(),
+            tier: tier_name.clone(),
+            etag: uploaded.etag.clone().unwrap_or_default(),
+            ..Default::default()
+        },
+        version_id: uploaded.version_id.map(|version| version.to_string()),
+        mod_time: uploaded.mod_time,
+        ..Default::default()
+    };
+
+    // Winner: transitions the object, persists transition_status=complete, and
+    // deletes the local data.
+    ecstore
+        .transition_object(bucket.as_str(), object, &transition_opts)
+        .await
+        .expect("winning transition should succeed");
+
+    let puts_after_winner: Vec<String> = backend
+        .op_log()
+        .await
+        .into_iter()
+        .filter_map(|op| match op {
+            MockWarmOp::Put { object, .. } => Some(object),
+            _ => None,
+        })
+        .collect();
+    assert_eq!(puts_after_winner.len(), 1, "winner must upload exactly one tier object");
+    let winner_remote_object = puts_after_winner[0].clone();
+
+    // Duplicate task with the same stale snapshot (same mod_time / etag), run
+    // immediately so the pre-fix stale metadata-cache entry is still live. It
+    // must observe transition_status=complete and return Ok without uploading.
+    ecstore
+        .transition_object(bucket.as_str(), object, &transition_opts)
+        .await
+        .expect("duplicate transition must return early without error");
+
+    let puts_after_duplicate = backend
+        .op_log()
+        .await
+        .into_iter()
+        .filter(|op| matches!(op, MockWarmOp::Put { .. }))
+        .count();
+    assert_eq!(puts_after_duplicate, 1, "duplicate transition must not attempt a second tier upload");
+
+    // The winner's remote object and local metadata must be intact.
+    assert!(backend.contains(&winner_remote_object).await);
+    let info = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
+        .await
+        .expect("object must remain transitioned after duplicate task");
+    assert_eq!(info.transitioned_object.status, "complete");
+    assert_eq!(info.transitioned_object.tier, tier_name);
+    assert_eq!(
+        info.transitioned_object.name, winner_remote_object,
+        "duplicate transition must not repoint metadata at a new remote object"
+    );
+    assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), object).await, payload);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn transition_rejects_stale_etag_before_remote_upload() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+    let backend = register_mock_tier(&tier_name).await;
+    let bucket = format!("test-api-transition-stale-etag-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/transition-stale-etag.txt";
+    (*ecstore)
+        .make_bucket(bucket.as_str(), &MakeBucketOptions::default())
+        .await
+        .expect("unversioned test bucket should be created");
+    let original = upload_test_object(&ecstore, bucket.as_str(), object, b"original transition candidate").await;
+    let replacement_payload = b"replacement committed before the stale transition starts";
+    let replacement = upload_test_object(&ecstore, bucket.as_str(), object, replacement_payload).await;
+    let transition_opts = ObjectOptions {
+        transition: lifecycle::lifecycle_contract::TransitionOptions {
+            status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(),
+            tier: tier_name,
+            etag: original.etag.clone().unwrap_or_default(),
+            ..Default::default()
+        },
+        version_id: replacement.version_id.map(|version| version.to_string()),
+        mod_time: replacement.mod_time,
+        ..Default::default()
+    };
+
+    ecstore
+        .transition_object(bucket.as_str(), object, &transition_opts)
+        .await
+        .expect_err("stale ETag must reject the transition before remote upload");
+    assert_eq!(backend.put_count().await, 0, "rejected stale work must not upload a remote candidate");
+    assert_eq!(
+        read_object_bytes(&ecstore, bucket.as_str(), object).await,
+        replacement_payload,
+        "the replacement object must remain local and readable"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn transition_commit_rejects_replaced_source() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+    let backend = register_mock_tier(&tier_name).await;
+    let bucket = format!("test-api-transition-replace-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/transition-replaced-source.txt";
+    let original_payload = b"transition source that must not replace a concurrent overwrite";
+    let replacement_payload = b"concurrent replacement must remain the authoritative local object";
+
+    (*ecstore)
+        .make_bucket(bucket.as_str(), &MakeBucketOptions::default())
+        .await
+        .expect("unversioned test bucket should be created");
+    let original = upload_test_object(&ecstore, bucket.as_str(), object, original_payload).await;
+    let transition_opts = ObjectOptions {
+        transition: lifecycle::lifecycle_contract::TransitionOptions {
+            status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(),
+            tier: tier_name.clone(),
+            etag: original.etag.clone().unwrap_or_default(),
+            ..Default::default()
+        },
+        version_id: original.version_id.map(|version| version.to_string()),
+        mod_time: original.mod_time,
+        ..Default::default()
+    };
+
+    let put_barrier = backend.arm_put_barrier().await;
+    let transition_store = Arc::clone(&ecstore);
+    let transition_bucket = bucket.clone();
+    let transition = tokio::spawn(async move {
+        transition_store
+            .transition_object(&transition_bucket, object, &transition_opts)
+            .await
+    });
+    put_barrier.wait_until_paused().await;
+
+    upload_test_object(&ecstore, bucket.as_str(), object, replacement_payload).await;
+    put_barrier.release();
+
+    transition
+        .await
+        .expect("transition task should not panic")
+        .expect_err("transition must reject a source replaced during remote upload");
+    assert_eq!(
+        backend.put_count().await,
+        1,
+        "the stale transition should upload exactly one remote candidate"
+    );
+    assert_eq!(
+        backend.remove_count().await,
+        1,
+        "the uncommitted remote candidate must be removed after identity revalidation fails"
+    );
+    assert_eq!(
+        backend.remove_versions().await,
+        backend.put_versions().await,
+        "cleanup must delete the exact object version returned by the tier PUT"
+    );
+    assert_eq!(
+        backend.object_count().await,
+        0,
+        "the rejected transition must not leave a remote candidate"
+    );
+    assert_eq!(
+        read_object_bytes(&ecstore, bucket.as_str(), object).await,
+        replacement_payload,
+        "the replacement object must remain readable and unmodified"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn cancelled_transition_waiting_for_prepared_reader_cleans_remote() {
+    temp_env::async_with_vars([(ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
+        let (_disk_paths, ecstore) = setup_test_env().await;
+        let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+        let backend = register_mock_tier(&tier_name).await;
+        let bucket = format!("test-api-transition-reader-lock-{}", &Uuid::new_v4().simple().to_string()[..8]);
+        let object = "test/transition-reader-lock.txt";
+        let payload = b"prepared reader must keep transition commit behind the namespace lock".repeat(1024);
+
+        (*ecstore)
+            .make_bucket(bucket.as_str(), &MakeBucketOptions::default())
+            .await
+            .expect("unversioned test bucket should be created");
+        let original = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
+        let transition_opts = ObjectOptions {
+            transition: lifecycle::lifecycle_contract::TransitionOptions {
+                status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(),
+                tier: tier_name,
+                etag: original.etag.clone().unwrap_or_default(),
+                ..Default::default()
+            },
+            version_id: original.version_id.map(|version| version.to_string()),
+            mod_time: original.mod_time,
+            ..Default::default()
+        };
+
+        let put_barrier = backend.arm_put_barrier().await;
+        let transition_store = Arc::clone(&ecstore);
+        let transition_bucket = bucket.clone();
+        let mut transition = tokio::spawn(async move {
+            transition_store
+                .transition_object(&transition_bucket, object, &transition_opts)
+                .await
+        });
+        tokio::select! {
+            result = &mut transition => panic!("transition ended before reaching the tier PUT barrier: {result:?}"),
+            () = put_barrier.wait_until_paused() => {}
+        }
+
+        let prepared_reader = ecstore
+            .prepare_get_object_reader(bucket.as_str(), object, None, HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("prepared reader should resolve while the local source still exists");
+        put_barrier.release();
+        assert!(
+            tokio::time::timeout(Duration::from_millis(200), &mut transition)
+                .await
+                .is_err(),
+            "transition commit must wait while a prepared reader owns the namespace read lock"
+        );
+
+        transition.abort();
+        assert!(
+            transition
+                .await
+                .expect_err("transition task should be cancelled")
+                .is_cancelled(),
+            "transition cancellation should not be reported as a task panic"
+        );
+        drop(prepared_reader);
+        assert_eq!(backend.put_count().await, 1, "the transition should upload exactly one remote candidate");
+        assert!(
+            backend.wait_for_object_count(0, TRANSITION_WAIT_TIMEOUT).await,
+            "cancelling while the commit waits for the namespace lock must clean the exact uploaded candidate"
+        );
+        assert_eq!(backend.remove_versions().await, backend.put_versions().await);
+        assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), object).await, payload);
+    })
+    .await;
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn lifecycle_transition_marks_dirty_disks_for_capacity_manager() {
+    let (disk_paths, ecstore) = setup_test_env().await;
+    let manager = create_isolated_manager(HybridStrategyConfig::default());
+    let _ = manager.get_dirty_disks().await;
+
+    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+    let _backend = register_mock_tier(&tier_name).await;
+
+    let bucket = format!("test-capacity-transition-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/object.txt";
+    let payload = b"transition should mark dirty scope";
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+    set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
+        .await
+        .expect("Failed to set lifecycle configuration");
+    let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
+
+    lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(ecstore.clone(), bucket.as_str())
+        .await
+        .expect("Failed to enqueue transitioned object");
+
+    let _ = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
+        .await
+        .expect("object should transition before dirty scope assertion");
+
+    let dirty_disks = manager.get_dirty_disks().await;
+    assert_eq!(dirty_disks.len(), disk_paths.len());
+
+    let actual_paths: std::collections::HashSet<_> = dirty_disks
+        .into_iter()
+        .map(|disk| stdfs::canonicalize(&disk.drive_path).unwrap().to_string_lossy().into_owned())
+        .collect();
+    let expected_paths: std::collections::HashSet<_> = disk_paths
+        .iter()
+        .map(|path| stdfs::canonicalize(path).unwrap().to_string_lossy().into_owned())
+        .collect();
+    assert_eq!(actual_paths, expected_paths);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
+async fn immediate_transition_timeout_eventually_completes_via_compensation() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+    let backend = register_mock_tier(&tier_name).await;
+
+    let bucket = format!("test-compensation-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/object.txt";
+    let payload = b"transition compensation should eventually complete";
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+    set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
+        .await
+        .expect("Failed to set lifecycle configuration");
+
+    with_forced_immediate_enqueue_timeout(&backend, || async {
+        let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
+    })
+    .await;
+
+    let info = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
+        .await
+        .expect("object should eventually transition after compensation backfill");
+
+    assert_eq!(backend.put_count().await, 1, "compensation should upload the version once");
+    assert_eq!(info.transitioned_object.status, "complete");
+    assert_eq!(info.transitioned_object.tier, tier_name);
+    assert!(backend.contains(&info.transitioned_object.name).await);
+}

rustfs/src/app/mod.rs

@@ -49,3 +49,5 @@ pub(crate) mod gating_test_env;
 mod lifecycle_transition_api_test;
 #[cfg(test)]
 mod put_prelookup_gating_test;
+#[cfg(test)]
+pub(crate) mod test_seams;

rustfs/src/app/object/delete.rs

@@ -2011,12 +2011,10 @@ mod tests {
                             .version_id(requested_version)
                             .build()
                             .expect("HEAD request");
-                        let get_error = usecase
-                            .execute_get_object(build_request(get, Method::GET))
+                        let get_error = crate::app::test_seams::get_object(&usecase, build_request(get, Method::GET))
                             .await
                             .expect_err("marker GET");
-                        let head_error = usecase
-                            .execute_head_object(build_request(head, Method::HEAD))
+                        let head_error = crate::app::test_seams::head_object(&usecase, build_request(head, Method::HEAD))
                             .await
                             .expect_err("marker HEAD");
                         for (method, error) in [("GET", get_error), ("HEAD", head_error)] {
@@ -2063,8 +2061,7 @@ mod tests {
                         .version_id(Some(original_id.clone()))
                         .build()
                         .expect("historical GET request");
-                    let response = usecase
-                        .execute_get_object(build_request(get, Method::GET))
+                    let response = crate::app::test_seams::get_object(&usecase, build_request(get, Method::GET))
                         .await
                         .expect("historical GET");
                     assert!(!response.headers.contains_key("x-amz-delete-marker"));
@@ -2080,8 +2077,7 @@ mod tests {
                         .version_id(Some(original_id.clone()))
                         .build()
                         .expect("historical HEAD request");
-                    let response = usecase
-                        .execute_head_object(build_request(head, Method::HEAD))
+                    let response = crate::app::test_seams::head_object(&usecase, build_request(head, Method::HEAD))
                         .await
                         .expect("historical HEAD");
                     assert_eq!(response.output.content_length, Some(i64::try_from(payload.len()).unwrap()));
@@ -2101,12 +2097,10 @@ mod tests {
                             .version_id(version)
                             .build()
                             .expect("missing HEAD request");
-                        let get_error = usecase
-                            .execute_get_object(build_request(get, Method::GET))
+                        let get_error = crate::app::test_seams::get_object(&usecase, build_request(get, Method::GET))
                             .await
                             .expect_err("missing GET");
-                        let head_error = usecase
-                            .execute_head_object(build_request(head, Method::HEAD))
+                        let head_error = crate::app::test_seams::head_object(&usecase, build_request(head, Method::HEAD))
                             .await
                             .expect_err("missing HEAD");
                         for error in [get_error, head_error] {
@@ -2201,7 +2195,7 @@ mod tests {
                         .version_id(Some(marker_id.clone()))
                         .build()
                         .unwrap();
-                    let get_err = Box::pin(usecase.execute_get_object(build_request(get, Method::GET)))
+                    let get_err = Box::pin(crate::app::test_seams::get_object(&usecase, build_request(get, Method::GET)))
                         .await
                         .expect_err("GET of a marker remains forbidden");
                     assert_eq!(get_err.code(), &S3ErrorCode::MethodNotAllowed);
@@ -2211,7 +2205,7 @@ mod tests {
                         .version_id(Some(marker_id.clone()))
                         .build()
                         .unwrap();
-                    let head_err = Box::pin(usecase.execute_head_object(build_request(head, Method::HEAD)))
+                    let head_err = Box::pin(crate::app::test_seams::head_object(&usecase, build_request(head, Method::HEAD)))
                         .await
                         .expect_err("HEAD of a marker remains forbidden");
                     assert_eq!(head_err.code(), &S3ErrorCode::MethodNotAllowed);
@@ -2287,7 +2281,7 @@ mod tests {
                         .version_id(Some(marker_id))
                         .build()
                         .unwrap();
-                    let missing = Box::pin(usecase.execute_get_object(build_request(removed, Method::GET)))
+                    let missing = Box::pin(crate::app::test_seams::get_object(&usecase, build_request(removed, Method::GET)))
                         .await
                         .expect_err("the marker version must actually be gone");
                     assert_eq!(
@@ -2300,7 +2294,7 @@ mod tests {
                         .key(key.to_string())
                         .build()
                         .unwrap();
-                    let response = Box::pin(usecase.execute_get_object(build_request(get, Method::GET)))
+                    let response = Box::pin(crate::app::test_seams::get_object(&usecase, build_request(get, Method::GET)))
                         .await
                         .expect("GET restores the historical payload");
                     let mut body = response.output.body.expect("GET body");

rustfs/src/app/object/put.rs

@@ -2674,9 +2674,7 @@ mod tests {
             .expect("metadata GET input must build");
         let metadata_usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
         let metadata_get = tokio::spawn(async move {
-            metadata_usecase
-                .execute_get_object(build_request(metadata_input, Method::GET))
-                .await
+            crate::app::test_seams::get_object(&metadata_usecase, build_request(metadata_input, Method::GET)).await
         });
         tokio::time::timeout(Duration::from_secs(2), metadata_entered.wait())
             .await
@@ -2705,7 +2703,10 @@ mod tests {
             .build()
             .expect("GET input must build");
         let get_usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
-        let get = tokio::spawn(async move { get_usecase.execute_get_object(build_request(get_input, Method::GET)).await });
+        let get =
+            tokio::spawn(
+                async move { crate::app::test_seams::get_object(&get_usecase, build_request(get_input, Method::GET)).await },
+            );
         tokio::time::timeout(Duration::from_secs(2), async {
             while !object_traffic_health.read_storage_stalled_for_test() {
                 tokio::task::yield_now().await;
@@ -4225,7 +4226,7 @@ mod tests {
                 .expect("SSE-C HEAD input must build");
             let mut head_request = build_request(head_input, Method::HEAD);
             head_request.headers = read_headers.clone();
-            let head = Box::pin(usecase.execute_head_object(head_request))
+            let head = Box::pin(crate::app::test_seams::head_object(&usecase, head_request))
                 .await
                 .expect("matching SSE-C key must authorize HEAD");
             assert_eq!(head.output.server_side_encryption, None);
@@ -4242,7 +4243,7 @@ mod tests {
                 .expect("SSE-C GET input must build");
             let mut get_request = build_request(get_input, Method::GET);
             get_request.headers = read_headers;
-            let get = Box::pin(usecase.execute_get_object(get_request))
+            let get = Box::pin(crate::app::test_seams::get_object(&usecase, get_request))
                 .await
                 .expect("matching SSE-C key must authorize GET");
             assert_eq!(get.output.server_side_encryption, None);

rustfs/src/app/storage_api.rs

@@ -59,6 +59,8 @@ pub(crate) mod s3 {
     #[cfg(test)]
     pub(crate) use s3s::dto::{GetObjectInput, HeadObjectInput, ListObjectsV2Input, ListObjectsV2Output};
     #[cfg(test)]
+    pub(crate) use s3s::dto::{GetObjectOutput, HeadObjectOutput, PutObjectInput, PutObjectOutput};
+    #[cfg(test)]
     pub(crate) use s3s::dto::{ListObjectsInput, StreamingBlob, UploadPartInput, UploadPartOutput};
     #[cfg(test)]
     pub(crate) use s3s::service::{S3Service, S3ServiceBuilder};

rustfs/src/app/test_seams/get_object.rs

@@ -0,0 +1,28 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::app::object_usecase::DefaultObjectUsecase;
+use crate::app::storage_api::s3::{GetObjectInput, GetObjectOutput, S3Request, S3Response, S3Result};
+use std::future::Future;
+
+/// Calls `DefaultObjectUsecase::execute_get_object` with today's arguments.
+///
+/// Owned by the GetObject batch (C). Tests outside that batch's files call this seam instead of the use-case method;
+/// when batch C changes the use-case signature it changes only this body.
+pub(crate) fn get_object(
+    usecase: &DefaultObjectUsecase,
+    req: S3Request<GetObjectInput>,
+) -> impl Future<Output = S3Result<S3Response<GetObjectOutput>>> + '_ {
+    usecase.execute_get_object(req)
+}

rustfs/src/app/test_seams/head_object.rs

@@ -0,0 +1,28 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::app::object_usecase::DefaultObjectUsecase;
+use crate::app::storage_api::s3::{HeadObjectInput, HeadObjectOutput, S3Request, S3Response, S3Result};
+use std::future::Future;
+
+/// Calls `DefaultObjectUsecase::execute_head_object` with today's arguments.
+///
+/// Owned by the HeadObject batch (C). Tests outside that batch's files call this seam instead of the use-case method;
+/// when batch C changes the use-case signature it changes only this body.
+pub(crate) fn head_object(
+    usecase: &DefaultObjectUsecase,
+    req: S3Request<HeadObjectInput>,
+) -> impl Future<Output = S3Result<S3Response<HeadObjectOutput>>> + '_ {
+    usecase.execute_head_object(req)
+}

rustfs/src/app/test_seams/list_objects_v2.rs

@@ -0,0 +1,28 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::app::bucket_usecase::DefaultBucketUsecase;
+use crate::app::storage_api::s3::{ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response, S3Result};
+use std::future::Future;
+
+/// Calls `DefaultBucketUsecase::execute_list_objects_v2` with today's arguments.
+///
+/// Owned by the ListObjectsV2 batch (B2). Tests outside that batch's files call this seam instead of the use-case
+/// method; when batch B2 changes the use-case signature it changes only this body.
+pub(crate) fn list_objects_v2(
+    usecase: &DefaultBucketUsecase,
+    req: S3Request<ListObjectsV2Input>,
+) -> impl Future<Output = S3Result<S3Response<ListObjectsV2Output>>> + '_ {
+    usecase.execute_list_objects_v2(req)
+}

rustfs/src/app/test_seams/mod.rs

@@ -0,0 +1,29 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Test-only seams over the use-case entry points that tests of other operation families call as an oracle.
+//!
+//! Each seam is owned by the batch that owns its operation. A test outside that family calls the seam, never the
+//! use-case method, so a signature change touches one seam body instead of another family's tests.
+//! `scripts/check_test_seams.py` enforces this.
+
+mod get_object;
+mod head_object;
+mod list_objects_v2;
+mod put_object;
+
+pub(crate) use get_object::get_object;
+pub(crate) use head_object::head_object;
+pub(crate) use list_objects_v2::list_objects_v2;
+pub(crate) use put_object::put_object;

rustfs/src/app/test_seams/put_object.rs

@@ -0,0 +1,30 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::app::object_usecase::DefaultObjectUsecase;
+use crate::app::storage_api::s3::{PutObjectInput, PutObjectOutput, S3Request, S3Response, S3Result};
+use crate::app::storage_api::test::ecfs::FS;
+use std::future::Future;
+
+/// Calls `DefaultObjectUsecase::execute_put_object` with today's arguments.
+///
+/// Owned by the PutObject batch (D). Tests outside that batch's files call this seam instead of the use-case method;
+/// when batch D changes the use-case signature it changes only this body.
+pub(crate) fn put_object<'a>(
+    usecase: &'a DefaultObjectUsecase,
+    fs: &'a FS,
+    req: S3Request<PutObjectInput>,
+) -> impl Future<Output = S3Result<S3Response<PutObjectOutput>>> + 'a {
+    usecase.execute_put_object(fs, req)
+}

rustfs/src/storage/access.rs

@@ -6900,23 +6900,27 @@ mod tests {
                 ));
                 let usecase = crate::app::object_usecase::DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
                 assert_eq!(
-                    usecase.execute_get_object(get).await.expect_err("bad GET part number").code(),
+                    crate::app::test_seams::get_object(&usecase, get)
+                        .await
+                        .expect_err("bad GET part number")
+                        .code(),
                     &S3ErrorCode::InvalidArgument
                 );
                 assert_eq!(
-                    usecase
-                        .execute_head_object(head)
+                    crate::app::test_seams::head_object(&usecase, head)
                         .await
                         .expect_err("range and part number conflict")
                         .code(),
                     &S3ErrorCode::InvalidArgument
                 );
                 assert_eq!(
-                    crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context)))
-                        .execute_list_objects_v2(list)
-                        .await
-                        .expect_err("negative max keys")
-                        .code(),
+                    crate::app::test_seams::list_objects_v2(
+                        &crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context))),
+                        list
+                    )
+                    .await
+                    .expect_err("negative max keys")
+                    .code(),
                     &S3ErrorCode::InvalidArgument
                 );
             }
@@ -7004,78 +7008,77 @@ mod tests {
             let incarnation = store.bucket_incarnation_id(&bucket).await.expect("created identity");
             sys.apply_for_incarnation(&bucket, incarnation, Some(&config)).await;
             let usecase = crate::app::object_usecase::DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
-            usecase
-                .execute_get_object(get.clone())
+            crate::app::test_seams::get_object(&usecase, get.clone())
                 .await
                 .expect("a local GET hit ignores source capture failure");
-            usecase
-                .execute_head_object(head.clone())
+            crate::app::test_seams::head_object(&usecase, head.clone())
                 .await
                 .expect("a local HEAD hit ignores source capture failure");
             get.input.key = "missing".into();
             head.input.key = "missing".into();
             assert_eq!(
-                usecase
-                    .execute_get_object(get.clone())
+                crate::app::test_seams::get_object(&usecase, get.clone())
                     .await
                     .expect_err("failed capture cannot rebind on GET miss")
                     .code(),
                 &S3ErrorCode::NoSuchBucket
             );
             assert_eq!(
-                usecase
-                    .execute_head_object(head.clone())
+                crate::app::test_seams::head_object(&usecase, head.clone())
                     .await
                     .expect_err("failed capture cannot rebind on HEAD miss")
                     .code(),
                 &S3ErrorCode::NoSuchBucket
             );
             assert_eq!(
-                crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context)))
-                    .execute_list_objects_v2(list.clone())
-                    .await
-                    .expect_err("failed capture cannot rebind on LIST")
-                    .code(),
+                crate::app::test_seams::list_objects_v2(
+                    &crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context))),
+                    list.clone()
+                )
+                .await
+                .expect_err("failed capture cannot rebind on LIST")
+                .code(),
                 &S3ErrorCode::NoSuchBucket
             );
             config.filter.prefix = Some("remote/".into());
             sys.apply_for_incarnation(&bucket, incarnation, Some(&config)).await;
             assert_eq!(
-                usecase
-                    .execute_get_object(get.clone())
+                crate::app::test_seams::get_object(&usecase, get.clone())
                     .await
                     .expect_err("filtered GET remains local")
                     .code(),
                 &S3ErrorCode::NoSuchKey
             );
             assert_eq!(
-                usecase
-                    .execute_head_object(head.clone())
+                crate::app::test_seams::head_object(&usecase, head.clone())
                     .await
                     .expect_err("filtered HEAD remains local")
                     .code(),
                 &S3ErrorCode::NoSuchKey
             );
             list.input.prefix = Some("local/".into());
-            let listing = crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context)))
-                .execute_list_objects_v2(list.clone())
-                .await
-                .expect("disjoint prefix needs no source identity");
+            let listing = crate::app::test_seams::list_objects_v2(
+                &crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context))),
+                list.clone(),
+            )
+            .await
+            .expect("disjoint prefix needs no source identity");
             assert_eq!(listing.output.key_count, Some(0));
             list.input.prefix = Some("remote/".into());
             list.input.max_keys = Some(0);
-            let listing = crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context)))
-                .execute_list_objects_v2(list)
-                .await
-                .expect("an empty page needs no source identity");
+            let listing = crate::app::test_seams::list_objects_v2(
+                &crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context))),
+                list,
+            )
+            .await
+            .expect("an empty page needs no source identity");
             assert_eq!(listing.output.key_count, Some(0));
 
             config.filter.prefix = None;
             config.policy.head = crate::on_demand_migration::HeadPolicy::LocalOnly;
             sys.apply_for_incarnation(&bucket, incarnation, Some(&config)).await;
             assert_eq!(
-                usecase
-                    .execute_head_object(head.clone())
+                crate::app::test_seams::head_object(&usecase, head.clone())
                     .await
                     .expect_err("local-only HEAD needs no source identity")
                     .code(),
@@ -7095,16 +7098,14 @@ mod tests {
                 .await
                 .expect("create local delete marker");
             assert_eq!(
-                usecase
-                    .execute_get_object(get)
+                crate::app::test_seams::get_object(&usecase, get)
                     .await
                     .expect_err("GET respects the delete marker before capture failure")
                     .code(),
                 &S3ErrorCode::NoSuchKey
             );
             assert_eq!(
-                usecase
-                    .execute_head_object(head)
+                crate::app::test_seams::head_object(&usecase, head)
                     .await
                     .expect_err("HEAD respects the delete marker before capture failure")
                     .code(),

scripts/README.md

@@ -57,6 +57,9 @@ their issue closes.
 | `test_object_generation_protocol_model.py` | dev-tool | Bounded single-slot, four-voter promise/accept design model and recovery/retirement examples; no production runtime evidence | `python3 scripts/test_object_generation_protocol_model.py`; [Generation contract](../docs/architecture/unified-object-generation.md#executable-design-model-and-acceptance-mapping) |
 | `junit_diff.py` | dev-tool | Case-by-case pass/fail/skip diff of two JUnit XML reports (reference vs candidate stack); exit 1 on any difference or stale `--allow` entry, 2 on unreadable input | `python3 scripts/junit_diff.py a.xml b.xml`; backlog T3.1 |
 | `test_junit_diff.py` | ci-gate | Contract tests and fixtures (`scripts/tests/junit_diff/`) for `junit_diff.py` | `make script-tests` |
+| `s3-tests/reference_parity.py` | ci-gate | Plans the reference parity targets, lists a run's failures for the single rerun, and compares every candidate's s3-tests JUnit against the pinned reference per node id with a tolerated-difference file | `.github/workflows/reference-parity.yml`; backlog T3.2 |
+| `s3-tests/test_reference_parity.py` | ci-gate | Contract tests for `reference_parity.py` | `make script-tests` |
+| `test_reference_binary.py` | ci-gate | Contract tests for `.github/actions/reference-binary/reference_binary.py` (pin resolution and verified unpacking) | `make script-tests` |
 | `e2e-run.sh` | ci-gate | Boots a rustfs server and runs the `s3s-e2e` black-box conformance tool against it | ci.yml `e2e-tests` jobs; `docs/testing/README.md` |
 | `run_ecstore_validation_suite.sh` | dev-tool | ecstore black-box validation suite (`quick`/`full`/`destructive`/`fuzz` profiles) | `docs/testing/README.md`, `docs/testing/ecstore-validation-suite-design.md` |
 | `run_e2e_tests.sh` | dev-tool | Local `e2e_test` crate runner (starts a server, applies filters, cleans up) | `crates/e2e_test/README.md` |
@@ -166,7 +169,7 @@ their issue closes.
 |---|---|---|---|
 | `codemods/` | dev-tool | Scripted, idempotent source migrations for the gateway migration (rustfs/backlog#2734); `s3_error_to_s3_types.py` moves s3s error-type paths onto `rustfs-s3-types` per crate closure and refuses the s3s edge allowlist | `make script-tests` (`--self-test`); usage in each script's docstring |
 | `fuzz/` | ci-gate | Unified cargo-fuzz runner and helpers for the `fuzz/` sub-workspace | fuzz.yml; `fuzz/README.md` |
-| `release/` | ci-gate | Release creation and DEB/RPM version-normalization helpers | build.yml; package.yml |
+| `release/` | ci-gate | Release creation and DEB/RPM version-normalization helpers | build.yml; package.yml; manual-main-packages.yml |
 | `s3-tests/` | ci-gate | ceph/s3-tests compatibility harness (allow-lists, patches, report tooling) | ci.yml; e2e-s3tests.yml; `scripts/s3-tests/README.md` |
 | `security/` | ci-gate | Workflow-pin enforcement and release supply-chain asset generation | audit.yml; build.yml |
 | `table-catalog/` | dev-tool | S3-Tables / pyiceberg validation suite | `docs/architecture/s3-tables-support-matrix.md` |

scripts/check_architecture_migration_rules.sh

@@ -2261,10 +2261,10 @@ fi
       rustfs/src/app/select_object.rs rustfs/src/app/*_usecase.rs rustfs/src/app/object || true
     rg -n --with-filename \
       'use super::(?:\{[^}]*\b(?:DynReader|HashReader|WriteEncryption|WritePlan|DecryptReader|EncryptReader|HardLimitReader|boxed_reader|wrap_reader|compression_metadata_value|is_disk_compressible|MIN_DISK_COMPRESSIBLE_SIZE|get_lock_acquire_timeout|is_valid_storage_class|StorageError|DiskError|is_all_buckets_not_found|is_err_bucket_not_found|is_err_object_not_found|is_err_version_not_found)\b|(?:object_api_utils::to_s3s_etag|storageclass|StorageError|DiskError|DynReader|HashReader|WriteEncryption|WritePlan|DecryptReader|EncryptReader|HardLimitReader|boxed_reader|wrap_reader|compression_metadata_value|is_disk_compressible|MIN_DISK_COMPRESSIBLE_SIZE|get_lock_acquire_timeout|is_valid_storage_class|is_all_buckets_not_found|is_err_bucket_not_found|is_err_object_not_found|is_err_version_not_found)\b)' \
-      rustfs/src/app/bucket_usecase.rs rustfs/src/app/object_usecase.rs rustfs/src/app/object rustfs/src/app/multipart_usecase.rs rustfs/src/app/lifecycle_transition_api_test.rs || true
+      rustfs/src/app/bucket_usecase.rs rustfs/src/app/object_usecase.rs rustfs/src/app/object rustfs/src/app/multipart_usecase.rs rustfs/src/app/lifecycle_transition_api_test.rs rustfs/src/app/lifecycle_transition_api_test || true
     rg -n --with-filename \
       'use super::(?:\{[^}]*\b(?:AppObjectLockConfigExt|AppReplicationConfigExt|AppVersioningConfigExt|predict_lifecycle_expiration|validate_restore_request|bucket_target_sys|lifecycle|metadata|metadata_sys|object_lock|policy_sys|quota|replication|tagging|target|utils|versioning_sys|transition_api|ObjectInfo|ObjectOptions)\b|(?:AppObjectLockConfigExt|AppReplicationConfigExt|AppVersioningConfigExt|predict_lifecycle_expiration|validate_restore_request|bucket_target_sys|lifecycle|metadata|metadata_sys|object_lock|policy_sys|quota|replication|tagging|target|utils|versioning_sys|transition_api|ObjectInfo|ObjectOptions)\b)|super::(?:lifecycle|metadata_sys|object_lock|quota|replication|tagging|target|utils|versioning_sys|transition_api)::|super::super::(?:metadata_sys|lifecycle|target)::' \
-      rustfs/src/app/bucket_usecase.rs rustfs/src/app/object_usecase.rs rustfs/src/app/object rustfs/src/app/multipart_usecase.rs rustfs/src/app/lifecycle_transition_api_test.rs rustfs/src/app/capacity_dirty_scope_test.rs rustfs/src/app/context.rs rustfs/src/app/context/handles.rs rustfs/src/app/context/interfaces.rs rustfs/src/app/context/runtime_sources.rs || true
+      rustfs/src/app/bucket_usecase.rs rustfs/src/app/object_usecase.rs rustfs/src/app/object rustfs/src/app/multipart_usecase.rs rustfs/src/app/lifecycle_transition_api_test.rs rustfs/src/app/lifecycle_transition_api_test rustfs/src/app/capacity_dirty_scope_test.rs rustfs/src/app/context.rs rustfs/src/app/context/handles.rs rustfs/src/app/context/interfaces.rs rustfs/src/app/context/runtime_sources.rs || true
   }
 ) >"$RUSTFS_APP_USECASE_STORAGE_API_BYPASS_HITS_FILE"
 

scripts/check_test_seams.py

@@ -0,0 +1,442 @@
+#!/usr/bin/env python3
+"""Guard: tests outside an operation's owning family call its use-case only through the test seam.
+
+Preparatory refactor P4 of the s3s-to-gateway migration. Each migration batch
+changes the use-case signatures of its own operations. Tests owned by another
+family that call those entry points as an oracle would break and pull a second
+batch into the same file, so they call the `#[cfg(test)]` seams in
+rustfs/src/app/test_seams/ instead; the owning batch then edits one seam body.
+
+For every entry point in SEAMS this guard fails when
+
+  * a direct method call `.execute_<op>(` appears in test code (a module or
+    item under `#[cfg(test)]`, a file whose module declaration or an ancestor's
+    is under `#[cfg(test)]`, or any file under rustfs/tests/ or rustfs/benches/)
+    in a file that is neither the seam nor one of the operation's owner files;
+  * the seam file is missing, or no longer calls the entry point exactly once;
+  * an owner file listed below does not exist (a stale table must not widen
+    what passes);
+  * a file holding a direct call cannot be placed in the module tree;
+  * no Rust file is found at all.
+
+Production calls (the legacy S3 trait, admin and metadata routes) are not test
+code and are not judged here. Comments and string literals are ignored.
+
+Usage:
+  scripts/check_test_seams.py
+  scripts/check_test_seams.py --self-test
+
+Exit status: 0 clean, 1 violations or missing inputs, 2 usage error.
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+import sys
+import tempfile
+import unittest
+from dataclasses import dataclass
+from pathlib import Path
+
+SCAN_ROOT = "rustfs"
+
+
+@dataclass(frozen=True)
+class Seam:
+    batch: str
+    seam_file: str
+    owners: tuple[str, ...]  # files that own the operation; each must exist
+    owner_dirs: tuple[str, ...] = ()  # directories a split of an owner file may create
+
+
+SEAMS = {
+    "execute_get_object": Seam(
+        batch="C",
+        seam_file="rustfs/src/app/test_seams/get_object.rs",
+        owners=(
+            "rustfs/src/app/object/get.rs",
+            "rustfs/src/app/object/head.rs",
+            "rustfs/src/app/object/conditional_read_bench.rs",
+            "rustfs/src/app/object/on_demand_migration_put.rs",
+        ),
+    ),
+    "execute_head_object": Seam(
+        batch="C",
+        seam_file="rustfs/src/app/test_seams/head_object.rs",
+        owners=(
+            "rustfs/src/app/object/get.rs",
+            "rustfs/src/app/object/head.rs",
+        ),
+    ),
+    "execute_list_objects_v2": Seam(
+        batch="B2",
+        seam_file="rustfs/src/app/test_seams/list_objects_v2.rs",
+        owners=("rustfs/src/app/bucket_usecase.rs",),
+        # P2 splits bucket_usecase.rs into bucket_usecase/*.rs; narrow this to the listing file once it exists.
+        owner_dirs=("rustfs/src/app/bucket_usecase/",),
+    ),
+    "execute_put_object": Seam(
+        batch="D",
+        seam_file="rustfs/src/app/test_seams/put_object.rs",
+        owners=(
+            "rustfs/src/app/object/put.rs",
+            "rustfs/src/app/lifecycle_transition_api_test/put.rs",
+        ),
+    ),
+}
+
+CFG_TEST = re.compile(r"#!?\[\s*cfg\s*\(\s*(?:all\s*\(\s*)?test\b")
+TEST_ATTR = re.compile(r"#\[\s*(?:tokio::)?test\b")
+CALL = re.compile(r"\.\s*(execute_get_object|execute_head_object|execute_list_objects_v2|execute_put_object)\s*\(")
+CHAR_LITERAL = re.compile(r"'(?:\\(?:[nrt\\0'\"]|x[0-9a-fA-F]{2}|u\{[0-9a-fA-F_]{1,6}\})|[^\\'\n])'")
+RAW_STRING = re.compile(r'b?r(#*)"')
+
+
+class InputError(Exception):
+    """A required input is missing or cannot be interpreted."""
+
+
+def code_mask(text: str) -> bytearray:
+    """1 for every character that is Rust code, 0 inside comments and literals."""
+    mask = bytearray(len(text))
+    i, n = 0, len(text)
+    while i < n:
+        c = text[i]
+        if text.startswith("//", i):
+            j = text.find("\n", i)
+            i = n if j < 0 else j
+            continue
+        if text.startswith("/*", i):
+            depth, j = 1, i + 2
+            while j < n and depth:
+                if text.startswith("/*", j):
+                    depth, j = depth + 1, j + 2
+                elif text.startswith("*/", j):
+                    depth, j = depth - 1, j + 2
+                else:
+                    j += 1
+            i = j
+            continue
+        raw = RAW_STRING.match(text, i)
+        if raw and (i == 0 or not (text[i - 1].isalnum() or text[i - 1] == "_")):
+            end = text.find('"' + raw.group(1), raw.end())
+            i = n if end < 0 else end + 1 + len(raw.group(1))
+            continue
+        if c == '"' or (text.startswith('b"', i) and (i == 0 or not text[i - 1].isalnum())):
+            j = i + (2 if c == "b" else 1)
+            while j < n and text[j] != '"':
+                j += 2 if text[j] == "\\" else 1
+            i = j + 1
+            continue
+        if c == "'":
+            char = CHAR_LITERAL.match(text, i)
+            if char:
+                i = char.end()
+                continue
+        mask[i] = 1
+        i += 1
+    return mask
+
+
+def test_regions(text: str, mask: bytearray) -> list[tuple[int, int]]:
+    """Spans of items under `#[cfg(test)]` or `#[test]`/`#[tokio::test]` (the whole file for `#![cfg(test)]`)."""
+    regions = []
+    for match in [*CFG_TEST.finditer(text), *TEST_ATTR.finditer(text)]:
+        if not mask[match.start()]:
+            continue
+        if text[match.start() + 1] == "!":
+            return [(0, len(text))]
+        regions.append((match.start(), _item_end(text, mask, match.start())))
+    return regions
+
+
+def _item_end(text: str, mask: bytearray, attr_start: int) -> int:
+    """End offset of the item an attribute at attr_start applies to."""
+    depth, i, n = 0, attr_start, len(text)
+    in_attr = True
+    while i < n:
+        if mask[i]:
+            ch = text[i]
+            if ch in "([{":
+                if ch == "{" and depth == 0 and not in_attr:
+                    return _matching(text, mask, i) + 1
+                depth += 1
+            elif ch in ")]}":
+                depth -= 1
+                if depth == 0 and ch == "]":
+                    in_attr = False
+            elif ch == ";" and depth == 0 and not in_attr:
+                return i + 1
+            elif ch == "#" and depth == 0:
+                in_attr = True
+        i += 1
+    raise InputError(f"unterminated item after attribute at offset {attr_start}")
+
+
+def _matching(text: str, mask: bytearray, open_pos: int) -> int:
+    depth = 0
+    for i in range(open_pos, len(text)):
+        if mask[i]:
+            depth += text[i] in "([{"
+            depth -= text[i] in ")]}"
+            if depth == 0:
+                return i
+    raise InputError(f"unbalanced brace at offset {open_pos}")
+
+
+def module_declaration(root: Path, path: str) -> tuple[list[str], str] | None:
+    """(candidate parent files, module name) for a file below src/, or None for a crate root."""
+    parts = Path(path)
+    if parts.name in ("lib.rs", "main.rs") and parts.parent.name == "src":
+        return None
+    if parts.name == "mod.rs":
+        name, directory = parts.parent.name, parts.parent.parent
+    else:
+        name, directory = parts.stem, parts.parent
+    if directory.name == "bin" and parts.name != "mod.rs":
+        return None
+    candidates = [directory.with_suffix(".rs"), directory / "mod.rs"]
+    if directory.name == "src":
+        candidates = [directory / "lib.rs", directory / "main.rs"]
+    return [str(parent) for parent in candidates if (root / parent).is_file()], name
+
+
+def is_test_only_file(root: Path, path: str, cache: dict[str, bool]) -> bool:
+    if path in cache:
+        return cache[path]
+    parts = Path(path).parts
+    if len(parts) > 1 and parts[1] in ("tests", "benches"):
+        cache[path] = True
+        return True
+    text = (root / path).read_text()
+    mask = code_mask(text)
+    if any(text[m.start() + 1] == "!" for m in CFG_TEST.finditer(text) if mask[m.start()]):
+        cache[path] = True
+        return True
+    declaration = module_declaration(root, path)
+    if declaration is None:
+        if Path(path).name in ("lib.rs", "main.rs") or Path(path).parent.name == "bin":
+            cache[path] = False
+            return False
+        raise InputError(f"{path}: cannot find the file that declares this module")
+    parents, name = declaration
+    declarations = []
+    for parent in parents:
+        parent_text = (root / parent).read_text()
+        parent_mask = code_mask(parent_text)
+        pattern = rf"(?m)^[ \t]*(?:pub(?:\([^)\n]*\))?[ \t]+)?mod[ \t]+{re.escape(name)}[ \t]*;"
+        for m in re.finditer(pattern, parent_text):
+            if parent_mask[m.start() + len(m.group()) - len(m.group().lstrip())]:
+                declarations.append((parent, parent_text, parent_mask, m.start()))
+    if len(declarations) != 1:
+        raise InputError(f"{path}: expected one `mod {name};` in {' or '.join(parents) or 'a parent file'}, found {len(declarations)}")
+    parent, parent_text, parent_mask, offset = declarations[0]
+    cfg_test = any(start <= offset < end for start, end in test_regions(parent_text, parent_mask))
+    result = cfg_test or is_test_only_file(root, parent, cache)
+    cache[path] = result
+    return result
+
+
+def rust_files(root: Path) -> list[str]:
+    listed = subprocess.run(
+        ["git", "ls-files", "--cached", "--others", "--exclude-standard", "--", f"{SCAN_ROOT}/*.rs"],
+        cwd=root,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if listed.returncode != 0:
+        raise InputError(f"git ls-files failed: {listed.stderr.strip()}")
+    files = sorted(line for line in listed.stdout.splitlines() if line and (root / line).is_file())
+    if not files:
+        raise InputError(f"no Rust files found under {SCAN_ROOT}/")
+    return files
+
+
+def check(root: Path, files: list[str] | None = None) -> list[str]:
+    errors: list[str] = []
+    for method, seam in SEAMS.items():
+        seam_path = root / seam.seam_file
+        if not seam_path.is_file():
+            errors.append(f"{seam.seam_file}: seam for {method} is missing")
+        else:
+            text = seam_path.read_text()
+            mask = code_mask(text)
+            calls = [m for m in CALL.finditer(text) if mask[m.start()] and m.group(1) == method]
+            if len(calls) != 1:
+                errors.append(f"{seam.seam_file}: must call {method} exactly once, found {len(calls)}")
+        for owner in seam.owners:
+            if not (root / owner).is_file():
+                errors.append(f"{owner}: owner file of {method} is missing; update SEAMS in {Path(__file__).name}")
+    files = rust_files(root) if files is None else files
+    cache: dict[str, bool] = {}
+    for path in files:
+        text = (root / path).read_text()
+        mask = code_mask(text)
+        calls = [m for m in CALL.finditer(text) if mask[m.start()]]
+        if not calls:
+            continue
+        whole_file = is_test_only_file(root, path, cache)
+        regions = test_regions(text, mask)
+        for match in calls:
+            method = match.group(1)
+            seam = SEAMS[method]
+            if path == seam.seam_file or path in seam.owners or path.startswith(seam.owner_dirs or ("\0",)):
+                continue
+            if whole_file or any(start <= match.start() < end for start, end in regions):
+                line = text.count("\n", 0, match.start()) + 1
+                errors.append(
+                    f"{path}:{line}: test calls {method} directly; batch {seam.batch} owns it, "
+                    f"call crate::app::test_seams::{method.removeprefix('execute_')} instead"
+                )
+    return errors
+
+
+def main(argv: list[str]) -> int:
+    if argv == ["--self-test"]:
+        suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTest)
+        return 0 if unittest.TextTestRunner(verbosity=1).run(suite).wasSuccessful() else 1
+    if argv:
+        print(__doc__.split("Usage:", 1)[1].split("\n\n", 1)[0], file=sys.stderr)
+        return 2
+    root = Path(__file__).resolve().parent.parent
+    try:
+        errors = check(root)
+    except InputError as error:
+        print(f"test seam guard: {error}", file=sys.stderr)
+        return 1
+    for error in errors:
+        print(error, file=sys.stderr)
+    if errors:
+        print(f"test seam guard failed: {len(errors)} finding(s)", file=sys.stderr)
+        return 1
+    print("test seam guard passed")
+    return 0
+
+
+# ---------------------------------------------------------------------------
+# Self-test
+
+
+SEAM_BODY = {
+    "execute_get_object": "pub(crate) fn get_object(u: &U, r: R) -> F { u.execute_get_object(r) }\n",
+    "execute_head_object": "pub(crate) fn head_object(u: &U, r: R) -> F { u.execute_head_object(r) }\n",
+    "execute_list_objects_v2": "pub(crate) fn list_objects_v2(u: &U, r: R) -> F { u.execute_list_objects_v2(r) }\n",
+    "execute_put_object": "pub(crate) fn put_object(u: &U, f: &Fs, r: R) -> F { u.execute_put_object(f, r) }\n",
+}
+
+
+class SelfTest(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.root = Path(self.tmp.name)
+        self.files: dict[str, str] = {}
+        self.write("rustfs/src/main.rs", "mod app;\n")
+        self.write(
+            "rustfs/src/app/mod.rs",
+            "mod object;\nmod bucket_usecase;\nmod lifecycle_transition_api_test;\nmod storage;\n"
+            "#[cfg(test)]\nmod test_seams;\n#[cfg(test)]\nmod gating_test;\n",
+        )
+        self.write("rustfs/src/app/object/mod.rs", "mod get;\nmod head;\nmod put;\n#[cfg(test)]\nmod conditional_read_bench;\nmod on_demand_migration_put;\n")
+        self.write("rustfs/src/app/test_seams/mod.rs", "mod get_object;\nmod head_object;\nmod list_objects_v2;\nmod put_object;\n")
+        for method, seam in SEAMS.items():
+            self.write(seam.seam_file, SEAM_BODY[method])
+            for owner in seam.owners:
+                if owner not in self.files:
+                    self.write(owner, "fn owner() {}\n")
+        self.write("rustfs/src/app/lifecycle_transition_api_test/mod.rs", "mod put;\nmod delete;\n")
+        self.write("rustfs/src/app/storage.rs", "fn prod(u: &U) { u.execute_get_object(r); }\n")
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def write(self, path: str, text: str):
+        target = self.root / path
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(text)
+        self.files[path] = text
+
+    def run_check(self) -> list[str]:
+        return check(self.root, sorted(p for p in self.files if p.endswith(".rs")))
+
+    # -- negative cases: each must fail --
+
+    def test_direct_call_in_cfg_test_module_of_non_owner_fails(self):
+        self.write("rustfs/src/app/storage.rs", "fn prod() {}\n#[cfg(test)]\nmod tests {\n    fn t() { u.execute_get_object(r); }\n}\n")
+        self.assertEqual(len(self.run_check()), 1)
+
+    def test_direct_call_split_across_lines_fails(self):
+        self.write("rustfs/src/app/storage.rs", "#[cfg(test)]\nmod tests {\n    fn t() { usecase\n        .execute_head_object(r); }\n}\n")
+        self.assertRegex(self.run_check()[0], r"storage\.rs:4: test calls execute_head_object directly")
+
+    def test_direct_call_in_cfg_test_helper_fn_fails(self):
+        self.write("rustfs/src/app/storage.rs", "#[cfg(test)]\n#[allow(dead_code)]\nfn helper() { u.execute_list_objects_v2(r); }\n")
+        self.assertEqual(len(self.run_check()), 1)
+
+    def test_file_declared_under_cfg_test_fails(self):
+        self.write("rustfs/src/app/gating_test.rs", "fn t() { u.execute_put_object(&fs, r); }\n")
+        self.assertEqual(len(self.run_check()), 1)
+
+    def test_file_whose_ancestor_is_cfg_test_fails(self):
+        self.write("rustfs/src/app/test_seams/extra.rs", "fn t() { u.execute_get_object(r); }\n")
+        self.write("rustfs/src/app/test_seams/mod.rs", self.files["rustfs/src/app/test_seams/mod.rs"] + "mod extra;\n")
+        self.assertEqual(len(self.run_check()), 1)
+
+    def test_put_from_non_owner_lifecycle_file_fails(self):
+        self.write("rustfs/src/app/lifecycle_transition_api_test/delete.rs", "#[test]\nfn t() { u.execute_put_object(&fs, r); }\n")
+        self.write("rustfs/src/app/mod.rs", self.files["rustfs/src/app/mod.rs"].replace("mod lifecycle", "#[cfg(test)]\nmod lifecycle"))
+        self.assertEqual(len(self.run_check()), 1)
+
+    def test_direct_call_in_test_fn_without_cfg_test_fails(self):
+        self.write("rustfs/src/app/storage.rs", "#[tokio::test]\nasync fn t() { u.execute_get_object(r).await; }\n")
+        self.assertEqual(len(self.run_check()), 1)
+
+    def test_integration_test_file_fails(self):
+        self.write("rustfs/tests/oracle.rs", "fn t() { u.execute_get_object(r); }\n")
+        self.assertEqual(len(self.run_check()), 1)
+
+    def test_missing_seam_fails(self):
+        (self.root / SEAMS["execute_get_object"].seam_file).unlink()
+        del self.files[SEAMS["execute_get_object"].seam_file]
+        self.assertRegex(" ".join(self.run_check()), "seam for execute_get_object is missing")
+
+    def test_seam_that_stopped_wrapping_fails(self):
+        self.write(SEAMS["execute_put_object"].seam_file, "pub(crate) fn put_object() { todo!() }\n")
+        self.assertRegex(" ".join(self.run_check()), "must call execute_put_object exactly once, found 0")
+
+    def test_missing_owner_file_fails(self):
+        owner = SEAMS["execute_put_object"].owners[0]
+        (self.root / owner).unlink()
+        del self.files[owner]
+        self.assertRegex(" ".join(self.run_check()), "owner file of execute_put_object is missing")
+
+    def test_undeclared_file_with_direct_call_fails_loudly(self):
+        self.write("rustfs/src/app/orphan.rs", "fn t() { u.execute_get_object(r); }\n")
+        with self.assertRaisesRegex(InputError, "expected one `mod orphan;`"):
+            self.run_check()
+
+    def test_no_rust_files_fails(self):
+        empty = Path(self.tmp.name) / "empty"
+        empty.mkdir()
+        subprocess.run(["git", "init", "-q"], cwd=empty, check=True)
+        with self.assertRaisesRegex(InputError, "no Rust files found"):
+            rust_files(empty)
+
+    # -- positive cases: each must pass --
+
+    def test_seam_call_in_non_owner_test_passes(self):
+        self.write("rustfs/src/app/storage.rs", "#[cfg(test)]\nmod tests {\n    fn t() { crate::app::test_seams::get_object(&u, r); }\n}\n")
+        self.assertEqual(self.run_check(), [])
+
+    def test_owner_test_and_production_calls_pass(self):
+        self.write("rustfs/src/app/object/put.rs", "fn prod() {}\n#[cfg(test)]\nmod tests {\n    fn t() { u.execute_put_object(&fs, r); }\n}\n")
+        self.assertEqual(self.run_check(), [])
+
+    def test_calls_in_comments_and_strings_are_ignored(self):
+        self.write("rustfs/src/app/storage.rs", '#[cfg(test)]\nmod tests {\n    // u.execute_get_object(r)\n    const S: &str = "u.execute_get_object(r)";\n}\n')
+        self.assertEqual(self.run_check(), [])
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

scripts/codemods/split_lifecycle_transition_api_test.py

@@ -0,0 +1,624 @@
+#!/usr/bin/env python3
+"""Split rustfs/src/app/lifecycle_transition_api_test.rs into one file per operation owner.
+
+Preparatory refactor P4 for the s3s-to-gateway migration. Nine batches each
+migrate one family of S3 operations and change that family's use-case
+signatures. While every lifecycle integration test lives in one 3,700-line
+file, two batches edit the same file. This codemod moves every top-level item
+of that file, verbatim with its attributes and comments, into one of the
+module files below, chosen by the explicit ASSIGNMENT table:
+
+  lifecycle_transition_api_test.rs      module root: license header, the
+                                        original `use` block, module list
+  lifecycle_transition_api_test/
+    common.rs        shared fixtures (no owner; frozen during batches)
+    transition.rs    ILM transition-engine tests that call no S3 use-case
+    lifecycle_config.rs  PutBucketLifecycleConfiguration (batch A3)
+    object_lock.rs   PutObjectRetention / PutObjectLegalHold (batch B1)
+    put.rs           PutObject (batch D)
+    copy.rs          CopyObject, including tests that mix put and copy (batch E)
+    delete.rs        DeleteObject / DeleteObjects (batch E)
+    restore.rs       RestoreObject (batch E)
+    multipart.rs     CompleteMultipartUpload (batch E)
+
+Items are moved verbatim. The only rewrites, applied to moved items and
+nowhere else, are the ones a move out of the parent module requires:
+
+  * `super::` becomes `super::super::` (the items now live one module deeper);
+  * `s3s::S3ErrorCode` becomes `S3ErrorCode`, imported once in the module root
+    from `crate::app::storage_api::s3`, and a function-local `use s3s::S3;`
+    becomes `use crate::app::storage_api::s3::S3;`, so no new file names s3s
+    (scripts/check_s3s_footprint.sh counts files);
+  * items placed in common.rs that a sibling file uses gain `pub(super)`;
+  * a call to a test seam's use-case entry point (`execute_get_object`,
+    `execute_head_object`, `execute_list_objects_v2`, `execute_put_object`) in
+    a file that does not own that operation becomes a call to the seam in
+    `crate::app::test_seams` with the same arguments (scripts/check_test_seams.py).
+
+Every non-blank line of the input lands in exactly one output file. The run
+fails (exit 1) and writes nothing when an item is missing from ASSIGNMENT, an
+ASSIGNMENT entry names no item, an item would be moved twice, a moved file
+still names s3s, or a cross-owner seam call cannot be rewritten.
+
+Usage:
+  scripts/codemods/split_lifecycle_transition_api_test.py [--no-fmt]
+  scripts/codemods/split_lifecycle_transition_api_test.py --self-test
+
+The input must be the unsplit file. A second run over the split tree finds the
+module root instead and exits 1 without writing; to regenerate, restore the
+unsplit file (for example `git show <rev>:rustfs/src/app/lifecycle_transition_api_test.rs`),
+re-run, and the output is byte-identical to the committed split. Unless
+--no-fmt is given, `rustfmt --edition 2024` formats the written files.
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+import sys
+import tempfile
+import unittest
+from dataclasses import dataclass
+from pathlib import Path
+
+SOURCE = Path("rustfs/src/app/lifecycle_transition_api_test.rs")
+TARGET_DIR = Path("rustfs/src/app/lifecycle_transition_api_test")
+
+# Module files in the order the root declares them, with a one-line owner note.
+MODULES = {
+    "common": "Shared fixtures for the lifecycle integration tests; owned by no operation batch.",
+    "copy": "CopyObject tests, including tests that also PUT (batch E owns CopyObject).",
+    "delete": "DeleteObject and DeleteObjects tests (batch E owns both).",
+    "lifecycle_config": "PutBucketLifecycleConfiguration tests (batch A3 owns the lifecycle operations).",
+    "multipart": "CompleteMultipartUpload tests (batch E owns CompleteMultipartUpload).",
+    "object_lock": "PutObjectRetention and PutObjectLegalHold tests (batch B1 owns both).",
+    "put": "PutObject tests (batch D owns PutObject).",
+    "restore": "RestoreObject tests (batch E owns RestoreObject).",
+    "transition": "ILM transition-engine tests that call no S3 use-case; owned by no operation batch.",
+}
+
+# Every top-level item that is not a `use` declaration, by name.
+ASSIGNMENT = {
+    # Fixtures. A fixture used by one owner's tests moves with them; common.rs keeps the ones several owners share.
+    "GLOBAL_ENV": "common",
+    "TRANSITION_WAIT_TIMEOUT": "common",
+    "RESTORE_SUSPENDED_WAIT_TIMEOUT": "restore",
+    "ENV_GET_CODEC_STREAMING_ENABLE": "transition",
+    "ENV_GET_CODEC_STREAMING_ROLLOUT": "transition",
+    "ENV_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED": "transition",
+    "ENV_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED": "transition",
+    "ENV_GET_CODEC_STREAMING_MIN_SIZE": "transition",
+    "RUSTFS_EXPECTED_CURRENT_VERSION_ID": "common",
+    "init_tracing": "common",
+    "setup_test_env": "common",
+    "create_test_bucket": "common",
+    "suspend_test_bucket": "restore",
+    "upload_test_object": "common",
+    "transition_uploaded_object_directly": "restore",
+    "set_bucket_lifecycle_transition_with_tier": "common",
+    "set_bucket_object_lock_enabled": "common",
+    "expiration_lifecycle_configuration": "lifecycle_config",
+    "expired_object_delete_marker_lifecycle_configuration": "lifecycle_config",
+    "del_marker_expiration_lifecycle_configuration": "lifecycle_config",
+    "register_mock_tier": "common",
+    "wait_for_transition": "common",
+    "wait_for_restore_completion": "restore",
+    "with_forced_immediate_enqueue_timeout": "common",
+    "with_get_codec_streaming_remote_probe_env": "transition",
+    "wait_for_object_absence": "delete",
+    "wait_for_delete_marker": "common",
+    "build_request": "common",
+    "streaming_blob_from_bytes": "common",
+    "read_object_bytes": "common",
+    "live_object_version_count": "common",
+    "install_malformed_versioning_config": "delete",
+    "install_malformed_replication_config": "delete",
+    "install_delete_replication_config": "common",
+    # ILM transition engine; no S3 use-case entry point.
+    "transition_wait_timeout_reports_metadata_state_and_read_failure": "transition",
+    "get_transitioned_object_uses_remote_codec_fallback_path": "transition",
+    "duplicate_transition_task_skips_already_transitioned_version": "transition",
+    "transition_rejects_stale_etag_before_remote_upload": "transition",
+    "transition_commit_rejects_replaced_source": "transition",
+    "cancelled_transition_waiting_for_prepared_reader_cleans_remote": "transition",
+    "lifecycle_transition_marks_dirty_disks_for_capacity_manager": "transition",
+    "immediate_transition_timeout_eventually_completes_via_compensation": "transition",
+    # PutBucketLifecycleConfiguration (A3).
+    "compensation_driven_delete_marker_still_honors_lifecycle_cleanup": "lifecycle_config",
+    "put_bucket_lifecycle_configuration_expires_existing_objects": "lifecycle_config",
+    "put_bucket_lifecycle_configuration_allows_expired_delete_marker_on_object_lock_bucket": "lifecycle_config",
+    "put_bucket_lifecycle_configuration_rejects_del_marker_expiration_on_object_lock_bucket": "lifecycle_config",
+    "put_bucket_lifecycle_configuration_rejects_zero_day_del_marker_expiration_on_object_lock_bucket": "lifecycle_config",
+    "put_bucket_lifecycle_configuration_rejects_zero_day_expiration": "lifecycle_config",
+    # PutObjectRetention / PutObjectLegalHold (B1).
+    "object_lock_handlers_schedule_replication": "object_lock",
+    # PutObject (D).
+    "put_object_if_none_match_existing_object_returns_precondition_failed": "put",
+    "put_object_computes_replication_decision_exactly_once": "put",
+    "put_object_extract_computes_replication_decision_per_entry": "put",
+    # CopyObject (E); mixed put+copy tests go here.
+    "copy_object_if_none_match_existing_destination_returns_precondition_failed": "copy",
+    "undo_copy_requires_the_observed_current_version": "copy",
+    "undo_copy_serializes_a_concurrent_put_until_after_commit": "copy",
+    "copy_object_allows_new_version_for_locked_destination_but_blocks_explicit_overwrite": "copy",
+    "concurrent_reverse_copy_object_does_not_deadlock_with_reader_locks": "copy",
+    "put_and_copy_object_transition_immediately_via_usecases": "copy",
+    "compensation_driven_copy_still_completes_transition": "copy",
+    "copy_object_computes_replication_decision_and_strips_stale_status": "copy",
+    # CompleteMultipartUpload (E).
+    "complete_multipart_upload_transitions_immediately_via_usecase": "multipart",
+    "compensation_driven_complete_multipart_upload_still_transitions": "multipart",
+    "compensation_driven_complete_multipart_upload_still_transitions_inner": "multipart",
+    # DeleteObject / DeleteObjects (E).
+    "undo_delete_only_removes_the_current_matching_delete_marker": "delete",
+    "delete_transitioned_object_removes_remote_tier_copy_via_usecase": "delete",
+    "compensation_driven_transition_still_cleans_remote_tier_on_delete": "delete",
+    "compensation_driven_versioned_delete_still_creates_delete_marker": "delete",
+    "delete_objects_resolves_bucket_versioning_once_per_request": "delete",
+    "delete_objects_treats_empty_directory_version_as_absent": "delete",
+    "delete_objects_preserves_explicit_null_directory_version": "delete",
+    "delete_object_versioning_config_failure_leaves_latest_object_intact": "delete",
+    "delete_object_replication_config_failure_leaves_latest_object_intact": "delete",
+    "delete_object_uses_one_replication_config_generation": "delete",
+    "replica_delete_uses_authorized_source_version_and_its_tags": "delete",
+    "force_delete_replication_config_failure_leaves_prefix_intact": "delete",
+    "replica_delete_replication_config_failure_leaves_object_intact": "delete",
+    "delete_objects_replication_config_failure_does_not_call_storage_delete": "delete",
+    "nonexistent_bucket_fails_before_versioning_lookup_on_get_head_delete": "delete",
+    # RestoreObject (E).
+    "restore_object_usecase_reports_ongoing_conflict": "restore",
+    "restore_object_usecase_recovers_cancelled_post_commit_generation": "restore",
+    "restore_object_usecase_recovers_cancelled_post_commit_generation_inner": "restore",
+    "restore_object_usecase_completes_suspended_null_version_in_place": "restore",
+    "restore_object_usecase_completes_suspended_null_version_in_place_inner": "restore",
+    "restore_object_usecase_accepts_exactly_one_of_two_concurrent_restores": "restore",
+}
+
+# Seam entry points: method name -> (seam function, module that owns the operation here).
+# A call in any other module is rewritten to `test_seams::<seam>(&receiver, args...)`.
+SEAMS = {
+    "execute_get_object": ("get_object", None),
+    "execute_head_object": ("head_object", None),
+    "execute_list_objects_v2": ("list_objects_v2", None),
+    "execute_put_object": ("put_object", "put"),
+}
+
+ROOT_EXTRA_USE = "use crate::app::storage_api::s3::S3ErrorCode;\n"
+S3S_PATH = re.compile(r'(^|[^"A-Za-z0-9_])s3s::')
+SUPER_PATH = re.compile(r"(?<![A-Za-z0-9_:])super::")
+ITEM_NAME = re.compile(
+    r"^(?:pub(?:\([^)]*\))?\s+)?(?:(?:async|unsafe|const|extern)\s+)*"
+    r"(?P<kind>fn|static|const|struct|enum|type|trait|mod|impl|macro_rules!|use)\s*(?P<name>[A-Za-z_][A-Za-z0-9_]*)?"
+)
+VISIBILITY_TARGET = re.compile(r"^(?P<indent>[ \t]*)(?P<head>(?:async\s+|unsafe\s+)*(?:fn|static|const)\s)", re.M)
+
+
+class SplitError(Exception):
+    """The input cannot be split exactly as the tables describe: exit 1."""
+
+
+# ---------------------------------------------------------------------------
+# Lexing: positions of code characters, skipping comments and literals.
+
+
+def code_mask(text: str) -> bytearray:
+    """1 for every character that is Rust code, 0 inside comments and literals."""
+    mask = bytearray(len(text))
+    i, n = 0, len(text)
+    while i < n:
+        c = text[i]
+        if text.startswith("//", i):
+            j = text.find("\n", i)
+            i = n if j < 0 else j
+            continue
+        if text.startswith("/*", i):
+            depth, j = 1, i + 2
+            while j < n and depth:
+                if text.startswith("/*", j):
+                    depth, j = depth + 1, j + 2
+                elif text.startswith("*/", j):
+                    depth, j = depth - 1, j + 2
+                else:
+                    j += 1
+            i = j
+            continue
+        raw = re.compile(r'b?r(#*)"').match(text, i)
+        if raw and (i == 0 or not (text[i - 1].isalnum() or text[i - 1] == "_")):
+            end = text.find('"' + raw.group(1), raw.end())
+            if end < 0:
+                raise SplitError("unterminated raw string")
+            i = end + 1 + len(raw.group(1))
+            continue
+        if c == '"' or (c == "b" and text.startswith('b"', i) and (i == 0 or not text[i - 1].isalnum())):
+            j = i + (2 if c == "b" else 1)
+            while j < n and text[j] != '"':
+                j += 2 if text[j] == "\\" else 1
+            i = j + 1
+            continue
+        if c == "'":
+            char = re.compile(r"'(?:\\(?:[nrt\\0'\"]|x[0-9a-fA-F]{2}|u\{[0-9a-fA-F_]{1,6}\})|[^\\'\n])'").match(text, i)
+            if char:
+                i = char.end()
+                continue
+        mask[i] = 1
+        i += 1
+    return mask
+
+
+@dataclass
+class Item:
+    name: str
+    kind: str
+    text: str  # verbatim, from its first comment/attribute line through its last line, with newline
+
+
+def split_items(text: str) -> tuple[str, list[Item]]:
+    """Return (license header, top-level items) and prove every non-blank line is covered once."""
+    mask = code_mask(text)
+    ends: list[int] = []
+    depth = 0
+    for i, ch in enumerate(text):
+        if not mask[i]:
+            continue
+        if ch in "([{":
+            depth += 1
+        elif ch in ")]}":
+            depth -= 1
+            if depth < 0:
+                raise SplitError(f"unbalanced closing '{ch}' at offset {i}")
+            if depth == 0 and ch == "}":
+                nxt = next((k for k in range(i + 1, len(text)) if mask[k] and not text[k].isspace()), None)
+                if nxt is None or text[nxt] != ";":
+                    ends.append(i)
+        elif ch == ";" and depth == 0:
+            ends.append(i)
+    if depth:
+        raise SplitError("unbalanced braces at end of file")
+    first_code = next(i for i in range(len(text)) if mask[i] and not text[i].isspace())
+    header_end = text.rfind("\n", 0, first_code) + 1
+    header = text[:header_end]
+    # The header is the leading comment block; anything else before the first item belongs to that item.
+    header_lines = header.splitlines(keepends=True)
+    while header_lines and header_lines[-1].strip():
+        header_lines.pop()
+    header = "".join(header_lines)
+    if not header.strip():
+        raise SplitError("the file does not start with a license header")
+    items: list[Item] = []
+    start = len(header)
+    for end in ends:
+        line_end = text.find("\n", end)
+        line_end = len(text) if line_end < 0 else line_end + 1
+        chunk = text[start:line_end]
+        stripped = chunk.lstrip("\n")
+        if not stripped.strip():
+            raise SplitError(f"empty item before offset {end}")
+        items.append(classify(stripped))
+        start = line_end
+    if text[start:].strip():
+        raise SplitError("text after the last item")
+    covered = [line for item in items for line in item.text.splitlines() if line.strip()]
+    covered = [line for line in header.splitlines() if line.strip()] + covered
+    expected = [line for line in text.splitlines() if line.strip()]
+    if covered != expected:
+        raise SplitError("item split does not cover every non-blank input line exactly once")
+    return header, items
+
+
+def classify(chunk: str) -> Item:
+    mask = code_mask(chunk)
+    depth = 0
+    for line_start, line in _lines_with_offsets(chunk):
+        body = "".join(ch if mask[line_start + k] else " " for k, ch in enumerate(line)).strip()
+        if not body:
+            continue
+        if depth == 0 and not body.startswith("#"):
+            match = ITEM_NAME.match(body)
+            if not match:
+                raise SplitError(f"unrecognised item head: {body[:80]!r}")
+            kind = match.group("kind")
+            return Item(name=match.group("name") or "", kind=kind, text=chunk)
+        depth += body.count("[") - body.count("]")
+    raise SplitError(f"item without a head: {chunk[:80]!r}")
+
+
+def _lines_with_offsets(text: str):
+    offset = 0
+    for line in text.splitlines(keepends=True):
+        yield offset, line.rstrip("\n")
+        offset += len(line)
+
+
+# ---------------------------------------------------------------------------
+# Rewrites applied to moved items.
+
+
+def rewrite_paths(text: str) -> str:
+    mask = code_mask(text)
+    out = []
+    last = 0
+    for match in SUPER_PATH.finditer(text):
+        if mask[match.start()]:
+            out.append(text[last : match.start()])
+            out.append("super::super::")
+            last = match.end()
+    out.append(text[last:])
+    text = "".join(out)
+    text = text.replace("use s3s::S3;", "use crate::app::storage_api::s3::S3;")
+    text = re.sub(r"(?<![A-Za-z0-9_:])s3s::S3ErrorCode\b", "S3ErrorCode", text)
+    return text
+
+
+def make_shared(text: str) -> str:
+    """Give the item head `pub(super)`; attributes and comments before it are untouched."""
+    mask = code_mask(text)
+    for match in VISIBILITY_TARGET.finditer(text):
+        if mask[match.start("head")] and _depth_at(text, mask, match.start("head")) == 0:
+            pos = match.start("head")
+            return text[:pos] + "pub(super) " + text[pos:]
+    raise SplitError(f"cannot find the head of shared item {text[:80]!r}")
+
+
+def _depth_at(text: str, mask: bytearray, pos: int) -> int:
+    depth = 0
+    for i in range(pos):
+        if mask[i]:
+            depth += text[i] in "([{"
+            depth -= text[i] in ")]}"
+    return depth
+
+
+def rewrite_seam_calls(text: str, module: str) -> tuple[str, int]:
+    """Rewrite `<receiver>.execute_<op>(args)` to `test_seams::<seam>(&<receiver>, args)` outside the owner."""
+    count = 0
+    for method, (seam, owner) in SEAMS.items():
+        if owner == module:
+            continue
+        while True:
+            mask = code_mask(text)
+            match = next((m for m in re.finditer(r"\.\s*" + method + r"\(", text) if mask[m.start()]), None)
+            if match is None:
+                break
+            recv_start = _receiver_start(text, match.start())
+            receiver = text[recv_start : match.start()].rstrip()
+            if not receiver:
+                raise SplitError(f"{module}: call to {method} without a receiver")
+            args_start = match.end()
+            args_end = _matching(text, mask, match.end() - 1)
+            args = text[args_start:args_end]
+            text = text[:recv_start] + f"test_seams::{seam}(&{receiver}, {args})" + text[args_end + 1 :]
+            count += 1
+    return text, count
+
+
+def _receiver_start(text: str, dot: int) -> int:
+    i = dot
+    while i > 0 and text[i - 1].isspace():
+        i -= 1
+    end = i
+    if end and text[end - 1] == ")":
+        depth = 0
+        while i > 0:
+            i -= 1
+            depth += text[i] == ")"
+            depth -= text[i] == "("
+            if depth == 0:
+                break
+    while i > 0 and (text[i - 1].isalnum() or text[i - 1] in "_:"):
+        i -= 1
+    if i == end or (i > 0 and text[i - 1] == "."):
+        raise SplitError(f"unsupported receiver before offset {dot}: only a name or a path call is rewritten")
+    return i
+
+
+def _matching(text: str, mask: bytearray, open_pos: int) -> int:
+    depth = 0
+    for i in range(open_pos, len(text)):
+        if not mask[i]:
+            continue
+        depth += text[i] in "([{"
+        depth -= text[i] in ")]}"
+        if depth == 0:
+            return i
+    raise SplitError(f"unbalanced call at offset {open_pos}")
+
+
+# ---------------------------------------------------------------------------
+# Assembly.
+
+
+def build(text: str) -> dict[Path, str]:
+    """Return the full content of every output file, keyed by repository-relative path."""
+    header, items = split_items(text)
+    license_block = header.rstrip("\n") + "\n"
+    uses = [item for item in items if item.kind == "use"]
+    moved = [item for item in items if item.kind != "use"]
+    if items[: len(uses)] != uses:
+        raise SplitError("a `use` declaration follows another item; the root keeps only the leading use block")
+    names = [item.name for item in moved]
+    duplicates = sorted({name for name in names if names.count(name) > 1})
+    if duplicates:
+        raise SplitError(f"items defined twice: {', '.join(duplicates)}")
+    unassigned = [name for name in names if name not in ASSIGNMENT]
+    stale = sorted(set(ASSIGNMENT) - set(names))
+    if unassigned or stale:
+        raise SplitError(
+            "ASSIGNMENT does not match the input:"
+            + "".join(f"\n  unassigned item: {name}" for name in unassigned)
+            + "".join(f"\n  no such item: {name}" for name in stale)
+        )
+    unknown = sorted({module for module in ASSIGNMENT.values() if module not in MODULES})
+    if unknown:
+        raise SplitError(f"ASSIGNMENT names undeclared modules: {', '.join(unknown)}")
+
+    bodies: dict[str, list[str]] = {module: [] for module in MODULES}
+    for item in moved:
+        module = ASSIGNMENT[item.name]
+        body, _ = rewrite_seam_calls(rewrite_paths(item.text), module)
+        bodies[module].append(body)
+
+    # A common item gains `pub(super)` only when a sibling uses it; the rest stay private to common.rs.
+    sibling_code = _code_only("\n".join(body for module, items in bodies.items() if module != "common" for body in items))
+    common_items = [item for item in moved if ASSIGNMENT[item.name] == "common"]
+    common_names = {item.name for item in common_items if re.search(rf"\b{re.escape(item.name)}\b", sibling_code)}
+    bodies["common"] = [
+        make_shared(body) if item.name in common_names else body for item, body in zip(common_items, bodies["common"])
+    ]
+    outputs: dict[Path, str] = {}
+    root_uses = "".join(item.text if item.text.endswith("\n") else item.text + "\n" for item in uses)
+    root = (
+        license_block
+        + "\n"
+        + "//! Lifecycle, transition and object-version integration tests, split by the batch that owns each operation.\n"
+        + "//!\n"
+        + "//! Generated by scripts/codemods/split_lifecycle_transition_api_test.py; each module file names its owner.\n"
+        + "\n"
+        + root_uses
+        + ROOT_EXTRA_USE
+        + "\n"
+        + "".join(f"mod {module};\n" for module in MODULES)
+    )
+    outputs[SOURCE] = root
+    for module, doc in MODULES.items():
+        content = bodies[module]
+        if not content:
+            raise SplitError(f"module {module} receives no item")
+        joined = "\n".join(content)
+        preamble = ["use super::*;\n"]
+        if module != "common" and any(re.search(rf"\b{re.escape(name)}\b", joined) for name in common_names):
+            preamble.append("use super::common::*;\n")
+        if "test_seams::" in joined:
+            preamble.append("use crate::app::test_seams;\n")
+        text_out = license_block + "\n" + f"//! {doc}\n" + "\n" + "".join(preamble) + "\n" + joined
+        leftover = [line for line in text_out.splitlines() if S3S_PATH.search(line)]
+        if leftover:
+            raise SplitError(f"{module}.rs still names s3s: {leftover[0].strip()}")
+        for method, (_, owner) in SEAMS.items():
+            if owner != module and re.search(r"\." + method + r"\(", _code_only(text_out)):
+                raise SplitError(f"{module}.rs still calls {method} directly")
+        outputs[TARGET_DIR / f"{module}.rs"] = text_out
+    return outputs
+
+
+def _code_only(text: str) -> str:
+    mask = code_mask(text)
+    return "".join(ch if mask[i] or ch == "\n" else " " for i, ch in enumerate(text))
+
+
+def run(root: Path, fmt: bool) -> int:
+    source = root / SOURCE
+    try:
+        text = source.read_text()
+    except OSError as error:
+        print(f"error: cannot read {SOURCE}: {error}", file=sys.stderr)
+        return 1
+    if re.search(r"(?m)^mod common;$", text):
+        print(f"error: {SOURCE} is already split; restore the unsplit file first", file=sys.stderr)
+        return 1
+    try:
+        outputs = build(text)
+    except SplitError as error:
+        print(f"error: {error}", file=sys.stderr)
+        return 1
+    stray = sorted(p.name for p in (root / TARGET_DIR).glob("*.rs") if TARGET_DIR / p.name not in outputs) if (
+        root / TARGET_DIR
+    ).is_dir() else []
+    if stray:
+        print(f"error: {TARGET_DIR} holds files this codemod does not write: {', '.join(stray)}", file=sys.stderr)
+        return 1
+    (root / TARGET_DIR).mkdir(parents=True, exist_ok=True)
+    for path, content in outputs.items():
+        (root / path).write_text(content)
+    if fmt:
+        paths = [str(root / path) for path in outputs]
+        result = subprocess.run(["rustfmt", "--edition", "2024", *paths], cwd=root, check=False)
+        if result.returncode != 0:
+            print("error: rustfmt failed on the written files", file=sys.stderr)
+            return 1
+    for path in outputs:
+        print(f"wrote {path} ({len((root / path).read_text().splitlines())} lines)")
+    return 0
+
+
+def main(argv: list[str]) -> int:
+    if argv == ["--self-test"]:
+        suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTest)
+        return 0 if unittest.TextTestRunner(verbosity=1).run(suite).wasSuccessful() else 1
+    if argv not in ([], ["--no-fmt"]):
+        print(__doc__.split("Usage:", 1)[1].split("\n\n", 1)[0], file=sys.stderr)
+        return 2
+    top = subprocess.run(["git", "rev-parse", "--show-toplevel"], capture_output=True, text=True, check=False)
+    if top.returncode != 0:
+        print("error: not inside a git checkout", file=sys.stderr)
+        return 2
+    return run(Path(top.stdout.strip()), fmt=argv != ["--no-fmt"])
+
+
+# ---------------------------------------------------------------------------
+# Self-test
+
+
+class SelfTest(unittest.TestCase):
+    HEADER = "// Copyright\n// License\n\n"
+
+    def items(self, body: str) -> list[Item]:
+        return split_items(self.HEADER + body)[1]
+
+    def test_items_keep_attributes_comments_and_braces_in_literals(self):
+        body = 'use a::{b, c};\n\n// note\n#[test]\nfn one() {\n    let s = "}";\n    let c = \'}\';\n}\n\nstatic X: u8 = 1;\n'
+        items = self.items(body)
+        self.assertEqual([(i.kind, i.name) for i in items], [("use", "a"), ("fn", "one"), ("static", "X")])
+        self.assertTrue(items[1].text.startswith("// note\n#[test]\n"))
+
+    def test_raw_strings_and_lifetimes_do_not_confuse_depth(self):
+        items = self.items("fn a<'x>(v: &'x str) {\n    let r = r#\"{\"#;\n}\nfn b() {}\n")
+        self.assertEqual([i.name for i in items], ["a", "b"])
+
+    def test_unbalanced_input_is_rejected(self):
+        with self.assertRaises(SplitError):
+            split_items(self.HEADER + "fn a() {\n")
+
+    def test_unassigned_item_is_rejected(self):
+        with self.assertRaisesRegex(SplitError, "unassigned item: not_in_table"):
+            build(self.HEADER + "use x::y;\nfn not_in_table() {}\n")
+
+    def test_super_paths_move_one_level_down_but_not_in_strings(self):
+        source = 'fn a() {\n    use super::x::Y;\n    let s = "super::z";\n}\n'
+        self.assertEqual(rewrite_paths(source), 'fn a() {\n    use super::super::x::Y;\n    let s = "super::z";\n}\n')
+
+    def test_s3s_error_code_and_trait_paths_are_rewritten(self):
+        source = "fn a() {\n    use s3s::S3;\n    f(&s3s::S3ErrorCode::NoSuchKey);\n}\n"
+        self.assertEqual(
+            rewrite_paths(source),
+            "fn a() {\n    use crate::app::storage_api::s3::S3;\n    f(&S3ErrorCode::NoSuchKey);\n}\n",
+        )
+
+    def test_shared_items_gain_pub_super_after_attributes(self):
+        self.assertEqual(make_shared("#[allow(x)]\nasync fn a() {}\n"), "#[allow(x)]\npub(super) async fn a() {}\n")
+        self.assertEqual(make_shared("static A: u8 = 1;\n"), "pub(super) static A: u8 = 1;\n")
+
+    def test_cross_owner_calls_use_the_seam(self):
+        source = "fn t() {\n    Box::pin(usecase\n        .execute_put_object(&fs, req(x)))\n}\n"
+        rewritten, count = rewrite_seam_calls(source, "copy")
+        self.assertEqual(count, 1)
+        self.assertEqual(rewritten, "fn t() {\n    Box::pin(test_seams::put_object(&usecase, &fs, req(x)))\n}\n")
+
+    def test_owner_calls_stay_direct(self):
+        source = "fn t() {\n    usecase.execute_put_object(&fs, req)\n}\n"
+        self.assertEqual(rewrite_seam_calls(source, "put"), (source, 0))
+
+    def test_call_expression_receivers_are_kept_whole(self):
+        source = "fn t() {\n    Usecase::with(Some(a)).execute_get_object(r)\n}\n"
+        rewritten, _ = rewrite_seam_calls(source, "delete")
+        self.assertEqual(rewritten, "fn t() {\n    test_seams::get_object(&Usecase::with(Some(a)), r)\n}\n")
+
+    def test_already_split_input_is_refused(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / SOURCE
+            path.parent.mkdir(parents=True)
+            path.write_text(self.HEADER + "mod common;\n")
+            self.assertEqual(run(Path(tmp), fmt=False), 1)
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

scripts/download_rc.sh

@@ -13,7 +13,7 @@ OUTPUT="$4"
 
 case "$ARCH" in
   amd64|arm64) ;;
-  *) echo "Unsupported rc architecture: $ARCH" >&2; exit 1 ;;
+  *) echo "Unsupported rc architecture: '$ARCH' (expected amd64 or arm64; an empty value means the caller's TARGETARCH build argument was not set)" >&2; exit 1 ;;
 esac
 case "$LIBC" in
   musl) LIBC_SUFFIX="" ;;

scripts/release/package_manual_artifacts.sh

@@ -0,0 +1,406 @@
+#!/usr/bin/env bash
+# Copyright 2026 RustFS Team
+# SPDX-License-Identifier: Apache-2.0
+# Manual zip/deb/rpm packaging for one Linux target.
+# Version rules and deb/rpm metadata come from package_versions.sh.
+# deb/rpm payloads are built by build_deb.sh and build_rpm.sh.
+# Dependency strings match scripts/release/deliver.py package().
+
+set -euo pipefail
+
+repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+helper="${repo_root}/scripts/release/package_versions.sh"
+
+die() {
+  printf 'package_manual_artifacts: %s\n' "$1" >&2
+  exit 1
+}
+
+usage() {
+  cat <<'EOF' >&2
+usage: package_manual_artifacts.sh matrix <x86|arm|all>
+       package_manual_artifacts.sh validate --version VER --arch ARCH --libc LIBC
+       package_manual_artifacts.sh package --version VER --arch ARCH --libc LIBC --binary PATH --output DIR [--zip-extra FILE]...
+EOF
+  exit 2
+}
+
+require_value() {
+  local flag=$1
+  # An empty value is a version/path error, not a missing argument. Flag-shaped
+  # values mean the next option was consumed by mistake.
+  if [[ $# -lt 2 || ${2:-} == --* ]]; then
+    die "missing value for ${flag}"
+  fi
+}
+
+resolve_version() {
+  local version=$1 arch=$2 libc=$3
+  local release_err prerelease_err release_out prerelease_out
+  release_err=$(mktemp)
+  prerelease_err=$(mktemp)
+  if release_out=$(bash "$helper" release "$version" "" "$arch" "$libc" 2>"$release_err"); then
+    printf 'release\n%s\n' "$release_out"
+    rm -f "$release_err" "$prerelease_err"
+    return 0
+  fi
+  if prerelease_out=$(bash "$helper" prerelease "$version" "" "$arch" "$libc" 2>"$prerelease_err"); then
+    printf 'prerelease\n%s\n' "$prerelease_out"
+    rm -f "$release_err" "$prerelease_err"
+    return 0
+  fi
+  printf 'invalid package version: %s\n' "$version" >&2
+  printf 'not a release: %s\n' "$(<"$release_err")" >&2
+  printf 'not a prerelease: %s\n' "$(<"$prerelease_err")" >&2
+  rm -f "$release_err" "$prerelease_err"
+  return 1
+}
+
+load_versions() {
+  local resolved line key value
+  resolved=$(resolve_version "$version" "$arch" "$libc") || exit 1
+  build_type=${resolved%%$'\n'*}
+  local rest=${resolved#*$'\n'}
+  deb_version=''
+  rpm_version=''
+  rpm_release=''
+  deb_file=''
+  rpm_file=''
+  while IFS= read -r line || [[ -n "$line" ]]; do
+    [[ -n "$line" ]] || continue
+    key=${line%%=*}
+    value=${line#*=}
+    case "$key" in
+      deb_version) deb_version=$value ;;
+      rpm_version) rpm_version=$value ;;
+      rpm_release) rpm_release=$value ;;
+      deb_file) deb_file=$value ;;
+      rpm_file) rpm_file=$value ;;
+      *) die "unexpected package_versions field: ${key}" ;;
+    esac
+  done <<< "$rest"
+  [[ -n "$deb_version" && -n "$rpm_version" && -n "$rpm_release" && -n "$deb_file" && -n "$rpm_file" ]] ||
+    die "incomplete package version output"
+  local stem=${deb_file%.deb}
+  [[ "$stem" == "${rpm_file%.rpm}" ]] || die "deb and rpm file names use different stems"
+  [[ "$stem" == "rustfs-linux-${arch}-${libc}-v${version}" ]] ||
+    die "package stem does not match rustfs-linux-${arch}-${libc}-v${version}"
+  zip_file="${stem}.zip"
+  case "$build_type" in
+    release | prerelease) ;;
+    *) die "unsupported build type: ${build_type}" ;;
+  esac
+}
+
+emit_resolved() {
+  printf 'build_type=%s\n' "$build_type"
+  printf 'deb_version=%s\n' "$deb_version"
+  printf 'rpm_version=%s\n' "$rpm_version"
+  printf 'rpm_release=%s\n' "$rpm_release"
+  printf 'deb_file=%s\n' "$deb_file"
+  printf 'rpm_file=%s\n' "$rpm_file"
+  printf 'zip_file=%s\n' "$zip_file"
+}
+
+emit_matrix() {
+  local choice=$1
+  command -v python3 >/dev/null 2>&1 || die "python3 is required"
+  python3 - "$choice" <<'PY'
+import json
+import sys
+
+choice = sys.argv[1]
+# Linux rows match the release matrix in .github/workflows/build.yml.
+# arch, libc, and build_os are the packaging names for those same rows.
+rows = [
+    {"target_id": "linux-x86_64-musl", "os": "sm-standard-4", "target": "x86_64-unknown-linux-musl", "cross": False, "platform": "linux", "rustflags": "", "features": "", "glibc": "", "arch": "x86_64", "libc": "musl", "build_os": "linux-x86_64"},
+    {"target_id": "linux-aarch64-musl", "os": "ubicloud-standard-16-arm", "target": "aarch64-unknown-linux-musl", "cross": False, "platform": "linux", "rustflags": "", "features": "", "glibc": "", "arch": "aarch64", "libc": "musl", "build_os": "linux-aarch64"},
+    {"target_id": "linux-x86_64-gnu", "os": "sm-standard-4", "target": "x86_64-unknown-linux-gnu", "cross": True, "platform": "linux", "rustflags": "", "features": "pyroscope", "glibc": "2.31", "arch": "x86_64", "libc": "gnu", "build_os": "linux-x86_64"},
+    {"target_id": "linux-aarch64-gnu", "os": "ubicloud-standard-16-arm", "target": "aarch64-unknown-linux-gnu", "cross": True, "platform": "linux", "rustflags": "", "features": "pyroscope", "glibc": "2.31", "arch": "aarch64", "libc": "gnu", "build_os": "linux-aarch64"},
+]
+if choice == "all":
+    selected = rows
+elif choice == "x86":
+    selected = [row for row in rows if row["arch"] == "x86_64"]
+elif choice == "arm":
+    selected = [row for row in rows if row["arch"] == "aarch64"]
+else:
+    raise SystemExit(
+        "package_manual_artifacts: unsupported architecture "
+        f"(expected x86, arm, or all): {choice}"
+    )
+if len(selected) == 0:
+    raise SystemExit("package_manual_artifacts: no targets selected")
+json.dump({"include": selected}, sys.stdout, separators=(",", ":"))
+sys.stdout.write("\n")
+PY
+}
+
+parse_identity() {
+  version=''
+  arch=''
+  libc=''
+  local saw_version=0 saw_arch=0 saw_libc=0
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --version)
+        require_value "$@"
+        saw_version=1
+        version=$2
+        shift 2
+        ;;
+      --arch)
+        require_value "$@"
+        saw_arch=1
+        arch=$2
+        shift 2
+        ;;
+      --libc)
+        require_value "$@"
+        saw_libc=1
+        libc=$2
+        shift 2
+        ;;
+      *)
+        die "unknown argument: $1"
+        ;;
+    esac
+  done
+  [[ "$saw_version" == 1 && "$saw_arch" == 1 && "$saw_libc" == 1 ]] ||
+    die "validate requires --version, --arch, and --libc"
+}
+
+abspath() {
+  local path=$1
+  local dir base
+  dir=$(cd "$(dirname "$path")" && pwd)
+  base=$(basename "$path")
+  printf '%s/%s' "$dir" "$base"
+}
+
+sha256_file() {
+  sha256sum "$1" | awk 'NR==1 { print $1 }'
+}
+
+package_binary() {
+  local binary='' output=''
+  local -a zip_extras=()
+  local saw_version=0 saw_arch=0 saw_libc=0
+  version=''
+  arch=''
+  libc=''
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --version)
+        require_value "$@"
+        saw_version=1
+        version=$2
+        shift 2
+        ;;
+      --arch)
+        require_value "$@"
+        saw_arch=1
+        arch=$2
+        shift 2
+        ;;
+      --libc)
+        require_value "$@"
+        saw_libc=1
+        libc=$2
+        shift 2
+        ;;
+      --binary)
+        require_value "$@"
+        binary=$2
+        shift 2
+        ;;
+      --output)
+        require_value "$@"
+        output=$2
+        shift 2
+        ;;
+      --zip-extra)
+        require_value "$@"
+        zip_extras+=("$2")
+        shift 2
+        ;;
+      *)
+        die "unknown argument: $1"
+        ;;
+    esac
+  done
+  [[ "$saw_version" == 1 && "$saw_arch" == 1 && "$saw_libc" == 1 && -n "$binary" && -n "$output" ]] ||
+    die "package requires --version, --arch, --libc, --binary, and --output"
+  [[ -f "$binary" && ! -L "$binary" ]] || die "binary must be a regular file"
+  load_versions
+
+  local -a extra_names=()
+  local extra base seen=' '
+  local has_json=0 has_sha=0
+  for extra in "${zip_extras[@]}"; do
+    base=$(basename "$extra")
+    case "$base" in
+      rustfs.cpu-symbol-catalog.json) has_json=1 ;;
+      rustfs.cpu-symbol-catalog.sha256) has_sha=1 ;;
+      *) die "unsupported zip member: ${base}" ;;
+    esac
+    [[ "$seen" != *" ${base} "* ]] || die "duplicate zip member: ${base}"
+    seen+=" ${base} "
+    [[ -f "$extra" && ! -L "$extra" ]] || die "zip extra must be a regular file: ${base}"
+    extra_names+=("$base")
+  done
+  [[ "$has_json" == "$has_sha" ]] ||
+    die "cpu symbol catalog json and sha256 must be packaged together"
+
+  local tool
+  for tool in zip unzip sha256sum dpkg-deb fakeroot rpm rpm2cpio cpio fpm; do
+    command -v "$tool" >/dev/null 2>&1 || die "required tool not found: ${tool}"
+  done
+
+  binary=$(abspath "$binary")
+  local -a absolute_extras=()
+  for extra in "${zip_extras[@]}"; do
+    absolute_extras+=("$(abspath "$extra")")
+  done
+  mkdir -p "$output"
+  output=$(cd "$output" && pwd)
+  local produced
+  for produced in "$zip_file" "$deb_file" "$rpm_file"; do
+    [[ ! -e "$output/$produced" ]] || die "refusing to overwrite ${output}/${produced}"
+  done
+
+  # Globals so the EXIT trap can still see them after this function returns.
+  work=$(mktemp -d)
+  stage=$(mktemp -d)
+  cleanup() {
+    rm -rf "${work:-}" "${stage:-}"
+  }
+  trap cleanup EXIT
+
+  local zip_dir="$work/zip"
+  mkdir -p "$zip_dir"
+  cp "$binary" "$zip_dir/rustfs"
+  chmod 755 "$zip_dir/rustfs"
+  local index=0
+  for extra in "${absolute_extras[@]}"; do
+    cp "$extra" "$zip_dir/${extra_names[$index]}"
+    index=$((index + 1))
+  done
+  (
+    cd "$zip_dir"
+    zip "$stage/$zip_file" rustfs "${extra_names[@]}"
+  )
+
+  local members expected
+  members=$(unzip -Z1 "$stage/$zip_file" | sort)
+  expected=$(printf '%s\n' rustfs "${extra_names[@]}" | sort)
+  [[ "$members" == "$expected" ]] || die "zip members do not match the release archive"
+  [[ "$(unzip -p "$stage/$zip_file" rustfs | sha256sum | awk '{print $1}')" == "$(sha256_file "$binary")" ]] ||
+    die "zip binary differs from the packaged executable"
+
+  ln -s "$repo_root/scripts" "$work/scripts"
+  ln -s "$repo_root/deploy" "$work/deploy"
+  cp "$repo_root/LICENSE" "$repo_root/README.md" "$work/"
+  mkdir -p "$work/bin"
+  cp "$binary" "$work/bin/rustfs"
+  chmod 755 "$work/bin/rustfs"
+
+  local deb_arch deb_depends rpm_depends
+  case "$arch" in
+    x86_64) deb_arch=amd64 ;;
+    aarch64) deb_arch=arm64 ;;
+    *) die "unsupported architecture: ${arch}" ;;
+  esac
+  case "$libc" in
+    gnu)
+      deb_depends='libc6 (>= 2.31)'
+      rpm_depends='glibc >= 2.31'
+      ;;
+    musl)
+      deb_depends=''
+      rpm_depends=''
+      ;;
+    *) die "unsupported libc: ${libc}" ;;
+  esac
+
+  (
+    cd "$work"
+    GITHUB_OUTPUT="$work/outputs" \
+      DEB_VERSION="$deb_version" \
+      RPM_VERSION="$rpm_version" \
+      RPM_RELEASE="$rpm_release" \
+      DEB_FILE="$deb_file" \
+      RPM_FILE="$rpm_file" \
+      DEB_ARCH="$deb_arch" \
+      RPM_ARCH="$arch" \
+      DEB_DEPENDS="$deb_depends" \
+      RPM_DEPENDS="$rpm_depends" \
+      LIBC="$libc" \
+      bash "$repo_root/scripts/release/build_deb.sh"
+    GITHUB_OUTPUT="$work/outputs" \
+      DEB_VERSION="$deb_version" \
+      RPM_VERSION="$rpm_version" \
+      RPM_RELEASE="$rpm_release" \
+      DEB_FILE="$deb_file" \
+      RPM_FILE="$rpm_file" \
+      DEB_ARCH="$deb_arch" \
+      RPM_ARCH="$arch" \
+      DEB_DEPENDS="$deb_depends" \
+      RPM_DEPENDS="$rpm_depends" \
+      LIBC="$libc" \
+      bash "$repo_root/scripts/release/build_rpm.sh"
+  )
+  cp "$work/$deb_file" "$stage/$deb_file"
+  cp "$work/$rpm_file" "$stage/$rpm_file"
+
+  local unpacked expected_hash actual_hash
+  expected_hash=$(sha256_file "$binary")
+  unpacked="$work/unpacked-deb"
+  mkdir -p "$unpacked"
+  dpkg-deb --extract "$stage/$deb_file" "$unpacked"
+  actual_hash=$(sha256_file "$unpacked/usr/bin/rustfs")
+  [[ "$actual_hash" == "$expected_hash" ]] || die "deb binary differs from the packaged executable"
+  [[ "$(dpkg-deb -f "$stage/$deb_file" Package)" == rustfs ]] || die "deb package name mismatch"
+  [[ "$(dpkg-deb -f "$stage/$deb_file" Version)" == "$deb_version" ]] || die "deb version mismatch"
+  [[ "$(dpkg-deb -f "$stage/$deb_file" Architecture)" == "$deb_arch" ]] || die "deb architecture mismatch"
+
+  unpacked="$work/unpacked-rpm"
+  mkdir -p "$unpacked"
+  rpm2cpio "$stage/$rpm_file" | (cd "$unpacked" && cpio --extract --make-directories --quiet ./usr/bin/rustfs)
+  actual_hash=$(sha256_file "$unpacked/usr/bin/rustfs")
+  [[ "$actual_hash" == "$expected_hash" ]] || die "rpm binary differs from the packaged executable"
+  local rpm_metadata expected_metadata
+  rpm_metadata=$(rpm -qp --qf '%{NAME}\n%{VERSION}\n%{RELEASE}\n%{ARCH}\n' "$stage/$rpm_file")
+  expected_metadata=$(printf 'rustfs\n%s\n%s\n%s' "$rpm_version" "$rpm_release" "$arch")
+  [[ "$rpm_metadata" == "$expected_metadata" ]] || die "rpm metadata mismatch"
+
+  mv "$stage/$zip_file" "$output/$zip_file"
+  mv "$stage/$deb_file" "$output/$deb_file"
+  mv "$stage/$rpm_file" "$output/$rpm_file"
+  emit_resolved
+}
+
+main() {
+  local command=${1:-}
+  case "$command" in
+    matrix)
+      [[ $# -eq 2 ]] || usage
+      emit_matrix "$2"
+      ;;
+    validate)
+      shift
+      parse_identity "$@"
+      load_versions
+      emit_resolved
+      ;;
+    package)
+      shift
+      package_binary "$@"
+      ;;
+    *)
+      usage
+      ;;
+  esac
+}
+
+main "$@"

scripts/release/test_manual_main_packages.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+# Copyright 2026 RustFS Team
+# SPDX-License-Identifier: Apache-2.0
+"""Contract tests for manual origin/main packaging."""
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+
+ROOT = Path(__file__).resolve().parents[2]
+SCRIPT = ROOT / "scripts" / "release" / "package_manual_artifacts.sh"
+WORKFLOW = ROOT / ".github" / "workflows" / "manual-main-packages.yml"
+sys.path.insert(0, str(ROOT / "scripts"))
+from ci_gate import workflow_document  # noqa: E402
+import validation_registry as registry  # noqa: E402
+
+
+def run_script(*args, check=False):
+    return subprocess.run(["bash", str(SCRIPT), *args], cwd=ROOT, text=True,
+                          capture_output=True, timeout=60, check=check)
+
+
+def trigger(text):
+    match = re.search(r"^on:\n(.*?)(?=^[^ #\n]|\Z)", text, re.M | re.S)
+    if match is None:
+        raise AssertionError("workflow has no event declaration")
+    return re.findall(r"^  (\w+):", match.group(1), re.M)
+
+
+class VersionAndMatrixTests(unittest.TestCase):
+    def test_release_and_prerelease_names_come_from_package_versions(self):
+        cases = {
+            ("1.2.3", "x86_64", "gnu"): """\
+build_type=release
+deb_version=1.2.3
+rpm_version=1.2.3
+rpm_release=1
+deb_file=rustfs-linux-x86_64-gnu-v1.2.3.deb
+rpm_file=rustfs-linux-x86_64-gnu-v1.2.3.rpm
+zip_file=rustfs-linux-x86_64-gnu-v1.2.3.zip
+""",
+            ("1.2.3-alpha.1", "aarch64", "gnu"): """\
+build_type=prerelease
+deb_version=1.2.3~alpha.1
+rpm_version=1.2.3~alpha.1
+rpm_release=1
+deb_file=rustfs-linux-aarch64-gnu-v1.2.3-alpha.1.deb
+rpm_file=rustfs-linux-aarch64-gnu-v1.2.3-alpha.1.rpm
+zip_file=rustfs-linux-aarch64-gnu-v1.2.3-alpha.1.zip
+""",
+            ("1.2.3-beta.2", "x86_64", "musl"): """\
+build_type=prerelease
+deb_version=1.2.3~beta.2
+rpm_version=1.2.3~beta.2
+rpm_release=1
+deb_file=rustfs-linux-x86_64-musl-v1.2.3-beta.2.deb
+rpm_file=rustfs-linux-x86_64-musl-v1.2.3-beta.2.rpm
+zip_file=rustfs-linux-x86_64-musl-v1.2.3-beta.2.zip
+""",
+            ("1.2.3-rc.4", "aarch64", "musl"): """\
+build_type=prerelease
+deb_version=1.2.3~rc.4
+rpm_version=1.2.3~rc.4
+rpm_release=1
+deb_file=rustfs-linux-aarch64-musl-v1.2.3-rc.4.deb
+rpm_file=rustfs-linux-aarch64-musl-v1.2.3-rc.4.rpm
+zip_file=rustfs-linux-aarch64-musl-v1.2.3-rc.4.zip
+""",
+        }
+        for (version, arch, libc), expected in cases.items():
+            with self.subTest(version=version, arch=arch, libc=libc):
+                result = run_script("validate", "--version", version, "--arch", arch, "--libc", libc)
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertEqual(result.stdout, expected)
+
+    def test_invalid_versions_fail_without_stdout(self):
+        for version in ("", "v1.2.3", "1.2", "01.2.3", "1.2.3-preview.1", "1.2.3-alpha",
+                        "1.2.3-rc.01", "1.2.3-rc.1-extra", "1.2.3 ", "1.2.3$(id)", "dev-abcdef"):
+            with self.subTest(version=version):
+                result = run_script("validate", "--version", version, "--arch", "x86_64", "--libc", "gnu")
+                self.assertNotEqual(result.returncode, 0)
+                self.assertEqual(result.stdout, "")
+                self.assertIn("invalid package version", result.stderr)
+                self.assertIn("not a release:", result.stderr)
+                self.assertIn("not a prerelease:", result.stderr)
+
+    def test_invalid_version_does_not_create_packages(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp)
+            binary = root / "rustfs"
+            binary.write_bytes(b"not packaged")
+            output = root / "dist"
+            result = run_script("package", "--version", "v1.2.3", "--arch", "x86_64", "--libc", "gnu",
+                                "--binary", str(binary), "--output", str(output))
+            self.assertNotEqual(result.returncode, 0)
+            self.assertEqual(result.stdout, "")
+            self.assertFalse(output.exists())
+
+    def test_catalog_members_must_be_the_official_pair(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp)
+            binary = root / "rustfs"
+            binary.write_bytes(b"not packaged")
+            catalog = root / "rustfs.cpu-symbol-catalog.json"
+            catalog.write_text("{}\n", encoding="utf-8")
+            result = run_script("package", "--version", "1.2.3", "--arch", "x86_64", "--libc", "gnu",
+                                "--binary", str(binary), "--output", str(root / "dist"),
+                                "--zip-extra", str(catalog))
+            self.assertNotEqual(result.returncode, 0)
+            self.assertIn("packaged together", result.stderr)
+            self.assertFalse((root / "dist").exists())
+            other = root / "notes.txt"
+            other.write_text("nope\n", encoding="utf-8")
+            rejected = run_script("package", "--version", "1.2.3", "--arch", "x86_64", "--libc", "musl",
+                                  "--binary", str(binary), "--output", str(root / "dist"),
+                                  "--zip-extra", str(other))
+            self.assertNotEqual(rejected.returncode, 0)
+            self.assertIn("unsupported zip member", rejected.stderr)
+
+    def test_architecture_matrix_matches_official_linux_rows(self):
+        source = (ROOT / ".github" / "workflows" / "build.yml").read_text(encoding="utf-8")
+        official = json.loads(re.search(r"all='(\{.*?\})'", source, re.S).group(1))["include"]
+        linux = [row for row in official if row["platform"] == "linux"]
+
+        def derived(row):
+            arch = "aarch64" if row["target"].startswith("aarch64-") else "x86_64"
+            libc = row["target_id"].rsplit("-", 1)[1]
+            return {**row, "arch": arch, "libc": libc, "build_os": f"linux-{arch}"}
+
+        expected = {
+            "all": [derived(row) for row in linux],
+            "x86": [derived(row) for row in linux if row["target"].startswith("x86_64-")],
+            "arm": [derived(row) for row in linux if row["target"].startswith("aarch64-")],
+        }
+        self.assertEqual([row["target_id"] for row in expected["x86"]],
+                         ["linux-x86_64-musl", "linux-x86_64-gnu"])
+        self.assertEqual([row["target_id"] for row in expected["arm"]],
+                         ["linux-aarch64-musl", "linux-aarch64-gnu"])
+        for choice, rows in expected.items():
+            with self.subTest(choice=choice):
+                result = run_script("matrix", choice, check=True)
+                self.assertEqual(json.loads(result.stdout), {"include": rows})
+        for choice in ("", "ALL", "linux", "x86_64", "x86,arm"):
+            with self.subTest(choice=choice):
+                result = run_script("matrix", choice)
+                self.assertNotEqual(result.returncode, 0)
+                self.assertEqual(result.stdout, "")
+                self.assertIn("unsupported architecture", result.stderr)
+
+
+class WorkflowContractTests(unittest.TestCase):
+    def test_dispatch_builds_origin_main_and_only_uploads_artifacts(self):
+        text = WORKFLOW.read_text(encoding="utf-8")
+        self.assertEqual(trigger(text), ["workflow_dispatch"])
+        document = workflow_document(text)
+        events = document.get("on", document.get(True))
+        self.assertEqual(set(events), {"workflow_dispatch"})
+        inputs = events["workflow_dispatch"]["inputs"]
+        self.assertEqual(inputs["architecture"]["type"], "choice")
+        self.assertEqual(inputs["architecture"]["options"], ["x86", "arm", "all"])
+        self.assertTrue(inputs["architecture"]["required"])
+        self.assertEqual(inputs["version"]["type"], "string")
+        self.assertTrue(inputs["version"]["required"])
+        self.assertEqual(document["permissions"], {"contents": "read"})
+        self.assertEqual(re.findall(r"^\s+ref:\s*(.+)$", text, re.M),
+                         ["main", "${{ needs.prepare.outputs.source_sha }}"])
+        self.assertIn("refs/heads/main", text)
+        self.assertNotIn("github.sha", text)
+        self.assertIn("package_manual_artifacts.sh validate", text)
+        self.assertIn("package_manual_artifacts.sh matrix", text)
+        self.assertIn("package_manual_artifacts.sh package", text)
+        self.assertIn("RUSTFS_BUILD_VERSION: ${{ needs.prepare.outputs.version }}", text)
+        self.assertIn("cargo zigbuild --release --locked", text)
+        self.assertIn("cargo build --release --locked", text)
+        self.assertIn("scripts/release/check_glibc.py", text)
+        self.assertIn('cache-save-if: \'false\'', text)
+        self.assertIn("actions/upload-artifact@", text)
+        self.assertIn("dist/*", text)
+        for forbidden in ("contents: write", "aws s3", "gh release", "softprops/action-gh-release",
+                          "docker push", "R2_ACCESS_KEY_ID", "workflow_call:"):
+            self.assertNotIn(forbidden, text)
+        self.assertEqual(text.count("persist-credentials: false"), text.count("actions/checkout@"))
+        item = registry.workflow_entry(".github/workflows/manual-main-packages.yml")
+        self.assertEqual(item["name"], "Manual main packages")
+        self.assertEqual(item["events"], ["workflow_dispatch"])
+        self.assertEqual(item["schedule"], [])
+        self.assertEqual(item["monitor"], "none")
+        self.assertNotEqual(item["origin"], "baseline")
+
+    def test_registry_accepts_the_added_workflow(self):
+        self.assertTrue(registry.check_wiring(registry.load_registry()))
+
+
+class PackageToolTests(unittest.TestCase):
+    def test_zip_deb_and_rpm_wrap_the_same_binary_when_tools_exist(self):
+        required = ("dpkg-deb", "fakeroot", "rpm", "rpm2cpio", "cpio", "fpm", "zip", "unzip")
+        missing = [tool for tool in required if shutil.which(tool) is None]
+        if missing or os.name != "posix":
+            self.skipTest("packaging tools not installed: " + ", ".join(missing))
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp)
+            payload = b"manual main package fixture\n"
+            binary = root / "rustfs"
+            binary.write_bytes(payload)
+            result = run_script("package", "--version", "9.9.9-rc.8", "--arch", "x86_64", "--libc", "gnu",
+                                "--binary", str(binary), "--output", str(root / "dist"))
+            self.assertEqual(result.returncode, 0, result.stderr)
+            dist = root / "dist"
+            self.assertEqual(sorted(path.name for path in dist.iterdir()), [
+                "rustfs-linux-x86_64-gnu-v9.9.9-rc.8.deb",
+                "rustfs-linux-x86_64-gnu-v9.9.9-rc.8.rpm",
+                "rustfs-linux-x86_64-gnu-v9.9.9-rc.8.zip",
+            ])
+            self.assertIn("deb_version=9.9.9~rc.8", result.stdout)
+            self.assertIn("zip_file=rustfs-linux-x86_64-gnu-v9.9.9-rc.8.zip", result.stdout)
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/s3-tests/reference_parity.py

@@ -0,0 +1,329 @@
+#!/usr/bin/env python3
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Plan and judge the s3-tests reference parity lane.
+
+Subcommands:
+  targets    validate the target list and print it as a JSON matrix
+  export-env validate one target's extra environment and print KEY=VALUE lines
+  failed     print the failed test names of one run, in implemented_tests.txt form
+  compare    compare every candidate against the reference, per pytest node id
+
+`compare` reads <reports>/<target>/selected-nodeids.txt, junit.xml and the
+optional rerun-junit.xml of every target. Each node id is reduced to pass /
+fail / skip / missing; a case that failed and then passed its single rerun is a
+pass and is listed as flaky. A tolerated entry must match a real difference.
+
+Exit codes: 0 no untolerated difference and no stale entry; 1 a difference, a
+stale tolerated entry or an incomplete run; 2 missing or malformed input.
+"""
+
+import argparse
+import json
+from pathlib import Path
+import re
+import sys
+import tomllib
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import junit_diff  # noqa: E402  (sibling script, not a package)
+
+PASS, FAIL, SKIP, MISSING = junit_diff.PASS, junit_diff.FAIL, junit_diff.SKIP, junit_diff.MISSING
+NAME = re.compile(r"[a-z][a-z0-9-]{0,39}")
+ENV_KEY = re.compile(r"RUSTFS_[A-Z0-9_]+")
+ENV_VALUE = re.compile(r"[A-Za-z0-9._,:/=+-]*")
+ISSUE = re.compile(r"https://github\.com/rustfs/[A-Za-z0-9_.-]+/issues/[1-9][0-9]*")
+NODEID = re.compile(r"[A-Za-z0-9_/]+\.py(::[A-Za-z0-9_]+)+(\[[^\]\s]*\])?")
+TEST_FILE_PREFIX = "s3tests/functional/test_s3.py::"
+BINARIES = ("reference", "candidate")
+
+
+class InputError(Exception):
+    """Input that cannot be trusted; maps to exit code 2."""
+
+
+# --- target list -----------------------------------------------------------
+
+def load_targets(path):
+    try:
+        data = tomllib.loads(Path(path).read_text(encoding="utf-8"))
+    except FileNotFoundError as exc:
+        raise InputError(f"{path}: target list is missing") from exc
+    except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError) as exc:
+        raise InputError(f"{path}: cannot read target list: {exc}") from exc
+    if data.get("schema") != 1 or type(data.get("schema")) is not int or set(data) != {"schema", "targets"}:
+        raise InputError(f"{path}: unsupported target list schema")
+    return validate_targets(data["targets"], str(path))
+
+
+def validate_targets(targets, where="targets"):
+    if not isinstance(targets, list) or len(targets) < 2:
+        raise InputError(f"{where}: need one reference and at least one candidate")
+    result, names = [], set()
+    for item in targets:
+        if not isinstance(item, dict) or not set(item) <= {"name", "binary", "env"}:
+            raise InputError(f"{where}: invalid target entry {item!r}")
+        name, binary, env = item.get("name"), item.get("binary"), item.get("env", {})
+        if not isinstance(name, str) or not NAME.fullmatch(name) or name in names:
+            raise InputError(f"{where}: invalid or duplicate target name {name!r}")
+        if binary not in BINARIES:
+            raise InputError(f"{where}: target {name} has unknown binary {binary!r}")
+        validate_env(env, f"{where}: target {name}")
+        names.add(name)
+        result.append({"name": name, "binary": binary, "env": dict(env)})
+    if [t["binary"] for t in result].count("reference") != 1:
+        raise InputError(f"{where}: exactly one target must use the reference binary")
+    return result
+
+
+def validate_env(env, where):
+    if not isinstance(env, dict):
+        raise InputError(f"{where}: env must be a table")
+    for key, value in env.items():
+        if not isinstance(key, str) or not ENV_KEY.fullmatch(key):
+            raise InputError(f"{where}: env key {key!r} must match RUSTFS_[A-Z0-9_]+")
+        if not isinstance(value, str) or not ENV_VALUE.fullmatch(value):
+            raise InputError(f"{where}: env value for {key} has unsupported characters")
+    return env
+
+
+# --- node ids and reports --------------------------------------------------
+
+def nodeid_to_case(nodeid):
+    """Map a pytest node id to the classname::name id that junit_diff uses."""
+    if not NODEID.fullmatch(nodeid):
+        raise InputError(f"invalid pytest node id {nodeid!r}")
+    path, *parts = nodeid.split("::")
+    module = path[: -len(".py")].replace("/", ".")
+    classname = ".".join([module, *parts[:-1]])
+    return junit_diff.case_id(classname, parts[-1])
+
+
+def read_selected(path):
+    try:
+        lines = Path(path).read_text(encoding="utf-8").splitlines()
+    except FileNotFoundError as exc:
+        raise InputError(f"{path}: selected node id list is missing") from exc
+    except (OSError, UnicodeDecodeError) as exc:
+        raise InputError(f"{path}: cannot read selected node ids: {exc}") from exc
+    nodeids = [line.strip() for line in lines if line.strip()]
+    if not nodeids:
+        raise InputError(f"{path}: selected node id list is empty")
+    if len(set(nodeids)) != len(nodeids):
+        raise InputError(f"{path}: duplicate selected node id")
+    by_case = {}
+    for nodeid in nodeids:
+        case = nodeid_to_case(nodeid)
+        if case in by_case:
+            raise InputError(f"{path}: node ids {by_case[case]} and {nodeid} collide")
+        by_case[case] = nodeid
+    return nodeids, by_case
+
+
+def target_states(directory):
+    """Return (nodeids, {nodeid: state}, [flaky nodeids]) for one target run."""
+    directory = Path(directory)
+    if not directory.is_dir():
+        raise InputError(f"{directory}: report directory is missing")
+    nodeids, by_case = read_selected(directory / "selected-nodeids.txt")
+    try:
+        first, _ = junit_diff.parse_report(str(directory / "junit.xml"))
+    except junit_diff.InputError as exc:
+        raise InputError(str(exc)) from exc
+    unexpected = sorted(set(first) - set(by_case))
+    if unexpected:
+        raise InputError(f"{directory}: junit.xml has unselected cases: {', '.join(unexpected[:5])}")
+    states = {by_case[case]: state for case, state in first.items()}
+    flaky = []
+    rerun_path = directory / "rerun-junit.xml"
+    if rerun_path.exists():
+        try:
+            rerun, _ = junit_diff.parse_report(str(rerun_path))
+        except junit_diff.InputError as exc:
+            raise InputError(str(exc)) from exc
+        for case, state in sorted(rerun.items()):
+            nodeid = by_case.get(case)
+            if nodeid is None or states.get(nodeid) != FAIL:
+                raise InputError(f"{rerun_path}: {case} was rerun without failing first")
+            if state == PASS:
+                flaky.append(nodeid)
+            states[nodeid] = PASS if state == PASS else FAIL
+    for nodeid in nodeids:
+        states.setdefault(nodeid, MISSING)
+    return nodeids, states, flaky
+
+
+def failed_names(directory):
+    _, states, _ = target_states(directory)
+    names = []
+    for nodeid, state in states.items():
+        if state == FAIL:
+            if not nodeid.startswith(TEST_FILE_PREFIX):
+                raise InputError(f"cannot express {nodeid} as an implemented_tests.txt entry")
+            names.append(nodeid[len(TEST_FILE_PREFIX):])
+    return sorted(names)
+
+
+# --- tolerated differences ---------------------------------------------------
+
+def read_tolerated(path, candidates):
+    try:
+        lines = Path(path).read_text(encoding="utf-8").splitlines()
+    except FileNotFoundError as exc:
+        raise InputError(f"{path}: tolerated-difference file is missing") from exc
+    except (OSError, UnicodeDecodeError) as exc:
+        raise InputError(f"{path}: cannot read tolerated differences: {exc}") from exc
+    entries = {}
+    for number, line in enumerate(lines, 1):
+        if not line.strip() or line.lstrip().startswith("#"):
+            continue
+        fields = [part.strip() for part in line.split(" | ")]
+        if len(fields) != 4 or not all(fields):
+            raise InputError(f"{path}:{number}: expected '<target> | <node id> | <issue URL> | <reason>'")
+        target, nodeid, issue, _reason = fields
+        if target not in candidates:
+            raise InputError(f"{path}:{number}: {target!r} is not a candidate target")
+        nodeid_to_case(nodeid)
+        if not ISSUE.fullmatch(issue):
+            raise InputError(f"{path}:{number}: tracking issue must be a https://github.com/rustfs/<repo>/issues/<n> URL")
+        if (target, nodeid) in entries:
+            raise InputError(f"{path}:{number}: duplicate entry for {target} {nodeid}")
+        entries[(target, nodeid)] = issue
+    return entries
+
+
+# --- comparison ----------------------------------------------------------------
+
+def count(states):
+    result = {PASS: 0, FAIL: 0, SKIP: 0, MISSING: 0}
+    for state in states.values():
+        result[state] += 1
+    return result
+
+
+def compare(targets, reports, tolerated_path):
+    reference = next(t["name"] for t in targets if t["binary"] == "reference")
+    candidates = [t["name"] for t in targets if t["binary"] == "candidate"]
+    tolerated = read_tolerated(tolerated_path, set(candidates))
+    runs = {t["name"]: target_states(Path(reports) / t["name"]) for t in targets}
+    selected = runs[reference][0]
+    for name, (nodeids, _, _) in runs.items():
+        if nodeids != selected:
+            raise InputError(f"{name}: selected node ids differ from the {reference} run")
+    ref_states = runs[reference][1]
+    differences = []
+    for name in candidates:
+        states = runs[name][1]
+        for nodeid in selected:
+            if ref_states[nodeid] != states[nodeid]:
+                issue = tolerated.get((name, nodeid))
+                differences.append({"target": name, "nodeid": nodeid, "reference": ref_states[nodeid],
+                                    "candidate": states[nodeid], "tolerated": issue})
+    seen = {(d["target"], d["nodeid"]) for d in differences}
+    stale = [{"target": t, "nodeid": n, "issue": i} for (t, n), i in sorted(tolerated.items()) if (t, n) not in seen]
+    incomplete = {name: [n for n in selected if run[1][n] == MISSING] for name, run in runs.items()}
+    return {
+        "reference": reference,
+        "candidates": candidates,
+        "selected": len(selected),
+        "counts": {name: count(run[1]) for name, run in runs.items()},
+        "flaky": {name: run[2] for name, run in runs.items()},
+        "differences": differences,
+        "stale_tolerated": stale,
+        "incomplete": {name: ids for name, ids in incomplete.items() if ids},
+    }
+
+
+def failed(summary):
+    return bool(summary["stale_tolerated"] or summary["incomplete"]
+                or any(not d["tolerated"] for d in summary["differences"]))
+
+
+def render(summary):
+    out = ["## s3-tests reference parity", ""]
+    out.append(f"Reference target `{summary['reference']}`; {summary['selected']} selected node ids per target.")
+    out += ["", "| Target | pass | fail | skip | missing | flaky (passed on rerun) |", "|---|---:|---:|---:|---:|---:|"]
+    for name, c in summary["counts"].items():
+        out.append(f"| `{name}` | {c[PASS]} | {c[FAIL]} | {c[SKIP]} | {c[MISSING]} | {len(summary['flaky'][name])} |")
+    diffs = summary["differences"]
+    out += ["", f"### Differences ({len(diffs)})", ""]
+    if diffs:
+        out += ["| Target | Node id | reference | candidate | Tolerated |", "|---|---|---|---|---|"]
+        for d in diffs:
+            out.append(f"| `{d['target']}` | `{d['nodeid']}` | {d['reference']} | {d['candidate']} | {d['tolerated'] or 'no'} |")
+    else:
+        out.append("No differences.")
+    if summary["stale_tolerated"]:
+        out += ["", "### Stale tolerated entries (no matching difference; remove them)", ""]
+        out += [f"- `{s['target']}` `{s['nodeid']}` ({s['issue']})" for s in summary["stale_tolerated"]]
+    if summary["incomplete"]:
+        out += ["", "### Incomplete runs (selected node ids without a result)", ""]
+        out += [f"- `{name}`: {len(ids)} missing, first `{ids[0]}`" for name, ids in summary["incomplete"].items()]
+    flaky = [(name, ids) for name, ids in summary["flaky"].items() if ids]
+    if flaky:
+        out += ["", "### Flaky (failed, then passed the single rerun; not a difference)", ""]
+        out += [f"- `{name}`: " + ", ".join(f"`{i}`" for i in ids) for name, ids in flaky]
+    out += ["", "Result: **" + ("FAIL" if failed(summary) else "PASS") + "**", ""]
+    return "\n".join(out)
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    sub = parser.add_subparsers(dest="mode", required=True)
+    tgt = sub.add_parser("targets")
+    tgt.add_argument("--config", required=True)
+    env = sub.add_parser("export-env")
+    env.add_argument("--target-json", required=True)
+    fail = sub.add_parser("failed")
+    fail.add_argument("--report", required=True, help="one target's report directory")
+    cmp_ = sub.add_parser("compare")
+    cmp_.add_argument("--targets-json", required=True)
+    cmp_.add_argument("--reports", required=True)
+    cmp_.add_argument("--tolerated", required=True)
+    cmp_.add_argument("--markdown", help="write the summary table here (appended)")
+    cmp_.add_argument("--json", help="write the machine-readable summary here")
+    args = parser.parse_args(argv)
+    try:
+        if args.mode == "targets":
+            print(json.dumps(load_targets(args.config), separators=(",", ":")))
+        elif args.mode == "export-env":
+            target = json.loads(args.target_json)
+            if not isinstance(target, dict):
+                raise InputError("target must be a JSON object")
+            for key, value in sorted(validate_env(target.get("env", {}), "target").items()):
+                print(f"{key}={value}")
+        elif args.mode == "failed":
+            for name in failed_names(args.report):
+                print(name)
+        else:
+            targets = validate_targets(json.loads(args.targets_json))
+            summary = compare(targets, args.reports, args.tolerated)
+            text = render(summary)
+            print(text)
+            if args.markdown:
+                with open(args.markdown, "a", encoding="utf-8") as handle:
+                    handle.write(text + "\n")
+            if args.json:
+                with open(args.json, "w", encoding="utf-8") as handle:
+                    json.dump(summary, handle, indent=2, sort_keys=True)
+                    handle.write("\n")
+            return 1 if failed(summary) else 0
+    except (InputError, json.JSONDecodeError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/s3-tests/test_reference_parity.py

@@ -0,0 +1,332 @@
+#!/usr/bin/env python3
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contract tests for reference_parity.py: target list, rerun merge, tolerated entries and exit codes."""
+
+import json
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+
+SCRIPT = Path(__file__).resolve().parent / "reference_parity.py"
+REPO = Path(__file__).resolve().parents[2]
+PREFIX = "s3tests/functional/test_s3.py::"
+CLASSNAME = "s3tests.functional.test_s3"
+TARGETS = [{"name": "reference", "binary": "reference", "env": {}},
+           {"name": "candidate-legacy", "binary": "candidate", "env": {}}]
+ISSUE = "https://github.com/rustfs/rustfs/issues/1"
+
+
+def junit(states):
+    cases = []
+    for name, state in states.items():
+        body = {"pass": "", "fail": "<failure message='x'/>", "error": "<error message='x'/>",
+                "skip": "<skipped/>"}[state]
+        cases.append(f'<testcase classname="{CLASSNAME}" name="{name}">{body}</testcase>')
+    return f'<testsuites><testsuite name="pytest">{"".join(cases)}</testsuite></testsuites>'
+
+
+def run(*args):
+    proc = subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True, check=False)
+    return proc.returncode, proc.stdout, proc.stderr
+
+
+class Lane:
+    """A temporary reports directory with one sub-directory per target."""
+
+    def __init__(self, case):
+        self.temp = tempfile.TemporaryDirectory()
+        case.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.reports = self.root / "reports"
+        self.tolerated = self.root / "tolerated.txt"
+        self.tolerated.write_text("# none\n")
+
+    def target(self, name, states, selected=None, rerun=None):
+        directory = self.reports / name
+        directory.mkdir(parents=True)
+        names = selected if selected is not None else list(states)
+        (directory / "selected-nodeids.txt").write_text("".join(PREFIX + n + "\n" for n in names))
+        (directory / "junit.xml").write_text(junit(states))
+        if rerun is not None:
+            (directory / "rerun-junit.xml").write_text(junit(rerun))
+        return directory
+
+    def compare(self, targets=None):
+        summary = self.root / "summary.json"
+        code, out, err = run("compare", "--targets-json", json.dumps(targets or TARGETS),
+                             "--reports", str(self.reports), "--tolerated", str(self.tolerated),
+                             "--json", str(summary))
+        data = json.loads(summary.read_text()) if summary.exists() else None
+        return code, out, err, data
+
+
+BASE = {"test_a": "pass", "test_b": "pass", "test_c": "skip"}
+
+
+class CompareTests(unittest.TestCase):
+    # --- negative cases ---
+
+    def test_pass_versus_fail_is_a_difference(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
+        code, out, _, data = lane.compare()
+        self.assertEqual(code, 1)
+        self.assertEqual(data["differences"], [{"target": "candidate-legacy", "nodeid": PREFIX + "test_a",
+                                                "reference": "pass", "candidate": "fail", "tolerated": None}])
+        self.assertIn("Result: **FAIL**", out)
+
+    def test_error_counts_as_fail(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_b": "error"})
+        code, _, _, data = lane.compare()
+        self.assertEqual(code, 1)
+        self.assertEqual(data["differences"][0]["candidate"], "fail")
+
+    def test_reference_fail_candidate_pass_is_also_a_difference(self):
+        lane = Lane(self)
+        lane.target("reference", {**BASE, "test_a": "fail"})
+        lane.target("candidate-legacy", BASE)
+        code, _, _, data = lane.compare()
+        self.assertEqual(code, 1)
+        self.assertEqual(data["differences"][0]["reference"], "fail")
+
+    def test_skip_versus_pass_is_a_difference(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_c": "pass"})
+        code, _, _, data = lane.compare()
+        self.assertEqual(code, 1)
+        self.assertEqual(data["differences"][0]["nodeid"], PREFIX + "test_c")
+
+    def test_case_absent_from_candidate_report_is_missing_and_incomplete(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {"test_a": "pass", "test_c": "skip"}, selected=list(BASE))
+        code, out, _, data = lane.compare()
+        self.assertEqual(code, 1)
+        self.assertEqual(data["differences"][0]["candidate"], "missing")
+        self.assertEqual(data["incomplete"], {"candidate-legacy": [PREFIX + "test_b"]})
+        self.assertIn("Incomplete runs", out)
+
+    def test_case_missing_from_both_reports_still_fails(self):
+        lane = Lane(self)
+        partial = {"test_a": "pass", "test_c": "skip"}
+        lane.target("reference", partial, selected=list(BASE))
+        lane.target("candidate-legacy", partial, selected=list(BASE))
+        code, _, _, data = lane.compare()
+        self.assertEqual(code, 1)
+        self.assertEqual(data["differences"], [])
+        self.assertEqual(set(data["incomplete"]), {"reference", "candidate-legacy"})
+
+    def test_failure_that_also_fails_the_rerun_stays_a_difference(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_a": "fail"}, rerun={"test_a": "fail"})
+        code, _, _, data = lane.compare()
+        self.assertEqual(code, 1)
+        self.assertEqual(data["flaky"]["candidate-legacy"], [])
+
+    def test_rerun_of_a_case_that_did_not_fail_is_rejected(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_a": "fail"}, rerun={"test_b": "pass"})
+        code, _, err, _ = lane.compare()
+        self.assertEqual(code, 2)
+        self.assertIn("rerun without failing first", err)
+
+    def test_missing_target_report_fails_rather_than_skips(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        code, _, err, data = lane.compare()
+        self.assertEqual(code, 2)
+        self.assertIsNone(data)
+        self.assertIn("candidate-legacy: report directory is missing", err)
+
+    def test_missing_junit_fails(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        (lane.target("candidate-legacy", BASE) / "junit.xml").unlink()
+        code, _, err, _ = lane.compare()
+        self.assertEqual(code, 2)
+        self.assertIn("cannot read", err)
+
+    def test_different_selected_lists_are_rejected(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {"test_a": "pass", "test_b": "pass"})
+        code, _, err, _ = lane.compare()
+        self.assertEqual(code, 2)
+        self.assertIn("selected node ids differ", err)
+
+    def test_unselected_case_in_junit_is_rejected(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_extra": "pass"}, selected=list(BASE))
+        code, _, err, _ = lane.compare()
+        self.assertEqual(code, 2)
+        self.assertIn("unselected cases", err)
+
+    def test_stale_tolerated_entry_fails(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", BASE)
+        lane.tolerated.write_text(f"candidate-legacy | {PREFIX}test_a | {ISSUE} | known drift\n")
+        code, out, _, data = lane.compare()
+        self.assertEqual(code, 1)
+        self.assertEqual(data["stale_tolerated"][0]["nodeid"], PREFIX + "test_a")
+        self.assertIn("Stale tolerated entries", out)
+
+    def test_tolerated_entry_for_another_target_does_not_cover_the_difference(self):
+        lane = Lane(self)
+        targets = TARGETS + [{"name": "candidate-gateway", "binary": "candidate", "env": {}}]
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
+        lane.target("candidate-gateway", {**BASE, "test_a": "fail"})
+        lane.tolerated.write_text(f"candidate-gateway | {PREFIX}test_a | {ISSUE} | gateway gap\n")
+        code, _, _, data = lane.compare(targets)
+        self.assertEqual(code, 1)
+        untolerated = [d["target"] for d in data["differences"] if not d["tolerated"]]
+        self.assertEqual(untolerated, ["candidate-legacy"])
+
+    def test_tolerated_entry_without_issue_url_is_rejected(self):
+        for line in (f"candidate-legacy | {PREFIX}test_a | rustfs#1 | reason",
+                     f"candidate-legacy | {PREFIX}test_a | {ISSUE} | ",
+                     f"candidate-legacy | {PREFIX}test_a | {ISSUE}",
+                     f"reference | {PREFIX}test_a | {ISSUE} | reason",
+                     f"candidate-legacy | test_a | {ISSUE} | reason"):
+            with self.subTest(line=line):
+                lane = Lane(self)
+                lane.target("reference", BASE)
+                lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
+                lane.tolerated.write_text(line + "\n")
+                code, _, err, _ = lane.compare()
+                self.assertEqual(code, 2, err)
+
+    def test_duplicate_tolerated_entry_is_rejected(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
+        entry = f"candidate-legacy | {PREFIX}test_a | {ISSUE} | reason\n"
+        lane.tolerated.write_text(entry * 2)
+        code, _, err, _ = lane.compare()
+        self.assertEqual(code, 2)
+        self.assertIn("duplicate entry", err)
+
+    def test_missing_tolerated_file_fails(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", BASE)
+        lane.tolerated.unlink()
+        code, _, err, _ = lane.compare()
+        self.assertEqual(code, 2)
+        self.assertIn("tolerated-difference file is missing", err)
+
+    # --- positive cases ---
+
+    def test_identical_runs_pass(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", BASE)
+        code, out, _, data = lane.compare()
+        self.assertEqual(code, 0)
+        self.assertEqual(data["counts"]["candidate-legacy"], {"pass": 2, "fail": 0, "skip": 1, "missing": 0})
+        self.assertIn("No differences.", out)
+        self.assertIn("Result: **PASS**", out)
+
+    def test_failure_that_passes_the_rerun_is_flaky_not_a_difference(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_a": "fail"}, rerun={"test_a": "pass"})
+        code, out, _, data = lane.compare()
+        self.assertEqual(code, 0)
+        self.assertEqual(data["flaky"]["candidate-legacy"], [PREFIX + "test_a"])
+        self.assertIn("Flaky", out)
+
+    def test_tolerated_difference_passes_and_is_labelled(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
+        lane.tolerated.write_text(f"# comment\ncandidate-legacy | {PREFIX}test_a | {ISSUE} | known drift\n")
+        code, out, _, data = lane.compare()
+        self.assertEqual(code, 0)
+        self.assertEqual(data["differences"][0]["tolerated"], ISSUE)
+        self.assertIn(ISSUE, out)
+
+
+class TargetTests(unittest.TestCase):
+    def write(self, text):
+        temp = tempfile.TemporaryDirectory()
+        self.addCleanup(temp.cleanup)
+        path = Path(temp.name) / "targets.toml"
+        path.write_text(text)
+        return path
+
+    def test_invalid_target_lists_are_rejected(self):
+        ref = '[[targets]]\nname = "reference"\nbinary = "reference"\n'
+        cand = '[[targets]]\nname = "candidate-legacy"\nbinary = "candidate"\n'
+        for label, text in {
+            "no schema": ref + cand,
+            "only reference": "schema = 1\n" + ref,
+            "two references": "schema = 1\n" + ref + ref.replace('"reference"\nbinary', '"other"\nbinary'),
+            "no reference": "schema = 1\n" + cand + cand.replace("candidate-legacy", "candidate-two"),
+            "duplicate name": "schema = 1\n" + ref + cand + cand,
+            "unknown binary": "schema = 1\n" + ref + cand.replace('binary = "candidate"', 'binary = "main"'),
+            "bad name": "schema = 1\n" + ref + cand.replace("candidate-legacy", "Candidate Legacy"),
+            "foreign env key": "schema = 1\n" + ref + cand + 'env = { PATH = "/tmp" }\n',
+            "env injection": "schema = 1\n" + ref + cand + 'env = { RUSTFS_X = "a\\nB=c" }\n',
+            "unknown key": "schema = 1\n" + ref + cand + 'stack = "gateway"\n',
+        }.items():
+            with self.subTest(label):
+                code, _, _ = run("targets", "--config", str(self.write(text)))
+                self.assertEqual(code, 2)
+
+    def test_missing_target_file_fails(self):
+        code, _, err = run("targets", "--config", "/nonexistent/targets.toml")
+        self.assertEqual(code, 2)
+        self.assertIn("target list is missing", err)
+
+    def test_export_env_rejects_foreign_keys(self):
+        code, _, _ = run("export-env", "--target-json", json.dumps({"env": {"LD_PRELOAD": "x"}}))
+        self.assertEqual(code, 2)
+
+    def test_repository_target_list_is_valid(self):
+        code, out, err = run("targets", "--config", str(REPO / ".config/reference-parity-targets.toml"))
+        self.assertEqual(code, 0, err)
+        self.assertEqual(json.loads(out), TARGETS)
+
+    def test_export_env_prints_validated_lines(self):
+        code, out, _ = run("export-env", "--target-json", json.dumps({"env": {"RUSTFS_S3_STACK": "gateway"}}))
+        self.assertEqual((code, out), (0, "RUSTFS_S3_STACK=gateway\n"))
+
+
+class FailedTests(unittest.TestCase):
+    def test_failed_lists_only_failures_in_test_list_form(self):
+        lane = Lane(self)
+        directory = lane.target("reference", {**BASE, "test_a": "fail", "test_d": "error"})
+        code, out, _ = run("failed", "--report", str(directory))
+        self.assertEqual((code, out), (0, "test_a\ntest_d\n"))
+
+    def test_failed_rejects_a_missing_report(self):
+        code, _, err = run("failed", "--report", "/nonexistent/report")
+        self.assertEqual(code, 2)
+        self.assertIn("report directory is missing", err)
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/test_reference_binary.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Exercise the reference-binary pin resolver and unpacker without network access."""
+
+import hashlib
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+import zipfile
+
+ROOT = Path(__file__).resolve().parents[1]
+SCRIPT = ROOT / ".github/actions/reference-binary/reference_binary.py"
+PIN = '''schema = 1
+repository = "rustfs/rustfs"
+version = "1.0.5"
+tag = "1.0.5"
+
+[platforms.linux-x86_64-gnu]
+asset = "rustfs-linux-x86_64-gnu-v1.0.5.zip"
+sha256 = "{sha}"
+executable = "rustfs"
+'''
+SHA = "a" * 64
+
+
+def run(*args):
+    proc = subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True, check=False)
+    return proc.returncode, proc.stdout, proc.stderr
+
+
+class Base(unittest.TestCase):
+    def setUp(self):
+        temp = tempfile.TemporaryDirectory()
+        self.addCleanup(temp.cleanup)
+        self.root = Path(temp.name)
+
+    def pin(self, text):
+        path = self.root / "pin.toml"
+        path.write_text(text)
+        return path
+
+    def archive(self, members):
+        path = self.root / "rustfs.zip"
+        with zipfile.ZipFile(path, "w") as bundle:
+            for name, data in members.items():
+                bundle.writestr(name, data)
+        return path, hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+class ResolveTests(Base):
+    def test_missing_pin_file_fails(self):
+        code, _, err = run("resolve", "--pin", str(self.root / "absent.toml"), "--platform", "linux-x86_64-gnu")
+        self.assertEqual(code, 1)
+        self.assertIn("reference pin file is missing", err)
+
+    def test_unpinned_platform_fails(self):
+        code, _, err = run("resolve", "--pin", str(self.pin(PIN.format(sha=SHA))), "--platform", "linux-aarch64-gnu")
+        self.assertEqual(code, 1)
+        self.assertIn("no pin for platform", err)
+
+    def test_malformed_pins_fail(self):
+        good = PIN.format(sha=SHA)
+        for label, text in {
+            "short digest": good.replace(SHA, "a" * 63),
+            "uppercase digest": good.replace(SHA, "A" * 64),
+            "no digest": good.replace(f'sha256 = "{SHA}"\n', ""),
+            "path in asset": good.replace('asset = "rustfs-', 'asset = "../rustfs-'),
+            "non-zip asset": good.replace("v1.0.5.zip", "v1.0.5.deb"),
+            "asset of another version": good.replace("v1.0.5.zip", "v1.0.4.zip"),
+            "path in executable": good.replace('executable = "rustfs"', 'executable = "bin/rustfs"'),
+            "wrong schema": good.replace("schema = 1", "schema = 2"),
+            "boolean schema": good.replace("schema = 1", "schema = true"),
+            "no repository": good.replace('repository = "rustfs/rustfs"\n', ""),
+            "unknown key": good + 'url = "https://example.invalid"\n',
+            "not toml": "schema = \n",
+        }.items():
+            with self.subTest(label):
+                code, _, _ = run("resolve", "--pin", str(self.pin(text)), "--platform", "linux-x86_64-gnu")
+                self.assertEqual(code, 1)
+
+    def test_resolves_identity_as_step_outputs(self):
+        code, out, err = run("resolve", "--pin", str(self.pin(PIN.format(sha=SHA))), "--platform", "linux-x86_64-gnu")
+        self.assertEqual(code, 0, err)
+        self.assertEqual(out.splitlines(), ["repository=rustfs/rustfs", "version=1.0.5", "tag=1.0.5",
+                                            "asset=rustfs-linux-x86_64-gnu-v1.0.5.zip", f"sha256={SHA}",
+                                            "executable=rustfs"])
+
+    def test_repository_pin_is_valid(self):
+        code, out, err = run("resolve", "--pin", str(ROOT / ".config/reference-binary.toml"),
+                             "--platform", "linux-x86_64-gnu")
+        self.assertEqual(code, 0, err)
+        self.assertIn("version=1.0.5", out.splitlines())
+
+
+class UnpackTests(Base):
+    def unpack(self, archive, sha):
+        return run("unpack", "--archive", str(archive), "--sha256", sha, "--executable", "rustfs",
+                   "--destination", str(self.root / "out"))
+
+    def test_digest_mismatch_fails_and_extracts_nothing(self):
+        archive, _ = self.archive({"rustfs": b"binary"})
+        code, _, err = self.unpack(archive, SHA)
+        self.assertEqual(code, 1)
+        self.assertIn("does not match the pinned digest", err)
+        self.assertFalse((self.root / "out").exists())
+
+    def test_missing_archive_fails(self):
+        code, _, err = self.unpack(self.root / "absent.zip", SHA)
+        self.assertEqual(code, 1)
+        self.assertIn("missing or unreadable", err)
+
+    def test_archive_without_top_level_executable_fails(self):
+        archive, sha = self.archive({"bin/rustfs": b"binary"})
+        code, _, err = self.unpack(archive, sha)
+        self.assertEqual(code, 1)
+        self.assertIn("exactly one top-level", err)
+
+    def test_empty_executable_fails(self):
+        archive, sha = self.archive({"rustfs": b""})
+        code, _, _ = self.unpack(archive, sha)
+        self.assertEqual(code, 1)
+
+    def test_non_zip_archive_fails(self):
+        archive = self.root / "rustfs.zip"
+        archive.write_bytes(b"not a zip")
+        code, _, err = self.unpack(archive, hashlib.sha256(b"not a zip").hexdigest())
+        self.assertEqual(code, 1)
+        self.assertIn("not a valid zip", err)
+
+    def test_extracts_only_the_executable(self):
+        archive, sha = self.archive({"rustfs": b"binary", "../escape": b"x", "README": b"r"})
+        code, out, err = self.unpack(archive, sha)
+        self.assertEqual(code, 0, err)
+        target = self.root / "out" / "rustfs"
+        self.assertEqual(out.strip(), f"path={target}")
+        self.assertEqual(target.read_bytes(), b"binary")
+        self.assertTrue(target.stat().st_mode & 0o111)
+        self.assertEqual(sorted(p.name for p in (self.root / "out").iterdir()), ["rustfs"])
+        self.assertFalse((self.root / "escape").exists())
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/test_release_packages.py

@@ -11,6 +11,7 @@ import subprocess
 import sys
 import tempfile
 import unittest
+import zipfile
 from unittest.mock import patch
 
 sys.path.insert(0, str(Path(__file__).resolve().parent / "release"))
@@ -63,6 +64,57 @@ class PackageIntegrationTests(unittest.TestCase):
                     deliver.package(fixtures.inputs(), "linux-x86_64-musl", root / "output")
             self.assertFalse(list((root / "output").glob("packages-*.json")))
 
+    def test_manual_packages_match_release_names_and_bytes(self):
+        script = Path(__file__).resolve().parent / "release" / "package_manual_artifacts.sh"
+        cases = (
+            ("9.9.9-rc.8", "x86_64", "gnu", True),
+            ("9.9.9", "aarch64", "musl", False),
+        )
+        for version, arch, libc, with_catalog in cases:
+            with self.subTest(version=version, arch=arch, libc=libc), tempfile.TemporaryDirectory() as temp:
+                root = Path(temp)
+                payload = f"manual-package:{version}:{arch}:{libc}\n".encode()
+                binary = root / "rustfs"
+                binary.write_bytes(payload)
+                command = ["bash", str(script), "package", "--version", version, "--arch", arch, "--libc", libc,
+                           "--binary", str(binary), "--output", str(root / "dist")]
+                if with_catalog:
+                    for name in ("rustfs.cpu-symbol-catalog.json", "rustfs.cpu-symbol-catalog.sha256"):
+                        (root / name).write_text(name + "\n", encoding="utf-8")
+                        command.extend(["--zip-extra", str(root / name)])
+                subprocess.run(command, check=True, text=True, timeout=180)
+                stem = f"rustfs-linux-{arch}-{libc}-v{version}"
+                dist = root / "dist"
+                self.assertEqual(sorted(path.name for path in dist.iterdir()),
+                                 [f"{stem}.deb", f"{stem}.rpm", f"{stem}.zip"])
+                expected_hash = hashlib.sha256(payload).hexdigest()
+                with zipfile.ZipFile(dist / f"{stem}.zip") as archive:
+                    names = sorted(archive.namelist())
+                    self.assertEqual(hashlib.sha256(archive.read("rustfs")).hexdigest(), expected_hash)
+                if with_catalog:
+                    self.assertEqual(names, ["rustfs", "rustfs.cpu-symbol-catalog.json", "rustfs.cpu-symbol-catalog.sha256"])
+                else:
+                    self.assertEqual(names, ["rustfs"])
+                deb = dist / f"{stem}.deb"
+                self.assertEqual(subprocess.check_output(["dpkg-deb", "-f", deb, "Version"], text=True).strip(),
+                                 version.replace("-", "~"))
+                self.assertEqual(subprocess.check_output(["dpkg-deb", "-f", deb, "Architecture"], text=True).strip(),
+                                 "amd64" if arch == "x86_64" else "arm64")
+                depends = subprocess.check_output(["dpkg-deb", "-f", deb, "Depends"], text=True).strip()
+                self.assertEqual(depends, "libc6 (>= 2.31)" if libc == "gnu" else "")
+                unpacked = root / "deb"
+                subprocess.run(["dpkg-deb", "--extract", str(deb), str(unpacked)], check=True)
+                self.assertEqual(hashlib.sha256((unpacked / "usr/bin/rustfs").read_bytes()).hexdigest(), expected_hash)
+                rpm = dist / f"{stem}.rpm"
+                identity = subprocess.check_output(
+                    ["rpm", "-qp", "--qf", "%{NAME} %{VERSION} %{RELEASE} %{ARCH}", str(rpm)], text=True)
+                self.assertEqual(identity, f"rustfs {version.replace('-', '~')} 1 {arch}")
+                requires = subprocess.check_output(["rpm", "-qp", "--requires", str(rpm)], text=True)
+                if libc == "gnu":
+                    self.assertIn("glibc >= 2.31", requires)
+                else:
+                    self.assertNotIn("glibc", requires)
+
 
 if __name__ == "__main__":
     unittest.main()