ci: allow macOS release builds to finish (#6976)

Co-authored-by: heihutu <heihutu@gmail.com>

.agents/skills/security-advisory-lessons/references/advisory-patterns.md

@@ -48,6 +48,7 @@ Update this file only when an advisory adds or changes a reusable lesson, affect
 
 ### S3 object actions, copy, multipart, and upload policy validation
 
+- `GHSA-g8w9-qw9q-fghr`: a valid presigned `PutObject` accepted extra `x-amz-tagging`, website redirect, and storage-class headers omitted from `SignedHeaders`. Lesson: a presigned URL is a bounded capability; reject `x-amz-*` headers that are not cryptographically bound by the signature so unsigned metadata cannot change authorization, lifecycle, redirect, cost, or durability semantics.
 - `GHSA-3ppv-fx5m-m749`: explicit `versionId` reads and copy sources authorized `s3:GetObject` instead of `s3:GetObjectVersion`. Lesson: version-specific object access must select version-specific actions for direct reads, `CopyObject`, and `UploadPartCopy`, with tests proving the backend is not reached on denial.
 - `GHSA-x298-9x87-fvjq`: anonymous `ListObjectVersions` fell back to `ListBucket` and returned before public-access-block gates. Lesson: compatibility fallbacks must converge on the same post-authorization checks as direct grants, especially `RestrictPublicBuckets` and anonymous data-plane denies.
 - `GHSA-mx42-j6wv-px98`: `UploadPartCopy` missed source authorization and allowed cross-bucket object exfiltration. Lesson: multipart copy must enforce the same source and destination contract as `CopyObject`.
@@ -119,7 +120,7 @@ Use these targeted searches when a diff touches security-sensitive code:
 ```bash
 rg -n "validate_admin_request|check_permissions|AdminAction::|deny_only|is_allowed" rustfs crates
 rg -n "authorize_operation|FtpsDriver|SftpDriver|RETR|MKD|SIZE|MDTM|CreateBucket|GetObject|HeadObject" crates/protocols rustfs
-rg -n "UploadPartCopy|upload_part_copy|CompleteMultipart|PostObject|content-length-range|starts-with" rustfs crates
+rg -n "UploadPartCopy|upload_part_copy|CompleteMultipart|PostObject|presign|SignedHeaders|content-length-range|starts-with" rustfs crates
 rg -n "ListBucketVersions|GetObjectVersion|versionId|VersionId|ExistingObjectTag|ForAllValues|ForAnyValue|POLICY_PLUGIN|opa" rustfs crates
 rg -n "normalize_extract_entry_key|Snowball|auto-extract|PathBuf::join|canonicalize|\\.\\.|x-forwarded-for|x-real-ip|SourceIp" rustfs crates
 rg -n "DEFAULT_SECRET|DEFAULT_ACCESS|TEST_PRIVATE_KEY|rustfs rpc|RUSTFS_RPC_SECRET" rustfs crates
@@ -136,6 +137,7 @@ rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates
 - Protocol frontend authz fixes: include denied `RETR`, `SIZE`/`MDTM`, `MKD`, bucket probe, and sibling allowed-operation cases, and assert denied paths do not reach the storage backend.
 - IAM fixes: include import/update/list service-account cases with attacker-controlled parent, claims, access key, secret key, and policy.
 - Copy/upload fixes: include cross-bucket, cross-user, source-denied, destination-denied, copy-source-condition, and multipart completion cases.
+- Presigned upload fixes: include a valid presign with extra unsigned tagging, redirect, and storage-class headers; require rejection before storage access, and verify explicitly signed equivalents still work.
 - Version-action fixes: include historical UUID, explicit current version, `null`, range, partNumber, presigned, STS/session, service-account, anonymous bucket-policy, copy source, and multipart-copy source cases.
 - Policy-condition fixes: include reserved-key header collisions, missing keys, partially overlapping multi-value sets, plugin mode, and built-in policy mode.
 - Path fixes: include encoded traversal, absolute path, nested traversal, archive entries with `..`, valid object keys that resemble traversal text but should be rejected, and canonical bucket/prefix boundary checks.

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
 sha256-darwin=d6aa36cfaae2c4d8590482c7e47138c5965b335b34a75f50d11ffc3366e9021e
-sha256-linux=c8315465f50c194faee36141cdbb1e15e59271e524d948564a69e2d5eb408f2a
+sha256-linux=e3eb4ab7fc72224abf58c546ac0706d6605d3bd26bac7d8ce338829fd3daecc2

.config/e2e-nightly-selection.txt

@@ -1 +1 @@
-sha256=9b9bc336b43b70d0e06e0adb5455bf035bb18945d85d60936eb6fe4d48e0e680
+sha256=9c2b958035a038ffd5ab98cac5f59a1b8e6a16e141f109ec7fb956afc0f11105

.config/e2e-repl-nightly-selection.txt

@@ -1 +1 @@
-sha256=655a3f3c1d042e694339d15caba7580518320322d1bac0f09450b37e6c09e2e7
+sha256=8d5517f5f2fc32d561782dfccd51b7f746f5e25b2835e37e100c883f7f18777d

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=294350518743cac8d7c41880a2835216e4b697908d7b0b1bc92b62816d94c59d
+sha256=dbebfbab9b9efd4eff31211e69dd32235dc00e207f2ab0dd919a1b2ac9e724c2

.config/nextest.toml

@@ -46,6 +46,11 @@ e2e-reliability = { max-threads = 1 }
 e2e-inline-boundaries = { max-threads = 1 }
 e2e-cluster-nightly = { max-threads = 1 }
 
+# Deep async storage futures are composed into tests across several crates.
+# Keep the test stack bounded but above libtest's 2 MiB default.
+[scripts.setup.ecstore-base-stack]
+command = ['sh', '-c', 'echo RUST_MIN_STACK=4194304 >> "$NEXTEST_ENV"']
+
 # These exact regression scenarios build deep async storage futures that exceed
 # libtest's 2 MiB spawned-thread stack on Linux. Give only their test processes
 # the same 32 MiB stack already used by the crate's dedicated large-stack tests.
@@ -63,6 +68,10 @@ command = ['sh', '-c', 'echo RUST_MIN_STACK=33554432 >> "$NEXTEST_ENV"']
 filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)'
 setup = 'ecstore-large-stack'
 
+[[profile.default.scripts]]
+filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
+setup = 'ecstore-base-stack'
+
 [[profile.default.scripts]]
 filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
 setup = 'lifecycle-large-stack'
@@ -100,12 +109,29 @@ test-group = 'embedded-test-ports'
 filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
 test-group = 'ecstore-serial-flaky'
 
+# Serialize the transition matrix tests. They build a 4-disk hermetic erasure
+# set, populate the get_object_metadata_cache, and assert generation lifecycle
+# semantics. serial_test's #[serial] has no effect across nextest's process
+# boundary, so concurrent execution races the shared metadata-cache generation
+# counter and causes spurious "metadata read should publish the generation"
+# panics. Preventive serialization, no retries.
+[[profile.default.overrides]]
+filter = 'package(rustfs-ecstore) & test(set_disk::transition_matrix_tests::)'
+test-group = 'ecstore-serial-flaky'
+
 # The durable ILM decommission regressions build isolated multi-pool stores and
 # deliberately take source or target disks offline while checking fencing.
 [[profile.default.overrides]]
 filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
 test-group = 'ecstore-serial-flaky'
 
+# Decommission entry and marker/barrier tests share process-wide fault hooks and
+# deterministic commit barriers. Keep the whole init decommission family in one
+# nextest group; serial_test alone cannot isolate separate test processes.
+[[profile.default.overrides]]
+filter = 'package(rustfs-ecstore) & test(/^store::init::tests::(decommission_|suspended_.*decommission)$/)'
+test-group = 'ecstore-serial-flaky'
+
 # Serialize the bucket-incarnation / lifecycle-fence tests. They drive
 # init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global
 # OnceLock state that serial_test's #[serial] cannot protect across nextest's
@@ -160,6 +186,10 @@ path = "junit.xml"
 filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)'
 setup = 'ecstore-large-stack'
 
+[[profile.ci.scripts]]
+filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
+setup = 'ecstore-base-stack'
+
 [[profile.ci.scripts]]
 filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
 setup = 'lifecycle-large-stack'
@@ -232,10 +262,20 @@ test-group = 'embedded-test-ports'
 filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
 test-group = 'ecstore-serial-flaky'
 
+# Serialize the transition matrix tests under the ci profile too (see the
+# matching default-profile override near the top). No retries.
+[[profile.ci.overrides]]
+filter = 'package(rustfs-ecstore) & test(set_disk::transition_matrix_tests::)'
+test-group = 'ecstore-serial-flaky'
+
 [[profile.ci.overrides]]
 filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
 test-group = 'ecstore-serial-flaky'
 
+[[profile.ci.overrides]]
+filter = 'package(rustfs-ecstore) & test(/^store::init::tests::(decommission_|suspended_.*decommission)$/)'
+test-group = 'ecstore-serial-flaky'
+
 # Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile
 # too (see the matching default-profile override near the top). No retries.
 [[profile.ci.overrides]]
@@ -452,23 +492,12 @@ path = "junit.xml"
 # parallel-safe — the same property e2e-smoke relies on. The exceptions are the
 # 4-disk reliability / degraded-read fault-injection tests and the fixed-port
 # Vault tests, both serialized below.
-# KNOWN-FAILURE EXCLUSIONS (characterization run 29381309848, 2026-07-15:
-# 341 ran / 32 failed on the suites' first automated run ever). Deterministic
-# product failures cannot be quarantined away with retries, so each family is
-# excluded here with its tracking issue, under the same discipline as the
-# ci-profile quarantine (docs/testing/README.md): every entry MUST cite one
-# OPEN issue, and the fixing PR MUST delete the exclusion. The passing
-# negative-path siblings of each family stay in as regression guards.
-#   * rustfs#4843 — over-limit archive entry paths hard-reject the whole
-#     archive even under ignore-errors semantics.
 [profile.e2e-full]
 default-filter = """
     package(e2e_test)
     & !test(/^protocols::/)
     & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
     & !test(/^replication_extension_test::/)
-    & !test(/^multipart_auth_test::test_signed_put_object_extract_skips_invalid_entry_when_ignore_errors_enabled$/)
-    & !test(/^snowball_auto_extract_test::tests::snowball_auto_extract_(ignores_invalid_entries_when_requested|supports_standard_headers_with_combined_extract_options)$/)
 """
 fail-fast = false
 

.github/actionlint.yaml

@@ -6,3 +6,4 @@ self-hosted-runner:
     - sm-standard-4
     - dind-sm-standard-2
     - smoke-testing
+    - pf-testing

.github/scheduled-validations.json

@@ -7,6 +7,11 @@
   { "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
   { "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
   { "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
+  {
+    "workflow": ".github/workflows/minio-interop.yml",
+    "max_age_hours": 36,
+    "never_ran_grace_until": "2026-09-08T00:00:00Z"
+  },
   { "workflow": ".github/workflows/nightly-gnu.yml", "max_age_hours": 36 },
   { "workflow": ".github/workflows/performance-ab.yml", "max_age_hours": 36 },
   {

.github/workflows/build.yml

@@ -244,7 +244,7 @@ jobs:
     needs: [ build-check, prepare-platform-matrix ]
     if: needs.build-check.outputs.should_build == 'true' && needs.prepare-platform-matrix.result == 'success'
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 150
+    timeout-minutes: 180
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       # Release binaries ship without dial9 telemetry and therefore do not need

.github/workflows/minio-interop.yml

@@ -20,27 +20,27 @@
 # each run with Docker and then runs the `#[ignore]` reader tests in
 # rustfs/src/storage/minio_generated_read_test.rs.
 #
-# Scope: end-to-end MinIO-to-RustFS SSE interop is NOT implemented yet. Both
+# Scope: MinIO-to-RustFS SSE read interop is implemented behind the `rio-v2`
-# envelope parsers reject MinIO's own wrapped-DEK shape — see
+# feature for MinIO's builtin static-KMS deployments — SSE-S3 and SSE-KMS
-# `is_data_key_envelope` in crates/kms/src/encryption/dek.rs and the
+# (single- and multipart) since rustfs/rustfs#6191, SSE-C detection since the
-# `deny_unknown_fields` `LocalSseDekEnvelope` in rustfs/src/storage/sse.rs — and
+# rustfs/backlog#1638 D2 close-out. This job is the standing evidence: it
-# closing that gap is tracked in rustfs/backlog#1638. Treat this job as the
+# regenerates real MinIO backend trees and proves byte-identical plaintext
-# harness for #1638, not as standing evidence that a MinIO migration reads back.
+# reconstruction. KES/MinKMS-backed MinIO objects remain unreadable by design
+# (their envelopes are sealed by the KES service, not by a key RustFS can
+# hold), and default RustFS builds do not include the read path — it is a
+# special-purpose migration capability, not a default-build feature.
 #
 # Runner: GitHub-hosted `ubuntu-latest`. It reliably ships Docker + Python,
 # unlike the self-hosted fleet, whose pods drift in Docker/pip availability
 # (see the infra note in e2e-s3tests.yml). Nightly + manual only.
-# DISABLED. This workflow is switched off in the repository's Actions settings
-# (state: disabled_manually) and does not run on any trigger, including its cron
-# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
-# reading this file, which has already misled at least one audit — hence this
-# banner. Re-enabling is a UI action; anyone doing so should first check that the
-# workflow still matches the current CI layout. See rustfs/backlog#1603.
 #
-# While disabled, this workflow is deliberately absent from
+# Enablement: this workflow was long disabled in the repository's Actions
-# .github/scheduled-validations.json — a disabled workflow can never satisfy the
+# settings (state: disabled_manually — a state that lives in GitHub's UI and is
-# freshness check. Whoever re-enables it must re-add the entry in the same
+# invisible in this file). The change that updated this banner also re-added
-# change so the freshness gate covers it again.
+# the .github/scheduled-validations.json entry; both only make sense together
+# with re-enabling the workflow in the Actions settings. If it is ever disabled
+# again, remove the scheduled-validations entry in the same change — a disabled
+# workflow can never satisfy the freshness check. See rustfs/backlog#1603.
 #
 name: minio-interop
 

.github/workflows/nix.yml

@@ -27,6 +27,7 @@ on:
     paths:
       - 'flake.nix'
       - 'flake.lock'
+      - 'nix/**'
       - 'Cargo.toml'
       - 'Cargo.lock'
       - '.github/workflows/nix.yml'
@@ -36,6 +37,7 @@ on:
     paths:
       - 'flake.nix'
       - 'flake.lock'
+      - 'nix/**'
       - 'Cargo.toml'
       - 'Cargo.lock'
       - '.github/workflows/nix.yml'

.github/workflows/package.yml

@@ -224,7 +224,7 @@ jobs:
           ZIP_FILE=$(find ./binary-artifact -name "*.zip" -type f | head -1)
           if [[ -z "$ZIP_FILE" ]]; then
             echo "❌ No binary artifact found"
-            ls -la ./binary-artifact/ || true
+            find ./binary-artifact -mindepth 1 -maxdepth 1 -print 2>/dev/null || true
             exit 1
           fi
 
@@ -239,7 +239,7 @@ jobs:
           fi
 
           chmod +x ./bin/rustfs
-          ls -lh ./bin/rustfs
+          stat --printf='%n %s bytes\n' ./bin/rustfs
           echo "✅ Binary extracted"
 
       - name: Build DEB package
@@ -336,7 +336,7 @@ jobs:
           fakeroot dpkg-deb --build "${PKG_DIR}"
 
           DEB_FILE="${PKG_DIR}.deb"
-          ls -lh "$DEB_FILE"
+          stat --printf='%n %s bytes\n' "$DEB_FILE"
           echo "deb_file=$DEB_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ DEB built: $DEB_FILE"
 
@@ -410,13 +410,14 @@ jobs:
             LICENSE=/usr/share/doc/rustfs/LICENSE \
             README.md=/usr/share/doc/rustfs/README.md
 
-          RPM_FILE=$(ls -1 rustfs-*.rpm 2>/dev/null | head -1)
+            RPM_FILE=$(find . -maxdepth 1 -type f -name 'rustfs-*.rpm' -print | head -1)
+            RPM_FILE="${RPM_FILE#./}"
           if [[ -z "$RPM_FILE" ]]; then
             echo "❌ RPM build failed"
             exit 1
           fi
 
-          ls -lh "$RPM_FILE"
+          stat --printf='%n %s bytes\n' "$RPM_FILE"
           echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ RPM built: $RPM_FILE"
 
@@ -552,11 +553,13 @@ jobs:
       - name: Print summary
         shell: bash
         run: |
-          echo "## 📦 Package Summary" >> "$GITHUB_STEP_SUMMARY"
+          {
-          echo "" >> "$GITHUB_STEP_SUMMARY"
+            echo "## 📦 Package Summary"
-          echo "| Item | Value |" >> "$GITHUB_STEP_SUMMARY"
+            echo ""
-          echo "|------|-------|" >> "$GITHUB_STEP_SUMMARY"
+            echo "| Item | Value |"
-          echo "| Version | \`${{ needs.resolve.outputs.version }}\` |" >> "$GITHUB_STEP_SUMMARY"
+            echo "|------|-------|"
-          echo "| Build Type | ${{ needs.resolve.outputs.build_type }} |" >> "$GITHUB_STEP_SUMMARY"
+            echo "| Version | \`${{ needs.resolve.outputs.version }}\` |"
-          echo "| Build Run | #${{ needs.resolve.outputs.build_run_id }} |" >> "$GITHUB_STEP_SUMMARY"
+            echo "| Build Type | ${{ needs.resolve.outputs.build_type }} |"
-          echo "| Package Status | ${{ needs.package.result }} |" >> "$GITHUB_STEP_SUMMARY"
+            echo "| Build Run | #${{ needs.resolve.outputs.build_run_id }} |"
+            echo "| Package Status | ${{ needs.package.result }} |"
+          } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/rustfs-heal-test.yml

@@ -15,10 +15,6 @@ on:
         description: 'Stop warp when surviving nodes reach N GiB'
         required: false
         default: '40'
-      heal_target_gb:
-        description: 'Outage node must reach N GiB after heal to pass'
-        required: false
-        default: '40'
       cleanup_before:
         description: 'Reset the nodes before the test (DESTROYS existing data/config)'
         type: boolean
@@ -34,7 +30,7 @@ permissions:
 # Only one test at a time: both this and the pool-expansion workflow mutate
 # the same test environment, so they share one concurrency group.
 concurrency:
-  group: rustfs-pool-expansion-test
+  group: rustfs-shared-functional-tests
   cancel-in-progress: false
 
 defaults:
@@ -47,17 +43,25 @@ env:
   RUSTFS_API_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
   RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
   RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
 
 jobs:
   heal-test:
     runs-on: smoke-testing
     timeout-minutes: 480
+    # Manual-only standalone run. Nightly chain already runs heal in
+    # rustfs-pool-expand-test.yml to avoid duplicate heal executions.
+    if: ${{ github.event_name == 'workflow_dispatch' }}
     steps:
-      - name: Checkout
+      - name: Checkout auto-testing scripts
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
           persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
       - name: Show environment
         run: |
@@ -67,11 +71,24 @@ jobs:
           warp --version || true
           df -h /data | tail -1
 
-      - name: Reset test environment (before)
+      - name: Cleanup environment (before)
         if: ${{ inputs.cleanup_before != 'false' }}
         run: |
-          chmod +x scripts/test/rustfs_heal_test.sh
+          set -euo pipefail
-          ./scripts/test/rustfs_heal_test.sh --reset -y
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
+            '
+          done
 
       - name: Install RustFS package & start cluster
         run: |
@@ -81,7 +98,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./scripts/test/rustfs_heal_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
 
       - name: Preflight checks
         run: |
@@ -91,18 +108,223 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./scripts/test/rustfs_heal_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
 
       - name: Run heal test (write -> outage -> heal -> verify)
+        id: test
         run: |
-          ./scripts/test/rustfs_heal_test.sh \
+          ./auto-testing/rustfs_heal_test.sh \
             --steps "3,4,5,6,7" -y \
             --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
             --stop-node-gb "${{ inputs.stop_node_gb }}" \
             --warp-stop-gb "${{ inputs.warp_stop_gb }}" \
-            --heal-target-gb "${{ inputs.heal_target_gb }}" \
             --log-file /tmp/rustfs-heal-test.log
 
+      - name: Generate report
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-heal-test.log
+          REPORT_FILE: /tmp/rustfs-heal-report.md
+        run: |
+          set -euo pipefail
+          PACKAGE_URL='${{ inputs.package_url }}'
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          {
+            echo "# RustFS heal test report"
+            echo ""
+            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Trigger: ${{ github.event_name }}"
+            echo "- Package: ${PACKAGE_SOURCE}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
+          } | tee "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload functional report to dashboard
+        if: always()
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-heal-report.md
+          SUITE: heal
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+
+          cat > /tmp/rustfs-functional-index.html <<'EOF'
+          <!doctype html>
+          <html lang="en">
+          <head>
+            <meta charset="utf-8" />
+            <meta name="viewport" content="width=device-width, initial-scale=1" />
+            <title>RustFS Functional Test Reports</title>
+            <style>
+              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
+              * { box-sizing: border-box; }
+              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
+              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
+              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
+              h1 { margin: 0 0 8px; font-size: 26px; }
+              p { margin: 0 0 14px; color: var(--muted); }
+              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
+              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
+              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
+              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
+              ul { list-style: none; margin: 0; padding: 0; }
+              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
+              a { color: var(--accent); text-decoration: none; }
+              a:hover { text-decoration: underline; }
+              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
+              .kv { margin: 6px 0; color: var(--text); }
+              .muted { color: var(--muted); }
+            </style>
+          </head>
+          <body>
+            <div class="wrap">
+              <div class="card">
+                <h1>RustFS Functional Test Reports</h1>
+                <p>Select a suite and date to view the build version used in that run.</p>
+                <div class="tabs" id="tabs"></div>
+                <ul id="list"></ul>
+                <div class="meta">
+                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
+                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
+                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
+                </div>
+              </div>
+            </div>
+            <script>
+              const suites = [
+                { key: 's3', label: 'S3 Compatibility' },
+                { key: 'kms', label: 'KMS' },
+                { key: 'tier', label: 'Tier' },
+                { key: 'heal', label: 'Heal' },
+                { key: 'pool', label: 'Pool Expansion' },
+                { key: 'security', label: 'Security' },
+                { key: 'upgrade', label: 'Upgrade' },
+              ];
+              const tabs = document.getElementById('tabs');
+              const list = document.getElementById('list');
+              const reportDate = document.getElementById('report-date');
+              const reportVersion = document.getElementById('report-version');
+              const reportLink = document.getElementById('report-link');
+
+              function parseVersion(markdown) {
+                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
+                return m ? m[1].trim() : 'N/A';
+              }
+
+              async function showReport(report) {
+                reportDate.textContent = report.name.replace('.md', '');
+                reportVersion.textContent = 'Loading...';
+                reportLink.href = report.html_url;
+                try {
+                  const res = await fetch(report.download_url, { cache: 'no-store' });
+                  if (!res.ok) {
+                    reportVersion.textContent = 'N/A';
+                    return;
+                  }
+                  const text = await res.text();
+                  reportVersion.textContent = parseVersion(text);
+                } catch (_e) {
+                  reportVersion.textContent = 'N/A';
+                }
+              }
+
+              async function loadSuite(suite) {
+                list.innerHTML = '<li>Loading...</li>';
+                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
+                try {
+                  const res = await fetch(api);
+                  if (!res.ok) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    return;
+                  }
+                  const data = await res.json();
+                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
+                  if (!files.length) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    reportDate.textContent = 'N/A';
+                    reportVersion.textContent = 'N/A';
+                    reportLink.href = '#';
+                    return;
+                  }
+                  list.innerHTML = '';
+                  files.forEach((f) => {
+                    const li = document.createElement('li');
+                    const btn = document.createElement('button');
+                    btn.className = 'report-btn';
+                    btn.textContent = f.name.replace('.md', '');
+                    btn.addEventListener('click', () => showReport(f));
+                    li.appendChild(btn);
+                    list.appendChild(li);
+                  });
+                  showReport(files[0]);
+                } catch (_e) {
+                  list.innerHTML = '<li>Failed to load reports.</li>';
+                  reportDate.textContent = 'N/A';
+                  reportVersion.textContent = 'N/A';
+                  reportLink.href = '#';
+                }
+              }
+
+              function setActive(key) {
+                for (const btn of tabs.querySelectorAll('button')) {
+                  btn.classList.toggle('active', btn.dataset.key === key);
+                }
+                loadSuite(key);
+              }
+
+              for (const suite of suites) {
+                const btn = document.createElement('button');
+                btn.textContent = suite.label;
+                btn.dataset.key = suite.key;
+                btn.addEventListener('click', () => setActive(suite.key));
+                tabs.appendChild(btn);
+              }
+              setActive('s3');
+            </script>
+          </body>
+          </html>
+          EOF
+
+          INDEX_PATH="functional/index.html"
+          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
+          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${INDEX_SHA}" ]; then
+            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          fi
+
       - name: Upload test logs
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
@@ -113,10 +335,24 @@ jobs:
             /tmp/rustfs-warp.*.log
           if-no-files-found: warn
 
-      - name: Reset test environment (after)
+      - name: Cleanup environment (after)
         if: ${{ always() && inputs.cleanup_after != 'false' }}
         run: |
-          ./scripts/test/rustfs_heal_test.sh --reset -y
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
+            '
+          done
 
       - name: Notify on failure
         if: failure()

.github/workflows/rustfs-kms-test.yml

@@ -0,0 +1,376 @@
+name: RustFS KMS Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      rustfs_version:
+        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        required: false
+        default: '1.0.0-rc.4-preview.1'
+      package_url:
+        description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
+        required: false
+        type: string
+      enforce_sse_key_policy:
+        description: 'Enable RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY (runs KMS-401/402)'
+        type: boolean
+        default: false
+      frame_v2:
+        description: 'Enable RUSTFS_ENCRYPTION_FRAME_V2 (runs KMS-318)'
+        type: boolean
+        default: false
+      config_secret:
+        description: 'Set RUSTFS_KMS_CONFIG_SECRET (runs KMS-107 config sealing)'
+        required: false
+        type: string
+  workflow_run:
+    # Strict shared-environment order: run after S3 compatibility test completes.
+    workflows: ["RustFS S3 Compatibility Test"]
+    types: [completed]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rustfs-shared-functional-tests
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  kms-test:
+    runs-on: smoke-testing
+    continue-on-error: true
+    timeout-minutes: 420
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }}
+    steps:
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          openssl version
+          docker --version || true
+
+      - name: Cleanup environment (before)
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
+            '
+          done
+
+      - name: Ensure docker (Vault container)
+        run: |
+          if ! command -v docker >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y docker.io
+          fi
+          sudo systemctl enable --now docker
+          docker info >/dev/null 2>&1 || sudo docker info >/dev/null 2>&1
+
+      - name: Run KMS suite
+        id: test
+        continue-on-error: true
+        env:
+          LOG_FILE: /tmp/rustfs-kms.log
+        run: |
+          set -euo pipefail
+          chmod +x auto-testing/rustfs-kms-test.sh
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          ARGS=(--all-topologies --backends "local,vault-kv2" -y --log-file "${LOG_FILE}")
+          EXTRA_ENV=""
+          if [ "${{ inputs.enforce_sse_key_policy }}" = "true" ]; then
+            EXTRA_ENV+="RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY=true"$'\n'
+          fi
+          if [ "${{ inputs.frame_v2 }}" = "true" ]; then
+            EXTRA_ENV+="RUSTFS_ENCRYPTION_FRAME_V2=true"$'\n'
+          fi
+          if [ -n "${{ inputs.config_secret }}" ]; then
+            EXTRA_ENV+="RUSTFS_KMS_CONFIG_SECRET=${{ inputs.config_secret }}"$'\n'
+          fi
+          if [ -n "${EXTRA_ENV}" ]; then
+            ARGS+=(--extra-env "${EXTRA_ENV}")
+          fi
+          if [ -n "${PACKAGE_URL}" ]; then
+            ARGS+=(--package-url "${PACKAGE_URL}")
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            ARGS+=(--version "${RUSTFS_VERSION}")
+          else
+            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
+          fi
+          ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
+
+      - name: Generate report
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-kms.log
+          REPORT_FILE: /tmp/rustfs-kms-report.md
+        run: |
+          set -euo pipefail
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          CASE_TABLE="/tmp/rustfs-kms-cases.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
+
+          rows = []
+          index = {}
+          try:
+            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+              for raw in fh:
+                line = ansi.sub('', raw).strip()
+                m = start_re.match(line)
+                if m:
+                  case_id, name = m.group(1), m.group(2)
+                  if case_id not in index:
+                    index[case_id] = len(rows)
+                    rows.append([case_id, name, 'RUNNING'])
+                  continue
+                m = done_re.match(line)
+                if m:
+                  status, case_id = m.group(1), m.group(2)
+                  if case_id in index:
+                    rows[index[case_id]][2] = status
+                  else:
+                    rows.append([case_id, case_id, status])
+                    index[case_id] = len(rows) - 1
+          except FileNotFoundError:
+            rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+            counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+            out.write('## Case Summary\n\n')
+            out.write(f"- Total: {len(rows)}\\n")
+            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+            out.write('\\n')
+            out.write('| Case | Name | Status |\\n')
+            out.write('| --- | --- | --- |\\n')
+            for case_id, name, status in rows:
+              out.write(f'| {case_id} | {name} | {status} |\\n')
+          PY
+          {
+            echo "# RustFS KMS test report"
+            echo ""
+            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Trigger: ${{ github.event_name }}"
+            echo "- Package: ${PACKAGE_SOURCE}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo ""
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
+          } | tee "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload functional report to dashboard
+        if: always()
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-kms-report.md
+          SUITE: kms
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+
+          cat > /tmp/rustfs-functional-index.html <<'EOF'
+          <!doctype html>
+          <html lang="en">
+          <head>
+            <meta charset="utf-8" />
+            <meta name="viewport" content="width=device-width, initial-scale=1" />
+            <title>RustFS Functional Test Reports</title>
+            <style>
+              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
+              * { box-sizing: border-box; }
+              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
+              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
+              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
+              h1 { margin: 0 0 8px; font-size: 26px; }
+              p { margin: 0 0 14px; color: var(--muted); }
+              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
+              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
+              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
+              ul { list-style: none; margin: 0; padding: 0; }
+              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
+              a { color: var(--accent); text-decoration: none; }
+              a:hover { text-decoration: underline; }
+            </style>
+          </head>
+          <body>
+            <div class="wrap">
+              <div class="card">
+                <h1>RustFS Functional Test Reports</h1>
+                <p>S3, KMS, Tier report tabs. Each tab lists reports by date.</p>
+                <div class="tabs" id="tabs"></div>
+                <ul id="list"></ul>
+              </div>
+            </div>
+            <script>
+              const suites = [
+                { key: 's3', label: 'S3 Compatibility' },
+                { key: 'kms', label: 'KMS' },
+                { key: 'tier', label: 'Tier' },
+                { key: 'heal', label: 'Heal' },
+                { key: 'pool', label: 'Pool Expansion' },
+                { key: 'security', label: 'Security' },
+                { key: 'upgrade', label: 'Upgrade' },
+              ];
+              const tabs = document.getElementById('tabs');
+              const list = document.getElementById('list');
+
+              async function loadSuite(suite) {
+                list.innerHTML = '<li>Loading...</li>';
+                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
+                try {
+                  const res = await fetch(api);
+                  if (!res.ok) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    return;
+                  }
+                  const data = await res.json();
+                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
+                  if (!files.length) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    return;
+                  }
+                  list.innerHTML = files.map(f => `<li><a href="${f.html_url}" target="_blank" rel="noreferrer">${f.name.replace('.md','')}</a></li>`).join('');
+                } catch (_e) {
+                  list.innerHTML = '<li>Failed to load reports.</li>';
+                }
+              }
+
+              function setActive(key) {
+                for (const btn of tabs.querySelectorAll('button')) {
+                  btn.classList.toggle('active', btn.dataset.key === key);
+                }
+                loadSuite(key);
+              }
+
+              for (const suite of suites) {
+                const btn = document.createElement('button');
+                btn.textContent = suite.label;
+                btn.dataset.key = suite.key;
+                btn.addEventListener('click', () => setActive(suite.key));
+                tabs.appendChild(btn);
+              }
+              setActive('s3');
+            </script>
+          </body>
+          </html>
+          EOF
+
+          INDEX_PATH="functional/index.html"
+          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
+          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${INDEX_SHA}" ]; then
+            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          fi
+
+      - name: Upload report and logs
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-kms-test-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-kms.log
+            /tmp/rustfs-kms-report.md
+          if-no-files-found: warn
+
+      - name: Cleanup environment (after)
+        if: always()
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
+            '
+          done
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS KMS suite failed"
+          echo "See the uploaded report and log artifacts for details."

.github/workflows/rustfs-performance-test.yml

@@ -0,0 +1,236 @@
+name: RustFS Performance Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      package_url:
+        description: 'Direct .deb URL (nightly/R2). Defaults to the latest nightly deb.'
+        required: false
+        type: string
+      test_method:
+        description: 'Benchmark method(s) to run (manual runs only; "all" = GET+PUT+MIXED)'
+        type: choice
+        options:
+          - all
+          - get
+          - put
+          - mixed
+        default: 'all'
+      object_size:
+        description: 'Object size(s) to test (manual runs only; "all" = all 10 sizes)'
+        type: choice
+        options:
+          - all
+          - 1KiB
+          - 4KiB
+          - 16KiB
+          - 128KiB
+          - 1MiB
+          - 4MiB
+          - 8MiB
+          - 16MiB
+          - 32MiB
+          - 64MiB
+        default: 'all'
+      warp_duration:
+        description: 'warp duration per round (e.g. 5m, 30s)'
+        required: false
+        default: '5m'
+      warp_concurrency:
+        description: 'warp concurrency'
+        required: false
+        default: '64'
+      cleanup_before:
+        description: 'Reset the nodes before the test (DESTROYS existing data/config)'
+        type: boolean
+        default: true
+      cleanup_after:
+        description: 'Reset the nodes after the test (DESTROYS test data/config)'
+        type: boolean
+        default: true
+  workflow_run:
+    # Run after the nightly build completes; the nightly deb is what the test installs.
+    workflows: ["Nightly GNU Build"]
+    types: [completed]
+
+permissions:
+  contents: read
+
+# Dedicated pf-testing runner/environment: own concurrency group so perf runs
+# never block (or are blocked by) the pool-expansion / heal tests.
+concurrency:
+  group: rustfs-performance-test
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  # Performance test uses its own node list (4 nodes); the shared
+  # RUSTFS_NODES secret is used by the 3-node pool-expansion / heal tests.
+  RUSTFS_NODES: ${{ secrets.RUSTFS_PERF_NODES || vars.RUSTFS_PERF_NODES || 'vm000 vm001 vm002 vm003' }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  # Package used by the nightly run (workflow_dispatch inputs are empty for
+  # workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml.
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  # Fixed benchmark result directory so later steps can read summary.md
+  RUSTFS_RESULT_DIR: /tmp/rustfs-perf-results
+  # Cross-repo token for uploading reports to rustfs/dashboard (set in repo settings)
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  performance-test:
+    runs-on: pf-testing
+    timeout-minutes: 900
+    # Run on manual dispatch, or when the nightly build completed successfully.
+    # Skipped when nightly failed.
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          warp --version || true
+          df -h /data | tail -1
+
+      - name: Reset test environment (before)
+        if: ${{ inputs.cleanup_before != 'false' }}
+        run: |
+          chmod +x auto-testing/rustfs_performance_test.sh
+          ./auto-testing/rustfs_performance_test.sh --step 1 -y
+
+      - name: Install RustFS package & start cluster (4x4)
+        run: |
+          ARGS=(--steps "2,3,4" -y)
+          if [ -n "${{ inputs.package_url }}" ]; then
+            ARGS+=(--package-url "${{ inputs.package_url }}")
+          else
+            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
+          fi
+          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
+
+      - name: Preflight checks
+        run: |
+          ARGS=(--preflight)
+          if [ -n "${{ inputs.package_url }}" ]; then
+            ARGS+=(--package-url "${{ inputs.package_url }}")
+          else
+            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
+          fi
+          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
+
+      - name: Run benchmark (GET/PUT/MIXED)
+        id: benchmark
+        run: |
+          # Empty on automatic (workflow_run) runs -> full 30 rounds.
+          # Manual dispatch can restrict method(s)/size(s).
+          export WARP_METHODS="${{ inputs.test_method }}"
+          export WARP_SIZES="${{ inputs.object_size }}"
+          ./auto-testing/rustfs_performance_test.sh \
+            --step 5 -y \
+            --warp-duration "${{ inputs.warp_duration || '5m' }}" \
+            --warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \
+            --log-file /tmp/rustfs-perf-test.log
+
+      - name: Analyze results
+        if: ${{ steps.benchmark.conclusion == 'success' }}
+        run: |
+          ./auto-testing/rustfs_performance_test.sh --step 6 -y
+
+      - name: Collect RustFS version info
+        if: ${{ steps.benchmark.conclusion == 'success' }}
+        env:
+          VERSION_FILE: /tmp/rustfs-version.txt
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES}"
+          [ "${#NODES[@]}" -gt 0 ] || { echo "RUSTFS_NODES is empty"; exit 1; }
+          NODE="${NODES[0]}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          {
+            echo "Node: ${NODE}"
+            echo "Command: rustfs --version"
+            echo ""
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
+              "${SSH_USER}@${NODE}" 'rustfs --version'
+          } > "${VERSION_FILE}"
+
+      - name: Upload report to dashboard (reports/YYYY-MM-DD.md)
+        if: ${{ steps.benchmark.conclusion == 'success' }}
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }}
+          VERSION_FILE: /tmp/rustfs-version.txt
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping report upload"
+            exit 0
+          fi
+          SUMMARY="${RESULT_DIR}/summary.md"
+          [ -f "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="reports/${DATE}.md"
+          {
+            echo "# RustFS nightly build performance testing report"
+            echo ""
+            echo "- **Date**: ${DATE}"
+            echo "- **Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- **Trigger**: ${{ github.event_name }}"
+            echo "- **Package**: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
+            echo ""
+            cat "${SUMMARY}"
+            echo ""
+            echo "## RustFS version"
+            echo '```text'
+            cat "${VERSION_FILE}"
+            echo '```'
+          } > /tmp/rustfs-perf-report.md
+          CONTENT="$(python3 -c 'import base64; print(base64.b64encode(open("/tmp/rustfs-perf-report.md","rb").read()).decode())')"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+            echo "updated ${REPORT_PATH} in rustfs/dashboard"
+          else
+            jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+            echo "created ${REPORT_PATH} in rustfs/dashboard"
+          fi
+
+      - name: Upload test logs & results
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-perf-test-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-perf-test*.log
+            /tmp/rustfs-perf-results/**
+            /tmp/rustfs-version.txt
+          if-no-files-found: warn
+
+      - name: Reset test environment (after)
+        if: ${{ always() && inputs.cleanup_after != 'false' }}
+        run: |
+          ./auto-testing/rustfs_performance_test.sh --step 7 -y
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS performance test failed"
+          echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
+          echo "See the uploaded log artifact for details."

.github/workflows/rustfs-s3-compat-test.yml

@@ -0,0 +1,406 @@
+name: RustFS S3 Compatibility Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      rustfs_version:
+        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        required: false
+        default: '1.0.0-rc.4-preview.1'
+      package_url:
+        description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
+        required: false
+        type: string
+  workflow_run:
+    # Run after upgrade compatibility completes; the nightly deb is what the test installs.
+    workflows: ["RustFS Upgrade Test"]
+    types: [completed]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rustfs-shared-functional-tests
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  s3-compat-test:
+    runs-on: smoke-testing
+    continue-on-error: true
+    timeout-minutes: 360
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          openssl version
+          df -h /data | tail -1
+
+      - name: Cleanup environment (before)
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs
+            '
+          done
+
+      - name: Run S3 compatibility suite
+        id: test
+        continue-on-error: true
+        env:
+          LOG_FILE: /tmp/rustfs-s3-compat.log
+        run: |
+          set -euo pipefail
+          chmod +x auto-testing/rustfs-s3-compat-test.sh
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          ARGS=(--all-topologies -y --log-file "${LOG_FILE}")
+          if [ -n "${PACKAGE_URL}" ]; then
+            ARGS+=(--package-url "${PACKAGE_URL}")
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            ARGS+=(--version "${RUSTFS_VERSION}")
+          else
+            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
+          fi
+          ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
+
+      - name: Generate report
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-s3-compat.log
+          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
+        run: |
+          set -euo pipefail
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          RUSTFS_VERSION_INFO="N/A"
+          if [ "${#NODES[@]}" -gt 0 ]; then
+            DETECTED_VERSION="$(ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
+              "${SSH_USER}@${NODES[0]}" 'rustfs --version' 2>/dev/null | tr -d '\r' | head -n 1 || true)"
+            if [ -n "${DETECTED_VERSION}" ]; then
+              RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
+            fi
+          fi
+          CASE_TABLE="/tmp/rustfs-s3-compat-cases.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
+
+          rows = []
+          index = {}
+          current = None
+          try:
+            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+              for raw in fh:
+                line = ansi.sub('', raw).strip()
+                m = start_re.match(line)
+                if m:
+                  case_id, name = m.group(1), m.group(2)
+                  current = case_id
+                  if case_id not in index:
+                    index[case_id] = len(rows)
+                    rows.append([case_id, name, 'RUNNING'])
+                  continue
+                m = done_re.match(line)
+                if m:
+                  status, case_id = m.group(1), m.group(2)
+                  if case_id in index:
+                    rows[index[case_id]][2] = status
+                  else:
+                    rows.append([case_id, case_id, status])
+                    index[case_id] = len(rows) - 1
+                  current = None
+          except FileNotFoundError:
+            rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+            counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+            out.write('## Case Summary\n\n')
+            out.write(f"- Total: {len(rows)}\\n")
+            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+            out.write('\\n')
+            out.write('| Case | Name | Status |\\n')
+            out.write('| --- | --- | --- |\\n')
+            for case_id, name, status in rows:
+              out.write(f'| {case_id} | {name} | {status} |\\n')
+          PY
+          {
+            echo "# RustFS S3 compatibility test report"
+            echo ""
+            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Trigger: ${{ github.event_name }}"
+            echo "- Package: ${PACKAGE_SOURCE}"
+            echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo ""
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
+          } | tee "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload functional report to dashboard
+        if: always()
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
+          SUITE: s3
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+
+          cat > /tmp/rustfs-functional-index.html <<'EOF'
+          <!doctype html>
+          <html lang="en">
+          <head>
+            <meta charset="utf-8" />
+            <meta name="viewport" content="width=device-width, initial-scale=1" />
+            <title>RustFS Functional Test Reports</title>
+            <style>
+              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
+              * { box-sizing: border-box; }
+              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
+              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
+              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
+              h1 { margin: 0 0 8px; font-size: 26px; }
+              p { margin: 0 0 14px; color: var(--muted); }
+              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
+              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
+              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
+              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
+              ul { list-style: none; margin: 0; padding: 0; }
+              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
+              a { color: var(--accent); text-decoration: none; }
+              a:hover { text-decoration: underline; }
+              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
+              .kv { margin: 6px 0; color: var(--text); }
+              .muted { color: var(--muted); }
+            </style>
+          </head>
+          <body>
+            <div class="wrap">
+              <div class="card">
+                <h1>RustFS Functional Test Reports</h1>
+                <p>Select a suite and date to view the build version used in that run.</p>
+                <div class="tabs" id="tabs"></div>
+                <ul id="list"></ul>
+                <div class="meta">
+                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
+                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
+                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
+                </div>
+              </div>
+            </div>
+            <script>
+              const suites = [
+                { key: 's3', label: 'S3 Compatibility' },
+                { key: 'kms', label: 'KMS' },
+                { key: 'tier', label: 'Tier' },
+                { key: 'heal', label: 'Heal' },
+                { key: 'pool', label: 'Pool Expansion' },
+                { key: 'security', label: 'Security' },
+                { key: 'upgrade', label: 'Upgrade' },
+              ];
+              const tabs = document.getElementById('tabs');
+              const list = document.getElementById('list');
+              const reportDate = document.getElementById('report-date');
+              const reportVersion = document.getElementById('report-version');
+              const reportLink = document.getElementById('report-link');
+
+              function parseVersion(markdown) {
+                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
+                return m ? m[1].trim() : 'N/A';
+              }
+
+              async function showReport(report) {
+                reportDate.textContent = report.name.replace('.md', '');
+                reportVersion.textContent = 'Loading...';
+                reportLink.href = report.html_url;
+                try {
+                  const res = await fetch(report.download_url, { cache: 'no-store' });
+                  if (!res.ok) {
+                    reportVersion.textContent = 'N/A';
+                    return;
+                  }
+                  const text = await res.text();
+                  reportVersion.textContent = parseVersion(text);
+                } catch (_e) {
+                  reportVersion.textContent = 'N/A';
+                }
+              }
+
+              async function loadSuite(suite) {
+                list.innerHTML = '<li>Loading...</li>';
+                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
+                try {
+                  const res = await fetch(api);
+                  if (!res.ok) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    return;
+                  }
+                  const data = await res.json();
+                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
+                  if (!files.length) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    reportDate.textContent = 'N/A';
+                    reportVersion.textContent = 'N/A';
+                    reportLink.href = '#';
+                    return;
+                  }
+                  list.innerHTML = '';
+                  files.forEach((f) => {
+                    const li = document.createElement('li');
+                    const btn = document.createElement('button');
+                    btn.className = 'report-btn';
+                    btn.textContent = f.name.replace('.md', '');
+                    btn.addEventListener('click', () => showReport(f));
+                    li.appendChild(btn);
+                    list.appendChild(li);
+                  });
+                  showReport(files[0]);
+                } catch (_e) {
+                  list.innerHTML = '<li>Failed to load reports.</li>';
+                  reportDate.textContent = 'N/A';
+                  reportVersion.textContent = 'N/A';
+                  reportLink.href = '#';
+                }
+              }
+
+              function setActive(key) {
+                for (const btn of tabs.querySelectorAll('button')) {
+                  btn.classList.toggle('active', btn.dataset.key === key);
+                }
+                loadSuite(key);
+              }
+
+              for (const suite of suites) {
+                const btn = document.createElement('button');
+                btn.textContent = suite.label;
+                btn.dataset.key = suite.key;
+                btn.addEventListener('click', () => setActive(suite.key));
+                tabs.appendChild(btn);
+              }
+              setActive('s3');
+            </script>
+          </body>
+          </html>
+          EOF
+
+          INDEX_PATH="functional/index.html"
+          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
+          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${INDEX_SHA}" ]; then
+            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          fi
+
+      - name: Upload report and logs
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-s3-compat-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-s3-compat.log
+            /tmp/rustfs-s3-compat-report.md
+          if-no-files-found: warn
+
+      - name: Cleanup environment (after)
+        if: always()
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs
+            '
+          done
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS S3 compatibility suite failed"
+          echo "See the uploaded report and log artifacts for details."

.github/workflows/rustfs-security-test.yml

@@ -0,0 +1,381 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: RustFS Security Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      rustfs_version:
+        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        required: false
+        default: '1.0.0-rc.4-preview.1'
+      package_url:
+        description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
+        required: false
+        type: string
+      topology:
+        description: 'Topology to run (all = SNSD, SNMD, MNMD)'
+        type: choice
+        options:
+          - all
+          - single-single
+          - single-multi
+          - multi-multi
+        default: all
+      oidc_live:
+        description: 'Run the live Keycloak OIDC/SSO gate as part of the suite'
+        type: boolean
+        default: true
+      cleanup_before:
+        description: 'Reset the nodes before the test (DESTROYS existing data/config)'
+        type: boolean
+        default: true
+      cleanup_after:
+        description: 'Reset the nodes after the test (DESTROYS test data/config)'
+        type: boolean
+        default: true
+  workflow_run:
+    # Runs last in the functional chain, after pool/heal, on the shared VMs.
+    workflows: ["RustFS Pool Expansion / Heal Test"]
+    types: [completed]
+
+permissions:
+  contents: read
+
+# The security suite uses the same shared VMs as the other functional tests,
+# so it must serialize with them instead of running in parallel.
+concurrency:
+  group: rustfs-shared-functional-tests
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  security-test:
+    runs-on: smoke-testing
+    continue-on-error: true
+    timeout-minutes: 360
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Checkout repository (for the OIDC live gate script)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          openssl version
+          aws --version || true
+          docker --version || true
+          df -h /data | tail -1
+
+      - name: Cleanup environment (before)
+        if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms
+            '
+          done
+
+      - name: Run security suite
+        id: test
+        continue-on-error: true
+        env:
+          REPORT_FILE: /tmp/rustfs-security-report.md
+          RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh
+        run: |
+          set -euo pipefail
+          chmod +x auto-testing/rustfs-security-test.sh
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          TOPOLOGY='${{ inputs.topology }}'
+          ARGS=(-y)
+          if [ "${TOPOLOGY}" = "all" ] || [ -z "${TOPOLOGY}" ] || [ "${TOPOLOGY}" = "null" ]; then
+            ARGS+=(--all-topologies)
+          else
+            ARGS+=(--topology "${TOPOLOGY}")
+          fi
+          if [ "${{ inputs.oidc_live }}" = "true" ] || [ "${{ github.event_name }}" != "workflow_dispatch" ]; then
+            ARGS+=(--oidc-live)
+          fi
+          if [ -n "${PACKAGE_URL}" ]; then
+            ARGS+=(--package-url "${PACKAGE_URL}")
+          elif [ -n "${RUSTFS_VERSION}" ] && [ "${RUSTFS_VERSION}" != "null" ]; then
+            ARGS+=(--version "${RUSTFS_VERSION}")
+          else
+            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
+          fi
+          ./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
+
+      - name: Generate report
+        if: always()
+        run: |
+          set -euo pipefail
+          if [ ! -f /tmp/rustfs-security-report.md ]; then
+            {
+              echo "# RustFS security test report"
+              echo ""
+              echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+              echo "- Trigger: ${{ github.event_name }}"
+              echo "- Test Step Outcome: failure (suite did not produce a report)"
+            } > /tmp/rustfs-security-report.md
+          fi
+          cat /tmp/rustfs-security-report.md >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload functional report to dashboard
+        if: always()
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-security-report.md
+          SUITE: security
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+
+          cat > /tmp/rustfs-functional-index.html <<'EOF'
+          <!doctype html>
+          <html lang="en">
+          <head>
+            <meta charset="utf-8" />
+            <meta name="viewport" content="width=device-width, initial-scale=1" />
+            <title>RustFS Functional Test Reports</title>
+            <style>
+              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
+              * { box-sizing: border-box; }
+              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
+              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
+              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
+              h1 { margin: 0 0 8px; font-size: 26px; }
+              p { margin: 0 0 14px; color: var(--muted); }
+              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
+              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
+              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
+              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
+              ul { list-style: none; margin: 0; padding: 0; }
+              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
+              a { color: var(--accent); text-decoration: none; }
+              a:hover { text-decoration: underline; }
+              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
+              .kv { margin: 6px 0; color: var(--text); }
+              .muted { color: var(--muted); }
+            </style>
+          </head>
+          <body>
+            <div class="wrap">
+              <div class="card">
+                <h1>RustFS Functional Test Reports</h1>
+                <p>Select a suite and date to view the build version used in that run.</p>
+                <div class="tabs" id="tabs"></div>
+                <ul id="list"></ul>
+                <div class="meta">
+                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
+                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
+                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
+                </div>
+              </div>
+            </div>
+            <script>
+              const suites = [
+                { key: 's3', label: 'S3 Compatibility' },
+                { key: 'kms', label: 'KMS' },
+                { key: 'tier', label: 'Tier' },
+                { key: 'heal', label: 'Heal' },
+                { key: 'pool', label: 'Pool Expansion' },
+                { key: 'security', label: 'Security' },
+                { key: 'upgrade', label: 'Upgrade' },
+              ];
+              const tabs = document.getElementById('tabs');
+              const list = document.getElementById('list');
+              const reportDate = document.getElementById('report-date');
+              const reportVersion = document.getElementById('report-version');
+              const reportLink = document.getElementById('report-link');
+
+              function parseVersion(markdown) {
+                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
+                return m ? m[1].trim() : 'N/A';
+              }
+
+              async function showReport(report) {
+                reportDate.textContent = report.name.replace('.md', '');
+                reportVersion.textContent = 'Loading...';
+                reportLink.href = report.html_url;
+                try {
+                  const res = await fetch(report.download_url, { cache: 'no-store' });
+                  if (!res.ok) {
+                    reportVersion.textContent = 'N/A';
+                    return;
+                  }
+                  const text = await res.text();
+                  reportVersion.textContent = parseVersion(text);
+                } catch (_e) {
+                  reportVersion.textContent = 'N/A';
+                }
+              }
+
+              async function loadSuite(suite) {
+                list.innerHTML = '<li>Loading...</li>';
+                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
+                try {
+                  const res = await fetch(api);
+                  if (!res.ok) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    return;
+                  }
+                  const data = await res.json();
+                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
+                  if (!files.length) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    reportDate.textContent = 'N/A';
+                    reportVersion.textContent = 'N/A';
+                    reportLink.href = '#';
+                    return;
+                  }
+                  list.innerHTML = '';
+                  files.forEach((f) => {
+                    const li = document.createElement('li');
+                    const btn = document.createElement('button');
+                    btn.className = 'report-btn';
+                    btn.textContent = f.name.replace('.md', '');
+                    btn.addEventListener('click', () => showReport(f));
+                    li.appendChild(btn);
+                    list.appendChild(li);
+                  });
+                  showReport(files[0]);
+                } catch (_e) {
+                  list.innerHTML = '<li>Failed to load reports.</li>';
+                  reportDate.textContent = 'N/A';
+                  reportVersion.textContent = 'N/A';
+                  reportLink.href = '#';
+                }
+              }
+
+              function setActive(key) {
+                for (const btn of tabs.querySelectorAll('button')) {
+                  btn.classList.toggle('active', btn.dataset.key === key);
+                }
+                loadSuite(key);
+              }
+
+              for (const suite of suites) {
+                const btn = document.createElement('button');
+                btn.textContent = suite.label;
+                btn.dataset.key = suite.key;
+                btn.addEventListener('click', () => setActive(suite.key));
+                tabs.appendChild(btn);
+              }
+              setActive('s3');
+            </script>
+          </body>
+          </html>
+          EOF
+
+          INDEX_PATH="functional/index.html"
+          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
+          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${INDEX_SHA}" ]; then
+            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          fi
+
+      - name: Upload report and logs
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-security-test-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-security-report.md
+            /tmp/rustfs-security.*/*
+          if-no-files-found: ignore
+          retention-days: 3
+
+      - name: Cleanup environment (after)
+        if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms
+            '
+          done
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS security test failed"
+          echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
+          echo "See the uploaded report and logs for details."

.github/workflows/rustfs-storage-test.yml

@@ -0,0 +1,422 @@
+name: RustFS Storage Engine Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      rustfs_version:
+        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        required: false
+        default: '1.0.0-rc.4-preview.1'
+      package_url:
+        description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
+        required: false
+        type: string
+      topology:
+        description: 'Topology to run (all = SNSD, SNMD, MNMD)'
+        type: choice
+        options:
+          - all
+          - single-single
+          - single-multi
+          - multi-multi
+        default: all
+  workflow_run:
+    # Strict shared-environment order: run after tier test completes.
+    workflows: ["RustFS Tier Test"]
+    types: [completed]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rustfs-shared-functional-tests
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  storage-test:
+    runs-on: smoke-testing
+    continue-on-error: true
+    timeout-minutes: 360
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          openssl version
+          df -h /data | tail -1
+
+      - name: Cleanup environment (before)
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
+            '
+          done
+
+      - name: Run storage engine suite
+        id: test
+        continue-on-error: true
+        env:
+          LOG_FILE: /tmp/rustfs-storage.log
+        run: |
+          set -euo pipefail
+          chmod +x auto-testing/rustfs-storage-test.sh
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          TOPOLOGY='${{ inputs.topology }}'
+          ARGS=(-y --log-file "${LOG_FILE}")
+          if [ "${TOPOLOGY}" = "all" ] || [ -z "${TOPOLOGY}" ] || [ "${TOPOLOGY}" = "null" ]; then
+            ARGS+=(--all-topologies)
+          else
+            ARGS+=(--topology "${TOPOLOGY}")
+          fi
+          if [ -n "${PACKAGE_URL}" ]; then
+            ARGS+=(--package-url "${PACKAGE_URL}")
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            ARGS+=(--version "${RUSTFS_VERSION}")
+          else
+            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
+          fi
+          ./auto-testing/rustfs-storage-test.sh "${ARGS[@]}"
+
+      - name: Generate report
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-storage.log
+          REPORT_FILE: /tmp/rustfs-storage-report.md
+        run: |
+          set -euo pipefail
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          RUSTFS_VERSION_INFO="N/A"
+          if [ "${#NODES[@]}" -gt 0 ]; then
+            DETECTED_VERSION="$(ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
+              "${SSH_USER}@${NODES[0]}" 'rustfs --version' 2>/dev/null | tr -d '\r' | head -n 1 || true)"
+            if [ -n "${DETECTED_VERSION}" ]; then
+              RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
+            fi
+          fi
+          CASE_TABLE="/tmp/rustfs-storage-cases.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
+
+          rows = []
+          index = {}
+          current = None
+          try:
+            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+              for raw in fh:
+                line = ansi.sub('', raw).strip()
+                m = start_re.match(line)
+                if m:
+                  case_id, name = m.group(1), m.group(2)
+                  current = case_id
+                  if case_id not in index:
+                    index[case_id] = len(rows)
+                    rows.append([case_id, name, 'RUNNING'])
+                  continue
+                m = done_re.match(line)
+                if m:
+                  status, case_id = m.group(1), m.group(2)
+                  if case_id in index:
+                    rows[index[case_id]][2] = status
+                  else:
+                    rows.append([case_id, case_id, status])
+                    index[case_id] = len(rows) - 1
+                  current = None
+          except FileNotFoundError:
+            rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+            counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+            out.write('## Case Summary\n\n')
+            out.write(f"- Total: {len(rows)}\\n")
+            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+            out.write('\\n')
+            out.write('| Case | Name | Status |\\n')
+            out.write('| --- | --- | --- |\\n')
+            for case_id, name, status in rows:
+              out.write(f'| {case_id} | {name} | {status} |\\n')
+          PY
+          {
+            echo "# RustFS storage engine test report"
+            echo ""
+            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Trigger: ${{ github.event_name }}"
+            echo "- Package: ${PACKAGE_SOURCE}"
+            echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo ""
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
+          } | tee "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload functional report to dashboard
+        if: always()
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-storage-report.md
+          SUITE: storage
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+
+          cat > /tmp/rustfs-functional-index.html <<'EOF'
+          <!doctype html>
+          <html lang="en">
+          <head>
+            <meta charset="utf-8" />
+            <meta name="viewport" content="width=device-width, initial-scale=1" />
+            <title>RustFS Functional Test Reports</title>
+            <style>
+              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
+              * { box-sizing: border-box; }
+              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
+              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
+              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
+              h1 { margin: 0 0 8px; font-size: 26px; }
+              p { margin: 0 0 14px; color: var(--muted); }
+              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
+              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
+              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
+              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
+              ul { list-style: none; margin: 0; padding: 0; }
+              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
+              a { color: var(--accent); text-decoration: none; }
+              a:hover { text-decoration: underline; }
+              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
+              .kv { margin: 6px 0; color: var(--text); }
+              .muted { color: var(--muted); }
+            </style>
+          </head>
+          <body>
+            <div class="wrap">
+              <div class="card">
+                <h1>RustFS Functional Test Reports</h1>
+                <p>Select a suite and date to view the build version used in that run.</p>
+                <div class="tabs" id="tabs"></div>
+                <ul id="list"></ul>
+                <div class="meta">
+                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
+                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
+                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
+                </div>
+              </div>
+            </div>
+            <script>
+              const suites = [
+                { key: 's3', label: 'S3 Compatibility' },
+                { key: 'kms', label: 'KMS' },
+                { key: 'tier', label: 'Tier' },
+                { key: 'storage', label: 'Storage Engine' },
+                { key: 'heal', label: 'Heal' },
+                { key: 'pool', label: 'Pool Expansion' },
+                { key: 'security', label: 'Security' },
+                { key: 'upgrade', label: 'Upgrade' },
+              ];
+              const tabs = document.getElementById('tabs');
+              const list = document.getElementById('list');
+              const reportDate = document.getElementById('report-date');
+              const reportVersion = document.getElementById('report-version');
+              const reportLink = document.getElementById('report-link');
+
+              function parseVersion(markdown) {
+                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
+                return m ? m[1].trim() : 'N/A';
+              }
+
+              async function showReport(report) {
+                reportDate.textContent = report.name.replace('.md', '');
+                reportVersion.textContent = 'Loading...';
+                reportLink.href = report.html_url;
+                try {
+                  const res = await fetch(report.download_url, { cache: 'no-store' });
+                  if (!res.ok) {
+                    reportVersion.textContent = 'N/A';
+                    return;
+                  }
+                  const text = await res.text();
+                  reportVersion.textContent = parseVersion(text);
+                } catch (_e) {
+                  reportVersion.textContent = 'N/A';
+                }
+              }
+
+              async function loadSuite(suite) {
+                list.innerHTML = '<li>Loading...</li>';
+                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
+                try {
+                  const res = await fetch(api);
+                  if (!res.ok) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    return;
+                  }
+                  const data = await res.json();
+                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
+                  if (!files.length) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    reportDate.textContent = 'N/A';
+                    reportVersion.textContent = 'N/A';
+                    reportLink.href = '#';
+                    return;
+                  }
+                  list.innerHTML = '';
+                  files.forEach((f) => {
+                    const li = document.createElement('li');
+                    const btn = document.createElement('button');
+                    btn.className = 'report-btn';
+                    btn.textContent = f.name.replace('.md', '');
+                    btn.addEventListener('click', () => showReport(f));
+                    li.appendChild(btn);
+                    list.appendChild(li);
+                  });
+                  showReport(files[0]);
+                } catch (_e) {
+                  list.innerHTML = '<li>Failed to load reports.</li>';
+                  reportDate.textContent = 'N/A';
+                  reportVersion.textContent = 'N/A';
+                  reportLink.href = '#';
+                }
+              }
+
+              function setActive(key) {
+                for (const btn of tabs.querySelectorAll('button')) {
+                  btn.classList.toggle('active', btn.dataset.key === key);
+                }
+                loadSuite(key);
+              }
+
+              for (const suite of suites) {
+                const btn = document.createElement('button');
+                btn.textContent = suite.label;
+                btn.dataset.key = suite.key;
+                btn.addEventListener('click', () => setActive(suite.key));
+                tabs.appendChild(btn);
+              }
+              setActive('s3');
+            </script>
+          </body>
+          </html>
+          EOF
+
+          INDEX_PATH="functional/index.html"
+          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
+          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${INDEX_SHA}" ]; then
+            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          fi
+
+      - name: Upload report and logs
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-storage-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-storage.log
+            /tmp/rustfs-storage-report.md
+          if-no-files-found: warn
+
+      - name: Cleanup environment (after)
+        if: always()
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs
+            '
+          done
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS storage engine suite failed"
+          echo "See the uploaded report and log artifacts for details."

.github/workflows/rustfs-tier-test.yml

@@ -0,0 +1,374 @@
+name: RustFS Tier Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      rustfs_version:
+        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        required: false
+        default: '1.0.0-rc.4-preview.1'
+      package_url:
+        description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
+        required: false
+        type: string
+  workflow_run:
+    # Strict shared-environment order: run after KMS test completes.
+    workflows: ["RustFS KMS Test"]
+    types: [completed]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rustfs-shared-functional-tests
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  tier-test:
+    runs-on: smoke-testing
+    continue-on-error: true
+    timeout-minutes: 420
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }}
+    steps:
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          openssl version
+          df -h /data | tail -1
+
+      - name: Cleanup environment (before)
+        run: |
+          set -euo pipefail
+          sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
+          sudo rm -f /tmp/rustfs-mosquitto.conf
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms
+            '
+          done
+
+      - name: Ensure MQTT broker + clients
+        run: |
+          set -euo pipefail
+          if ! command -v mosquitto_sub >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y mosquitto-clients
+          fi
+          command -v docker >/dev/null 2>&1 || { echo 'docker not found on runner'; exit 1; }
+          sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
+          cat <<'EOF' | sudo tee /tmp/rustfs-mosquitto.conf >/dev/null
+          listener 1883 0.0.0.0
+          allow_anonymous true
+          EOF
+          sudo docker run -d --name rustfs-test-mqtt -p 1883:1883 \
+            -v /tmp/rustfs-mosquitto.conf:/mosquitto/config/mosquitto.conf:ro \
+            eclipse-mosquitto:2 >/dev/null
+          for _ in {1..10}; do
+            if ss -tln 2>/dev/null | grep -q ':1883'; then
+              break
+            fi
+            sleep 1
+          done
+          ss -tln 2>/dev/null | grep -q ':1883' || {
+            echo 'mosquitto container is not listening on 1883'
+            sudo docker logs rustfs-test-mqtt || true
+            exit 1
+          }
+
+      - name: Run tier suite
+        id: test
+        continue-on-error: true
+        env:
+          LOG_FILE: /tmp/rustfs-tier.log
+        run: |
+          set -euo pipefail
+          chmod +x auto-testing/rustfs-tier-test.sh
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          ARGS=(--all-topologies -y --log-file "${LOG_FILE}")
+          if [ -n "${PACKAGE_URL}" ]; then
+            ARGS+=(--package-url "${PACKAGE_URL}")
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            ARGS+=(--version "${RUSTFS_VERSION}")
+          else
+            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
+          fi
+          ./auto-testing/rustfs-tier-test.sh "${ARGS[@]}"
+
+      - name: Generate report
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-tier.log
+          REPORT_FILE: /tmp/rustfs-tier-report.md
+        run: |
+          set -euo pipefail
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          CASE_TABLE="/tmp/rustfs-tier-cases.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
+
+          rows = []
+          index = {}
+          try:
+            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+              for raw in fh:
+                line = ansi.sub('', raw).strip()
+                m = start_re.match(line)
+                if m:
+                  case_id, name = m.group(1), m.group(2)
+                  if case_id not in index:
+                    index[case_id] = len(rows)
+                    rows.append([case_id, name, 'RUNNING'])
+                  continue
+                m = done_re.match(line)
+                if m:
+                  status, case_id = m.group(1), m.group(2)
+                  if case_id in index:
+                    rows[index[case_id]][2] = status
+                  else:
+                    rows.append([case_id, case_id, status])
+                    index[case_id] = len(rows) - 1
+          except FileNotFoundError:
+            rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+            counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+            out.write('## Case Summary\n\n')
+            out.write(f"- Total: {len(rows)}\\n")
+            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+            out.write('\\n')
+            out.write('| Case | Name | Status |\\n')
+            out.write('| --- | --- | --- |\\n')
+            for case_id, name, status in rows:
+              out.write(f'| {case_id} | {name} | {status} |\\n')
+          PY
+          {
+            echo "# RustFS tier test report"
+            echo ""
+            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Trigger: ${{ github.event_name }}"
+            echo "- Package: ${PACKAGE_SOURCE}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo ""
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
+          } | tee "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload functional report to dashboard
+        if: always()
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-tier-report.md
+          SUITE: tier
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+
+          cat > /tmp/rustfs-functional-index.html <<'EOF'
+          <!doctype html>
+          <html lang="en">
+          <head>
+            <meta charset="utf-8" />
+            <meta name="viewport" content="width=device-width, initial-scale=1" />
+            <title>RustFS Functional Test Reports</title>
+            <style>
+              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
+              * { box-sizing: border-box; }
+              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
+              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
+              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
+              h1 { margin: 0 0 8px; font-size: 26px; }
+              p { margin: 0 0 14px; color: var(--muted); }
+              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
+              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
+              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
+              ul { list-style: none; margin: 0; padding: 0; }
+              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
+              a { color: var(--accent); text-decoration: none; }
+              a:hover { text-decoration: underline; }
+            </style>
+          </head>
+          <body>
+            <div class="wrap">
+              <div class="card">
+                <h1>RustFS Functional Test Reports</h1>
+                <p>S3, KMS, Tier report tabs. Each tab lists reports by date.</p>
+                <div class="tabs" id="tabs"></div>
+                <ul id="list"></ul>
+              </div>
+            </div>
+            <script>
+              const suites = [
+                { key: 's3', label: 'S3 Compatibility' },
+                { key: 'kms', label: 'KMS' },
+                { key: 'tier', label: 'Tier' },
+                { key: 'heal', label: 'Heal' },
+                { key: 'pool', label: 'Pool Expansion' },
+                { key: 'security', label: 'Security' },
+                { key: 'upgrade', label: 'Upgrade' },
+              ];
+              const tabs = document.getElementById('tabs');
+              const list = document.getElementById('list');
+
+              async function loadSuite(suite) {
+                list.innerHTML = '<li>Loading...</li>';
+                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
+                try {
+                  const res = await fetch(api);
+                  if (!res.ok) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    return;
+                  }
+                  const data = await res.json();
+                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
+                  if (!files.length) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    return;
+                  }
+                  list.innerHTML = files.map(f => `<li><a href="${f.html_url}" target="_blank" rel="noreferrer">${f.name.replace('.md','')}</a></li>`).join('');
+                } catch (_e) {
+                  list.innerHTML = '<li>Failed to load reports.</li>';
+                }
+              }
+
+              function setActive(key) {
+                for (const btn of tabs.querySelectorAll('button')) {
+                  btn.classList.toggle('active', btn.dataset.key === key);
+                }
+                loadSuite(key);
+              }
+
+              for (const suite of suites) {
+                const btn = document.createElement('button');
+                btn.textContent = suite.label;
+                btn.dataset.key = suite.key;
+                btn.addEventListener('click', () => setActive(suite.key));
+                tabs.appendChild(btn);
+              }
+              setActive('s3');
+            </script>
+          </body>
+          </html>
+          EOF
+
+          INDEX_PATH="functional/index.html"
+          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
+          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${INDEX_SHA}" ]; then
+            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          fi
+
+      - name: Upload report and logs
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-tier-test-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-tier.log
+            /tmp/rustfs-tier-report.md
+          if-no-files-found: warn
+
+      - name: Cleanup environment (after)
+        if: always()
+        run: |
+          set -euo pipefail
+          sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
+          sudo rm -f /tmp/rustfs-mosquitto.conf
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms
+            '
+          done
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS tier suite failed"
+          echo "See the uploaded report and log artifacts for details."

.github/workflows/rustfs-upgrade-test.yml

@@ -0,0 +1,477 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: RustFS Upgrade Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      from_version:
+        description: 'OLD RustFS release tag (e.g. 1.0.0-rc.4-preview.1)'
+        required: false
+        default: '1.0.0-rc.4-preview.1'
+      from_url:
+        description: 'OLD .deb URL. Overrides from_version.'
+        required: false
+        type: string
+      to_version:
+        description: 'NEW RustFS release tag (leave empty for latest nightly)'
+        required: false
+      to_url:
+        description: 'NEW .deb URL. Overrides to_version / nightly default.'
+        required: false
+        type: string
+      topology:
+        description: 'Topology to run (all = SNSD, SNMD, MNMD)'
+        type: choice
+        options:
+          - all
+          - single-single
+          - single-multi
+          - multi-multi
+        default: all
+      backends:
+        description: 'KMS backends to run (local,vault-kv2)'
+        required: false
+        default: 'local,vault-kv2'
+      cleanup_before:
+        description: 'Reset the nodes before the test (DESTROYS existing data/config)'
+        type: boolean
+        default: true
+      cleanup_after:
+        description: 'Reset the nodes after the test (DESTROYS test data/config)'
+        type: boolean
+        default: true
+  workflow_run:
+    # Runs first in the functional chain: upgrade compatibility gates the
+    # nightly suites that follow (S3 -> KMS -> Tier -> Pool/Heal -> Security).
+    workflows: ["Nightly GNU Build"]
+    types: [completed]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rustfs-shared-functional-tests
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  upgrade-test:
+    runs-on: smoke-testing
+    continue-on-error: true
+    timeout-minutes: 420
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          openssl version
+          aws --version || true
+          docker --version || true
+          df -h /data | tail -1
+
+      - name: Cleanup environment (before)
+        if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
+            '
+          done
+
+      - name: Ensure docker (Vault container)
+        run: |
+          if ! command -v docker >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y docker.io
+          fi
+          sudo systemctl enable --now docker
+          docker info >/dev/null 2>&1 || sudo docker info >/dev/null 2>&1
+
+      - name: Run upgrade compatibility suite
+        id: test
+        continue-on-error: true
+        env:
+          LOG_FILE: /tmp/rustfs-upgrade.log
+        run: |
+          set -euo pipefail
+          chmod +x auto-testing/rustfs-upgrade-test.sh
+          FROM_URL='${{ inputs.from_url }}'
+          FROM_VERSION='${{ inputs.from_version }}'
+          TO_URL='${{ inputs.to_url }}'
+          TO_VERSION='${{ inputs.to_version }}'
+          TOPOLOGY='${{ inputs.topology }}'
+          BACKENDS='${{ inputs.backends }}'
+          ARGS=(-y --log-file "${LOG_FILE}")
+          if [ "${TOPOLOGY}" = "all" ] || [ -z "${TOPOLOGY}" ] || [ "${TOPOLOGY}" = "null" ]; then
+            ARGS+=(--all-topologies)
+          else
+            ARGS+=(--topology "${TOPOLOGY}")
+          fi
+          if [ -n "${BACKENDS}" ] && [ "${BACKENDS}" != "null" ]; then
+            ARGS+=(--backends "${BACKENDS}")
+          fi
+          if [ -n "${FROM_URL}" ]; then
+            ARGS+=(--from-url "${FROM_URL}")
+          elif [ -n "${FROM_VERSION}" ] && [ "${FROM_VERSION}" != "null" ]; then
+            ARGS+=(--from-version "${FROM_VERSION}")
+          fi
+          if [ -n "${TO_URL}" ]; then
+            ARGS+=(--to-url "${TO_URL}")
+          elif [ -n "${TO_VERSION}" ] && [ "${TO_VERSION}" != "null" ]; then
+            ARGS+=(--to-version "${TO_VERSION}")
+          else
+            ARGS+=(--to-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
+          fi
+          ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
+
+      - name: Generate report
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-upgrade.log
+          REPORT_FILE: /tmp/rustfs-upgrade-report.md
+        run: |
+          set -euo pipefail
+          FROM_URL='${{ inputs.from_url }}'
+          FROM_VERSION='${{ inputs.from_version }}'
+          TO_URL='${{ inputs.to_url }}'
+          TO_VERSION='${{ inputs.to_version }}'
+          if [ -n "${FROM_URL}" ]; then
+            FROM_SOURCE="${FROM_URL}"
+          elif [ -n "${FROM_VERSION}" ]; then
+            FROM_SOURCE="version ${FROM_VERSION}"
+          else
+            FROM_SOURCE="release (default)"
+          fi
+          if [ -n "${TO_URL}" ]; then
+            TO_SOURCE="${TO_URL}"
+          elif [ -n "${TO_VERSION}" ]; then
+            TO_SOURCE="version ${TO_VERSION}"
+          else
+            TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          CASE_TABLE="/tmp/rustfs-upgrade-cases.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
+
+          rows = []
+          index = {}
+          try:
+            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+              for raw in fh:
+                line = ansi.sub('', raw).strip()
+                m = start_re.match(line)
+                if m:
+                  case_id, name = m.group(1), m.group(2)
+                  if case_id not in index:
+                    index[case_id] = len(rows)
+                    rows.append([case_id, name, 'RUNNING'])
+                  continue
+                m = done_re.match(line)
+                if m:
+                  status, case_id = m.group(1), m.group(2)
+                  if case_id in index:
+                    rows[index[case_id]][2] = status
+                  else:
+                    rows.append([case_id, case_id, status])
+                    index[case_id] = len(rows) - 1
+          except FileNotFoundError:
+            rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+            counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+            out.write('## Case Summary\n\n')
+            out.write(f"- Total: {len(rows)}\\n")
+            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+            out.write('\\n')
+            out.write('| Case | Name | Status |\\n')
+            out.write('| --- | --- | --- |\\n')
+            for case_id, name, status in rows:
+              out.write(f'| {case_id} | {name} | {status} |\\n')
+          PY
+          {
+            echo "# RustFS upgrade compatibility report"
+            echo ""
+            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Trigger: ${{ github.event_name }}"
+            echo "- From: ${FROM_SOURCE}"
+            echo "- To: ${TO_SOURCE}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo ""
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
+          } | tee "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload functional report to dashboard
+        if: always()
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-upgrade-report.md
+          SUITE: upgrade
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+
+          cat > /tmp/rustfs-functional-index.html <<'EOF'
+          <!doctype html>
+          <html lang="en">
+          <head>
+            <meta charset="utf-8" />
+            <meta name="viewport" content="width=device-width, initial-scale=1" />
+            <title>RustFS Functional Test Reports</title>
+            <style>
+              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
+              * { box-sizing: border-box; }
+              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
+              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
+              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
+              h1 { margin: 0 0 8px; font-size: 26px; }
+              p { margin: 0 0 14px; color: var(--muted); }
+              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
+              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
+              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
+              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
+              ul { list-style: none; margin: 0; padding: 0; }
+              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
+              a { color: var(--accent); text-decoration: none; }
+              a:hover { text-decoration: underline; }
+              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
+              .kv { margin: 6px 0; color: var(--text); }
+              .muted { color: var(--muted); }
+            </style>
+          </head>
+          <body>
+            <div class="wrap">
+              <div class="card">
+                <h1>RustFS Functional Test Reports</h1>
+                <p>Select a suite and date to view the build version used in that run.</p>
+                <div class="tabs" id="tabs"></div>
+                <ul id="list"></ul>
+                <div class="meta">
+                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
+                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
+                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
+                </div>
+              </div>
+            </div>
+            <script>
+              const suites = [
+                { key: 'upgrade', label: 'Upgrade' },
+                { key: 's3', label: 'S3 Compatibility' },
+                { key: 'kms', label: 'KMS' },
+                { key: 'tier', label: 'Tier' },
+                { key: 'heal', label: 'Heal' },
+                { key: 'pool', label: 'Pool Expansion' },
+                { key: 'security', label: 'Security' },
+              ];
+              const tabs = document.getElementById('tabs');
+              const list = document.getElementById('list');
+              const reportDate = document.getElementById('report-date');
+              const reportVersion = document.getElementById('report-version');
+              const reportLink = document.getElementById('report-link');
+
+              function parseVersion(markdown) {
+                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
+                return m ? m[1].trim() : 'N/A';
+              }
+
+              async function showReport(report) {
+                reportDate.textContent = report.name.replace('.md', '');
+                reportVersion.textContent = 'Loading...';
+                reportLink.href = report.html_url;
+                try {
+                  const res = await fetch(report.download_url, { cache: 'no-store' });
+                  if (!res.ok) {
+                    reportVersion.textContent = 'N/A';
+                    return;
+                  }
+                  const text = await res.text();
+                  reportVersion.textContent = parseVersion(text);
+                } catch (_e) {
+                  reportVersion.textContent = 'N/A';
+                }
+              }
+
+              async function loadSuite(suite) {
+                list.innerHTML = '<li>Loading...</li>';
+                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
+                try {
+                  const res = await fetch(api);
+                  if (!res.ok) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    return;
+                  }
+                  const data = await res.json();
+                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
+                  if (!files.length) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    reportDate.textContent = 'N/A';
+                    reportVersion.textContent = 'N/A';
+                    reportLink.href = '#';
+                    return;
+                  }
+                  list.innerHTML = '';
+                  files.forEach((f) => {
+                    const li = document.createElement('li');
+                    const btn = document.createElement('button');
+                    btn.className = 'report-btn';
+                    btn.textContent = f.name.replace('.md', '');
+                    btn.addEventListener('click', () => showReport(f));
+                    li.appendChild(btn);
+                    list.appendChild(li);
+                  });
+                  showReport(files[0]);
+                } catch (_e) {
+                  list.innerHTML = '<li>Failed to load reports.</li>';
+                  reportDate.textContent = 'N/A';
+                  reportVersion.textContent = 'N/A';
+                  reportLink.href = '#';
+                }
+              }
+
+              function setActive(key) {
+                for (const btn of tabs.querySelectorAll('button')) {
+                  btn.classList.toggle('active', btn.dataset.key === key);
+                }
+                loadSuite(key);
+              }
+
+              for (const suite of suites) {
+                const btn = document.createElement('button');
+                btn.textContent = suite.label;
+                btn.dataset.key = suite.key;
+                btn.addEventListener('click', () => setActive(suite.key));
+                tabs.appendChild(btn);
+              }
+              setActive('upgrade');
+            </script>
+          </body>
+          </html>
+          EOF
+
+          INDEX_PATH="functional/index.html"
+          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
+          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${INDEX_SHA}" ]; then
+            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          fi
+
+      - name: Upload report and logs
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-upgrade-test-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-upgrade-report.md
+            /tmp/rustfs-upgrade.*/*
+          if-no-files-found: ignore
+          retention-days: 3
+
+      - name: Cleanup environment (after)
+        if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
+            '
+          done
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS upgrade compatibility test failed"
+          echo "From: ${{ inputs.from_url || inputs.from_version || 'release (default)' }}"
+          echo "To: ${{ inputs.to_url || inputs.to_version || 'nightly (R2 latest)' }}"
+          echo "See the uploaded report and logs for details."

AGENTS.md

@@ -31,8 +31,13 @@ This file contains repository-wide rules. Use the nearest subdirectory
 - An existing clean, isolated task worktree is sufficient. Create another
   worktree only when the current checkout is shared, dirty with unrelated work,
   or belongs to another task.
-- Never commit from a shared checkout. Use an `overtrue/` feature branch unless
+- Never commit from a shared checkout.
-  the user requests another name.
+- Use a task-specific branch named `<type>/<topic>`, such as `fix/...`,
+  `feat/...`, `test/...`, or `docs/...`, unless the user specifies a name.
+- Do not include agent, tool, contributor, account, or organization names in
+  branch names.
+- Push to the user-requested remote or the repository's configured push remote.
+  Do not hard-code or infer a remote from an account name.
 - Check free space before artifact-heavy builds, tests, coverage, or downloads.
   Re-check before a broad gate when space is tight.
 - Remove only task-owned temporary/build artifacts. Never delete another task's

Cargo.toml

@@ -72,7 +72,7 @@ edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
 rust-version = "1.97.1"
-version = "1.0.0-rc.4"
+version = "1.0.0-rc.5"
 homepage = "https://rustfs.com"
 description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
 keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
@@ -89,55 +89,55 @@ redundant_clone = "warn"
 
 [workspace.dependencies]
 # RustFS Internal Crates
-rustfs = { path = "./rustfs", version = "1.0.0-rc.4" }
+rustfs = { path = "./rustfs", version = "1.0.0-rc.5" }
-rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.4" }
+rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.5" }
-rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.4" }
+rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.5" }
-rustfs-scanner-contracts = { path = "crates/scanner-contracts", version = "1.0.0-rc.4" }
+rustfs-scanner-contracts = { path = "crates/scanner-contracts", version = "1.0.0-rc.5" }
-rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.4" }
+rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.5" }
-rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.4" }
+rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.5" }
-rustfs-common = { path = "crates/common", version = "1.0.0-rc.4" }
+rustfs-common = { path = "crates/common", version = "1.0.0-rc.5" }
-rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.4" }
+rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.5" }
-rustfs-config = { path = "./crates/config", version = "1.0.0-rc.4" }
+rustfs-config = { path = "./crates/config", version = "1.0.0-rc.5" }
-rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.4" }
+rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.5" }
-rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.4" }
+rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.5" }
-rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.4" }
+rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.5" }
-rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.4" }
+rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.5" }
-rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.4" }
+rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.5" }
-rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.4" }
+rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.5" }
-rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.4" }
+rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.5" }
-rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.4" }
+rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.5" }
-rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.4" }
+rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.5" }
-rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.4" }
+rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.5" }
-rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.4" }
+rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.5" }
-rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.4" }
+rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.5" }
-rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.4" }
+rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.5" }
-rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.4" }
+rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.5" }
-rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.4" }
+rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.5" }
-rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.4", default-features = false }
+rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.5", default-features = false }
-rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.4" }
+rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.5" }
-rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.4" }
+rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.5" }
-rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.4" }
+rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.5" }
-rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.4" }
+rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.5" }
-rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.4" }
+rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.5" }
-rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.4" }
+rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.5" }
-rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.4" }
+rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.5" }
-rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.4" }
+rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.5" }
-rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.4" }
+rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.5" }
-rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.4" }
+rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.5" }
-rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.4" }
+rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.5" }
-rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.4" }
+rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.5" }
-rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.4" }
+rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.5" }
-rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.4" }
+rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.5" }
-rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.4" }
+rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.5" }
-rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.4" }
+rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.5" }
-rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.4" }
+rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.5" }
-rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.4" }
+rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.5" }
-rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.4" }
+rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.5" }
-rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.4" }
+rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.5" }
-rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.4" }
+rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.5" }
-rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.4" }
+rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.5" }
-rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.4" }
+rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.5" }
-rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.4" }
+rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.5" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
@@ -155,7 +155,7 @@ futures-util = "0.3.34"
 pollster = "1.0.1"
 pulsar = { default-features = false, version = "6.9.0" }
 lapin = { default-features = false, version = "4.10.0" }
-hyper = { version = "1.11.0" }
+hyper = { version = "1.11.1" }
 hyper-rustls = { default-features = false, version = "0.27.9" }
 hyper-util = { version = "0.1.20" }
 http = "1.5.0"
@@ -198,7 +198,7 @@ serde_urlencoded = "0.7.1"
 # have incompatible APIs. Keep them exact-pinned and monitor upstream for stable
 # releases.
 aes-gcm = { version = "=0.11.1" }
-argon2 = { version = "=0.6.0-rc.8" }
+argon2 = { version = "=0.6.0" }
 blake2 = "=0.11.0"
 chacha20poly1305 = { version = "=0.11.0" }
 crc-fast = "1.10.0"
@@ -232,7 +232,8 @@ tokio-postgres-rustls = "0.14.0"
 # Utilities and Tools
 anyhow = "1.0.104"
 arc-swap = "1.9.2"
-astral-tokio-tar = "0.6.4"
+# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until bounded extension parsing is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published tokio-tar release exposes the extension limits used here.
+astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" }
 atoi = "3.1.0"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.11.0" }
@@ -247,7 +248,7 @@ base64-simd = "0.8.0"
 brotli = "8.0.4"
 clap = { version = "4.6.6" }
 const-str = { version = "1.1.0" }
-convert_case = "0.11.0"
+convert_case = "0.12.0"
 criterion = { version = "0.8" }
 crossbeam-queue = "0.3.13"
 crossbeam-channel = "0.5.16"
@@ -257,7 +258,7 @@ datafusion = { default-features = false, version = "55.0.0" }
 derive_builder = "0.20.2"
 enumset = "1.1.14"
 faster-hex = "0.10.0"
-flate2 = "1.1.9"
+flate2 = "1.1.10"
 glob = "0.3.4"
 google-cloud-storage = "1.18.0"
 google-cloud-auth = "1.16.0"
@@ -304,7 +305,7 @@ rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.4" }
 rust-embed = { version = "8.12.0" }
 rustc-hash = { version = "2.1.3" }
-s3s = { git = "https://github.com/rustfs/s3s.git", rev = "0f6f83d98b37fd9edcaa3be573db4aa8f568e088", version = "0.15.0", features = ["minio"] }
+s3s = { git = "https://github.com/rustfs/s3s.git", rev = "9c4690d8e73fc8d184031a19b2c4539ebc77d180", version = "0.15.0", features = ["minio"] }
 serial_test = "4.0.1"
 shadow-rs = { default-features = false, version = "2.0.0" }
 siphasher = "1.0.3"
@@ -355,7 +356,7 @@ pyroscope = { version = "2.1.1" }
 libunftp = { version = "0.23.0" }
 unftp-core = "0.1.0"
 suppaftp = { version = "10.0.2" }
-rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
+rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
 russh = { version = "0.63.1" }
 russh-sftp = "2.4.0"
 

README.md

@@ -16,7 +16,7 @@
 </p>
 
 <p align="center">
-  <a href="https://docs.rustfs.com/installation/">Getting Started</a>
+  <a href="https://docs.rustfs.com/en/installation">Getting Started</a>
   · <a href="https://docs.rustfs.com/">Docs</a>
   · <a href="https://github.com/rustfs/rustfs/issues">Bug reports</a>
   · <a href="https://github.com/rustfs/rustfs/discussions">Discussions</a>
@@ -115,7 +115,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
 # Using specific version
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.4
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.5
 ```
 
 If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
@@ -245,6 +245,26 @@ nix build
 nix run
 ```
 
+The flake also exports a NixOS module and the RustFS `rc` client. Add the
+module to your system and provide credentials through runtime files (for
+example, sops-nix or agenix) so secrets are never stored in the Nix store:
+
+```nix
+imports = [ inputs.rustfs.nixosModules.rustfs ];
+
+services.rustfs = {
+  enable = true;
+  accessKeyFile = "/run/secrets/rustfs-access-key";
+  secretKeyFile = "/run/secrets/rustfs-secret-key";
+  volumes = [ "/var/lib/rustfs" ];
+};
+```
+
+Install the S3-compatible client with
+`nix profile install github:rustfs/rustfs#rustfs-client` (the executable is named
+`rc`), or use `inputs.rustfs.packages.${pkgs.system}.rustfs-client` in a system
+configuration.
+
 ### 6\. X-CMD (Option 6)
 
 If you are an [x-cmd](https://www.x-cmd.com/install/rustfs) user:

README_ZH.md

@@ -16,7 +16,7 @@
 </p>
 
 <p align="center">
-  <a href="https://docs.rustfs.com/installation/">快速开始</a>
+  <a href="https://docs.rustfs.com/zh/installation">快速开始</a>
   · <a href="https://docs.rustfs.com/">文档</a>
   · <a href="https://github.com/rustfs/rustfs/issues">报告 Bug</a>
   · <a href="https://github.com/rustfs/rustfs/discussions">社区讨论</a>
@@ -112,7 +112,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
 # 使用指定版本运行
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.4
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.5
 ```
 
 如果您通过绑定挂载启用 TLS 证书目录，也请用同样方式准备该目录：
@@ -191,6 +191,12 @@ nix build
 nix run
 ```
 
+该 Flake 同时提供 NixOS 模块和 RustFS `rc` 客户端。将
+`inputs.rustfs.nixosModules.rustfs` 加入 `imports`，并通过运行时密钥文件
+（例如 sops-nix 或 agenix）配置 `accessKeyFile` 与 `secretKeyFile`，避免密钥
+进入 Nix store。客户端包为
+`inputs.rustfs.packages.${pkgs.system}.rustfs-client`，安装后的命令名为 `rc`。
+
 ### 6\. X-CMD (Option 6)
 
 如果你是 [x-cmd](https://www.x-cmd.com/install/rustfs) 用户：

crates/concurrency/src/workload.rs

@@ -178,6 +178,76 @@ pub trait WorkloadAdmissionSnapshotProvider {
     fn workload_admission_snapshot(&self) -> WorkloadAdmissionRegistrySnapshot;
 }
 
+/// Foreground workload pressure observed against a configured utilization threshold.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ForegroundPressure {
+    /// Foreground workload class whose utilization reached its threshold.
+    pub class: WorkloadClass,
+    /// Observed utilization percentage for the class.
+    pub usage_pct: usize,
+    /// Configured threshold percentage that the observed utilization reached.
+    pub threshold_pct: usize,
+}
+
+impl ForegroundPressure {
+    /// Return a stable reason label for logs and metrics.
+    pub const fn reason(self) -> &'static str {
+        match self.class {
+            WorkloadClass::ForegroundRead => "foreground_read_pressure",
+            WorkloadClass::ForegroundWrite => "foreground_write_pressure",
+            _ => "foreground_pressure",
+        }
+    }
+}
+
+/// Return the strongest foreground pressure in `snapshot`, if any.
+///
+/// A zero threshold disables its class. `Saturated` counts as full utilization
+/// regardless of the reported limit; otherwise a class contributes only when it
+/// reports a non-zero limit, with a missing active count read as zero. When both
+/// classes are above their threshold the higher utilization wins.
+///
+/// Callers own the enable switch: this function evaluates thresholds only.
+pub fn foreground_pressure(
+    snapshot: &WorkloadAdmissionRegistrySnapshot,
+    read_threshold_pct: usize,
+    write_threshold_pct: usize,
+) -> Option<ForegroundPressure> {
+    [
+        (WorkloadClass::ForegroundRead, read_threshold_pct),
+        (WorkloadClass::ForegroundWrite, write_threshold_pct),
+    ]
+    .into_iter()
+    .filter_map(|(class, threshold_pct)| {
+        if threshold_pct == 0 {
+            return None;
+        }
+
+        let entry = snapshot.get(class)?;
+        let usage_pct = if matches!(entry.state, AdmissionState::Saturated) {
+            100
+        } else {
+            let limit = entry.limit?;
+            if limit == 0 {
+                return None;
+            }
+            entry
+                .active
+                .unwrap_or(0)
+                .saturating_mul(100)
+                .checked_div(limit)
+                .unwrap_or(100)
+        };
+
+        (usage_pct >= threshold_pct).then_some(ForegroundPressure {
+            class,
+            usage_pct,
+            threshold_pct,
+        })
+    })
+    .max_by_key(|pressure| pressure.usage_pct)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -314,4 +384,205 @@ mod tests {
 
         assert!(err.to_string().contains("unexpected"));
     }
+
+    fn counted(
+        class: WorkloadClass,
+        state: AdmissionState,
+        active: Option<usize>,
+        limit: Option<usize>,
+    ) -> WorkloadAdmissionSnapshot {
+        WorkloadAdmissionSnapshot::new(class, state).with_counts(active, None, limit)
+    }
+
+    fn registry(entries: Vec<WorkloadAdmissionSnapshot>) -> WorkloadAdmissionRegistrySnapshot {
+        WorkloadAdmissionRegistrySnapshot::new(entries)
+    }
+
+    #[test]
+    fn foreground_pressure_reason_labels_cover_non_foreground_classes() {
+        let read = ForegroundPressure {
+            class: WorkloadClass::ForegroundRead,
+            usage_pct: 90,
+            threshold_pct: 80,
+        };
+        let write = ForegroundPressure {
+            class: WorkloadClass::ForegroundWrite,
+            usage_pct: 90,
+            threshold_pct: 80,
+        };
+        let repair = ForegroundPressure {
+            class: WorkloadClass::Repair,
+            usage_pct: 90,
+            threshold_pct: 80,
+        };
+
+        assert_eq!(read.reason(), "foreground_read_pressure");
+        assert_eq!(write.reason(), "foreground_write_pressure");
+        assert_eq!(repair.reason(), "foreground_pressure");
+    }
+
+    #[test]
+    fn foreground_pressure_is_disabled_when_both_thresholds_are_zero() {
+        let snapshot = registry(vec![
+            counted(WorkloadClass::ForegroundRead, AdmissionState::Saturated, Some(8), Some(8)),
+            counted(WorkloadClass::ForegroundWrite, AdmissionState::Saturated, Some(8), Some(8)),
+        ]);
+
+        assert_eq!(foreground_pressure(&snapshot, 0, 0), None);
+    }
+
+    #[test]
+    fn foreground_pressure_skips_only_the_class_whose_threshold_is_zero() {
+        let snapshot = registry(vec![
+            counted(WorkloadClass::ForegroundRead, AdmissionState::Open, Some(10), Some(10)),
+            counted(WorkloadClass::ForegroundWrite, AdmissionState::Open, Some(9), Some(10)),
+        ]);
+
+        assert_eq!(
+            foreground_pressure(&snapshot, 0, 80),
+            Some(ForegroundPressure {
+                class: WorkloadClass::ForegroundWrite,
+                usage_pct: 90,
+                threshold_pct: 80,
+            })
+        );
+        assert_eq!(
+            foreground_pressure(&snapshot, 80, 0),
+            Some(ForegroundPressure {
+                class: WorkloadClass::ForegroundRead,
+                usage_pct: 100,
+                threshold_pct: 80,
+            })
+        );
+    }
+
+    #[test]
+    fn foreground_pressure_ignores_missing_entries() {
+        let snapshot = registry(vec![counted(WorkloadClass::Scanner, AdmissionState::Saturated, Some(8), Some(8))]);
+
+        assert_eq!(foreground_pressure(&snapshot, 1, 1), None);
+    }
+
+    #[test]
+    fn foreground_pressure_ignores_missing_and_zero_limits() {
+        let missing_limit = registry(vec![counted(
+            WorkloadClass::ForegroundRead,
+            AdmissionState::Throttled,
+            Some(8),
+            None,
+        )]);
+        let zero_limit = registry(vec![counted(
+            WorkloadClass::ForegroundWrite,
+            AdmissionState::Throttled,
+            Some(8),
+            Some(0),
+        )]);
+
+        assert_eq!(foreground_pressure(&missing_limit, 1, 1), None);
+        assert_eq!(foreground_pressure(&zero_limit, 1, 1), None);
+    }
+
+    #[test]
+    fn foreground_pressure_treats_saturated_as_full_without_reading_limit() {
+        let snapshot = registry(vec![
+            counted(WorkloadClass::ForegroundRead, AdmissionState::Saturated, None, None),
+            counted(WorkloadClass::ForegroundWrite, AdmissionState::Saturated, Some(0), Some(0)),
+        ]);
+
+        assert_eq!(
+            foreground_pressure(&snapshot, 100, 0),
+            Some(ForegroundPressure {
+                class: WorkloadClass::ForegroundRead,
+                usage_pct: 100,
+                threshold_pct: 100,
+            })
+        );
+        assert_eq!(
+            foreground_pressure(&snapshot, 0, 100),
+            Some(ForegroundPressure {
+                class: WorkloadClass::ForegroundWrite,
+                usage_pct: 100,
+                threshold_pct: 100,
+            })
+        );
+    }
+
+    #[test]
+    fn foreground_pressure_reads_missing_active_as_zero() {
+        let snapshot = registry(vec![counted(WorkloadClass::ForegroundRead, AdmissionState::Open, None, Some(8))]);
+
+        assert_eq!(foreground_pressure(&snapshot, 1, 1), None);
+    }
+
+    #[test]
+    fn foreground_pressure_returns_the_higher_utilization_when_both_classes_exceed() {
+        let read_higher = registry(vec![
+            counted(WorkloadClass::ForegroundRead, AdmissionState::Open, Some(19), Some(20)),
+            counted(WorkloadClass::ForegroundWrite, AdmissionState::Open, Some(17), Some(20)),
+        ]);
+        let write_higher = registry(vec![
+            counted(WorkloadClass::ForegroundRead, AdmissionState::Open, Some(17), Some(20)),
+            counted(WorkloadClass::ForegroundWrite, AdmissionState::Open, Some(19), Some(20)),
+        ]);
+
+        assert_eq!(
+            foreground_pressure(&read_higher, 80, 80),
+            Some(ForegroundPressure {
+                class: WorkloadClass::ForegroundRead,
+                usage_pct: 95,
+                threshold_pct: 80,
+            })
+        );
+        assert_eq!(
+            foreground_pressure(&write_higher, 80, 80),
+            Some(ForegroundPressure {
+                class: WorkloadClass::ForegroundWrite,
+                usage_pct: 95,
+                threshold_pct: 80,
+            })
+        );
+    }
+
+    #[test]
+    fn foreground_pressure_breaks_utilization_ties_toward_the_write_class() {
+        let snapshot = registry(vec![
+            counted(WorkloadClass::ForegroundRead, AdmissionState::Open, Some(18), Some(20)),
+            counted(WorkloadClass::ForegroundWrite, AdmissionState::Open, Some(18), Some(20)),
+        ]);
+
+        assert_eq!(
+            foreground_pressure(&snapshot, 80, 80),
+            Some(ForegroundPressure {
+                class: WorkloadClass::ForegroundWrite,
+                usage_pct: 90,
+                threshold_pct: 80,
+            })
+        );
+    }
+
+    #[test]
+    fn foreground_pressure_triggers_exactly_at_the_threshold_and_not_below() {
+        let at_threshold = registry(vec![counted(
+            WorkloadClass::ForegroundRead,
+            AdmissionState::Open,
+            Some(8),
+            Some(10),
+        )]);
+        let below_threshold = registry(vec![counted(
+            WorkloadClass::ForegroundRead,
+            AdmissionState::Open,
+            Some(7),
+            Some(10),
+        )]);
+
+        assert_eq!(
+            foreground_pressure(&at_threshold, 80, 80),
+            Some(ForegroundPressure {
+                class: WorkloadClass::ForegroundRead,
+                usage_pct: 80,
+                threshold_pct: 80,
+            })
+        );
+        assert_eq!(foreground_pressure(&below_threshold, 80, 80), None);
+    }
 }

crates/config/src/constants/capacity.rs

@@ -59,20 +59,20 @@ pub const ENV_CAPACITY_MAX_TIMEOUT: &str = "RUSTFS_CAPACITY_MAX_TIMEOUT";
 // ============================================================================
 
 /// Scheduled update interval in seconds
-/// Default: 120 seconds (2 minutes)
+/// Default: 600 seconds (10 minutes)
-pub const DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS: u64 = 120;
+pub const DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS: u64 = 600;
 
 /// Write trigger delay in seconds
-/// Default: 5 seconds
+/// Default: 30 seconds
-pub const DEFAULT_WRITE_TRIGGER_DELAY_SECS: u64 = 5;
+pub const DEFAULT_WRITE_TRIGGER_DELAY_SECS: u64 = 30;
 
 /// Write frequency threshold (writes per minute)
-/// Default: 5 writes/minute
+/// Default: 20 writes/minute
-pub const DEFAULT_WRITE_FREQUENCY_THRESHOLD: usize = 5;
+pub const DEFAULT_WRITE_FREQUENCY_THRESHOLD: usize = 20;
 
 /// Fast update threshold in seconds
-/// Default: 30 seconds
+/// Default: 120 seconds
-pub const DEFAULT_FAST_UPDATE_THRESHOLD_SECS: u64 = 30;
+pub const DEFAULT_FAST_UPDATE_THRESHOLD_SECS: u64 = 120;
 
 /// Maximum files threshold for sampling
 /// Default: 200,000 files
@@ -129,4 +129,16 @@ mod tests {
         assert_eq!(ENV_CAPACITY_MIN_TIMEOUT, "RUSTFS_CAPACITY_MIN_TIMEOUT");
         assert_eq!(ENV_CAPACITY_MAX_TIMEOUT, "RUSTFS_CAPACITY_MAX_TIMEOUT");
     }
+
+    #[test]
+    fn test_capacity_default_values() {
+        assert_eq!(DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS, 600);
+        assert_eq!(DEFAULT_WRITE_TRIGGER_DELAY_SECS, 30);
+        assert_eq!(DEFAULT_WRITE_FREQUENCY_THRESHOLD, 20);
+        assert_eq!(DEFAULT_FAST_UPDATE_THRESHOLD_SECS, 120);
+        assert_eq!(DEFAULT_MAX_FILES_THRESHOLD, 200_000);
+        assert_eq!(DEFAULT_STAT_TIMEOUT_SECS, 3);
+        assert_eq!(DEFAULT_SAMPLE_RATE, 200);
+        assert_eq!(DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, 600);
+    }
 }

crates/config/src/constants/object.rs

@@ -297,7 +297,7 @@ const _: () = assert!(!DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE);
 pub const ENV_PUT_LARGE_FOREGROUND_ADMISSION_ENABLE: &str = "RUSTFS_PUT_LARGE_FOREGROUND_ADMISSION_ENABLE";
 pub const DEFAULT_PUT_LARGE_FOREGROUND_ADMISSION_ENABLE: bool = true;
 
-/// Maximum large foreground PutObject requests admitted concurrently per process.
+/// Maximum automatic foreground write requests admitted concurrently per process.
 ///
 /// `0` derives a conservative default from the local disk-read scheduler cap,
 /// currently clamped to protect the commit path without making ordinary high
@@ -305,14 +305,24 @@ pub const DEFAULT_PUT_LARGE_FOREGROUND_ADMISSION_ENABLE: bool = true;
 pub const ENV_PUT_LARGE_FOREGROUND_ADMISSION_LIMIT: &str = "RUSTFS_PUT_LARGE_FOREGROUND_ADMISSION_LIMIT";
 pub const DEFAULT_PUT_LARGE_FOREGROUND_ADMISSION_LIMIT: usize = 0;
 
-/// Minimum object size that enters automatic large PutObject admission.
+/// Minimum direct PutObject size that enters automatic foreground write admission.
 ///
 /// Requests with an unknown size are treated as large because the write pressure
 /// cannot be bounded from headers.
 pub const ENV_PUT_LARGE_FOREGROUND_ADMISSION_MIN_SIZE_BYTES: &str = "RUSTFS_PUT_LARGE_FOREGROUND_ADMISSION_MIN_SIZE_BYTES";
 pub const DEFAULT_PUT_LARGE_FOREGROUND_ADMISSION_MIN_SIZE_BYTES: usize = 32 * 1024 * 1024;
 
-/// Time in milliseconds a large foreground PutObject waits for a permit.
+/// Minimum UploadPart size that enters automatic foreground write admission.
+///
+/// Multipart pressure is often many moderate-sized parts rather than one very
+/// large request. The default gates every multipart part through the same permit
+/// pool as large/unknown-size PutObject while keeping small direct PUTs on the
+/// legacy path.
+pub const ENV_PUT_MULTIPART_FOREGROUND_ADMISSION_MIN_SIZE_BYTES: &str =
+    "RUSTFS_PUT_MULTIPART_FOREGROUND_ADMISSION_MIN_SIZE_BYTES";
+pub const DEFAULT_PUT_MULTIPART_FOREGROUND_ADMISSION_MIN_SIZE_BYTES: usize = 0;
+
+/// Time in milliseconds an automatic foreground write waits for a permit.
 ///
 /// A short wait smooths transient bursts while still returning S3
 /// `SlowDown`/503 before body ingest when the node is already saturated.

crates/config/src/constants/scanner.rs

@@ -198,11 +198,11 @@ pub const ENV_SCANNER_IDLE_MODE: &str = "RUSTFS_SCANNER_IDLE_MODE";
 /// Environment variable that controls scanner cache save timeout in seconds.
 /// The scanner enforces a minimum value of `1`.
 /// - Unit: seconds (u64).
-/// - Example: `export RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS=30`
+/// - Example: `export RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS=14`
 pub const ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS: &str = "RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS";
 
 /// Default scanner cache save timeout in seconds.
-pub const DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS: u64 = 30;
+pub const DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS: u64 = 14;
 
 /// Environment variable that caps concurrent scanner set tasks.
 /// A value of `0` keeps the existing topology-based concurrency.

crates/e2e_test/Cargo.toml

@@ -100,7 +100,8 @@ aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a",
 aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
 aws-config = { workspace = true }
 aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
-async-compression = { workspace = true, features = ["tokio", "bzip2", "xz"] }
+aws-smithy-types.workspace = true
+async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
 async-trait = { workspace = true }
 flate2.workspace = true
 http.workspace = true
@@ -114,6 +115,7 @@ rustfs-signer.workspace = true
 # server's implementation: a shared helper could agree with a bug on both sides.
 data-encoding = { workspace = true }
 hmac = { workspace = true }
+minlz.workspace = true
 sha1 = { workspace = true }
 serde_urlencoded = { workspace = true }
 tracing = { workspace = true }

crates/e2e_test/src/admin_auth_test.rs

@@ -31,14 +31,9 @@
 
 #[cfg(test)]
 mod tests {
-    use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, rustfs_binary_path};
+    use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
-    use aws_sdk_s3::config::{Credentials, Region};
+    use aws_sdk_s3::Client;
     use aws_sdk_s3::error::ProvideErrorMetadata;
-    use aws_sdk_s3::{Client, Config};
-    use http::header::HOST;
-    use rustfs_signer::constants::UNSIGNED_PAYLOAD;
-    use rustfs_signer::sign_v4;
-    use s3s::Body;
     use std::error::Error;
     use std::io::Read;
     use std::process::{Command, Stdio};
@@ -87,10 +82,10 @@ mod tests {
     }
 
     /// Send a SigV4-signed request to `path` (optionally with a JSON `body`) and
-    /// return `(status, body)`. Uses the `UNSIGNED_PAYLOAD` content hash so a
+    /// return `(status, body)`.
-    /// request body can be attached without the caller pre-hashing it — the
+    ///
-    /// server verifies the signature against the same sentinel, exactly as the
+    /// Thin wrapper over [`crate::common::admin_request`], kept local so the
-    /// AWS SDKs / MinIO client do for streaming/unsigned payloads.
+    /// call sites below keep their `Option<&str>` body shape.
     async fn signed_request(
         base_url: &str,
         method: http::Method,
@@ -99,47 +94,13 @@ mod tests {
         access_key: &str,
         secret_key: &str,
     ) -> Result<(reqwest::StatusCode, String), Box<dyn Error + Send + Sync>> {
-        let url = format!("{base_url}{path}");
+        crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await
-        let uri = url.parse::<http::Uri>()?;
-        let authority = uri.authority().ok_or("missing authority")?.to_string();
-        let body_bytes = body.map(|b| b.as_bytes().to_vec()).unwrap_or_default();
-
-        // The signature is computed over `UNSIGNED_PAYLOAD`, so the body bytes do
-        // not participate in the SigV4 hash — sign over an empty body and attach
-        // the real payload to the wire request below.
-        let request = http::Request::builder()
-            .method(method.clone())
-            .uri(uri)
-            .header(HOST, authority)
-            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-        let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
-
-        let client = local_http_client();
-        let mut rb = client.request(method, url.as_str());
-        for (name, value) in signed.headers() {
-            rb = rb.header(name, value);
-        }
-        if !body_bytes.is_empty() {
-            rb = rb.body(body_bytes);
-        }
-        let resp = rb.send().await?;
-        let status = resp.status();
-        let text = resp.text().await?;
-        Ok((status, text))
     }
 
     /// Build an S3 client bound to explicit credentials (used to exercise the S3
     /// data plane with rotated / stale root credentials).
     fn s3_client_with(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
-        let credentials = Credentials::new(access_key, secret_key, None, None, "sec4-admin-auth");
+        env.create_s3_client_with_credentials(access_key, secret_key)
-        let config = Config::builder()
-            .credentials_provider(credentials)
-            .region(Region::new("us-east-1"))
-            .endpoint_url(&env.url)
-            .force_path_style(true)
-            .behavior_version_latest()
-            .build();
-        Client::from_conf(config)
     }
 
     /// Create a non-admin IAM user via the admin `add-user` API using the root
@@ -151,12 +112,7 @@ mod tests {
         access_key: &str,
         secret_key: &str,
     ) -> Result<(), Box<dyn Error + Send + Sync>> {
-        let path = format!("/rustfs/admin/v3/add-user?accessKey={access_key}");
+        crate::common::admin_create_user(env, access_key, secret_key).await
-        let body = serde_json::json!({ "secretKey": secret_key, "status": "enabled" }).to_string();
-        let (status, resp) =
-            signed_request(&env.url, http::Method::PUT, &path, Some(&body), &env.access_key, &env.secret_key).await?;
-        assert!(status.is_success(), "add-user should succeed (status={status}, body={resp})");
-        Ok(())
     }
 
     /// A fully authenticated but non-admin credential must be rejected with

crates/e2e_test/src/admin_mfa_test.rs

@@ -59,8 +59,8 @@ mod tests {
 
     /// One signed admin request, returning the status and the raw body.
     ///
-    /// Signs with `UNSIGNED_PAYLOAD` so the body does not participate in the
+    /// Thin wrapper over [`crate::common::admin_request`], kept local so the
-    /// hash, matching how the other admin e2e tests drive these routes.
+    /// call sites below keep their `Option<&str>` body shape.
     async fn signed_request(
         base_url: &str,
         method: http::Method,
@@ -69,30 +69,7 @@ mod tests {
         access_key: &str,
         secret_key: &str,
     ) -> Result<(reqwest::StatusCode, String), Box<dyn Error + Send + Sync>> {
-        let url = format!("{base_url}{path}");
+        crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await
-        let uri = url.parse::<http::Uri>()?;
-        let authority = uri.authority().ok_or("missing authority")?.to_string();
-        let body_bytes = body.map(|b| b.as_bytes().to_vec()).unwrap_or_default();
-
-        let request = http::Request::builder()
-            .method(method.clone())
-            .uri(uri)
-            .header(HOST, authority)
-            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-        let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
-
-        let client = local_http_client();
-        let mut builder = client.request(method, url.as_str());
-        for (name, value) in signed.headers() {
-            builder = builder.header(name, value);
-        }
-        if !body_bytes.is_empty() {
-            builder = builder.body(body_bytes);
-        }
-        let response = builder.send().await?;
-        let status = response.status();
-        let text = response.text().await?;
-        Ok((status, text))
     }
 
     /// A SigV4-signed `AssumeRole` form POST, optionally carrying a second factor.

crates/e2e_test/src/bucket_policy_check_test.rs

@@ -15,39 +15,23 @@
 //! Regression test for Issue #1423
 //! Verifies that Bucket Policies are honored for Authenticated Users.
 
-use crate::common::{RustFSTestEnvironment, init_logging};
+use crate::common::{AdminTransport, RustFSTestEnvironment, admin_create_user_via, init_logging};
-use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::Client;
 use aws_sdk_s3::error::ProvideErrorMetadata;
-use aws_sdk_s3::{Client, Config};
 use tracing::info;
 
+/// This suite deliberately drives the admin API through the external `awscurl`
+/// binary, so user creation pins `AdminTransport::Awscurl`.
 async fn create_user(
     env: &RustFSTestEnvironment,
     username: &str,
     password: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let create_user_body = serde_json::json!({
+    admin_create_user_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, username, password).await
-        "secretKey": password,
-        "status": "enabled"
-    })
-    .to_string();
-
-    let create_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
-    crate::common::awscurl_put(&create_user_url, &create_user_body, &env.access_key, &env.secret_key).await?;
-    Ok(())
 }
 
 fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
-    let credentials = Credentials::new(access_key, secret_key, None, None, "test-user");
+    env.create_s3_client_with_credentials(access_key, secret_key)
-    let config = Config::builder()
-        .credentials_provider(credentials)
-        .region(Region::new("us-east-1"))
-        .endpoint_url(&env.url)
-        .force_path_style(true)
-        .behavior_version_latest()
-        .build();
-
-    Client::from_conf(config)
 }
 
 #[tokio::test]

crates/e2e_test/src/cluster_multidrive_pool_test.rs

@@ -27,8 +27,10 @@
 //! Readiness is established by the harness's `start()` handshake (TCP reachability
 //! plus an S3 `ListBuckets` poll) — there are no fixed sleeps.
 //!
-//! Out of scope for this block (tracked separately): network fault injection
+//! The volume-proxy smoke below also proves that the socket-level fault proxy
-//! (toxiproxy / socket proxy) and 5GiB large-object budgets.
+//! can be installed before startup without changing the client-facing node URL.
+//! A full lock-plane partition matrix and 5GiB large-object budget remain
+//! tracked separately.
 
 use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
 
@@ -76,6 +78,28 @@ async fn cluster_multidrive_single_pool_smoke() -> TestResult {
     Ok(())
 }
 
+/// 4 nodes x 4 drives, single pool: exercise the maximum local erasure layout
+/// supported by the cluster harness. This remains in the nightly lane because
+/// it starts four real server processes and sixteen data directories.
+#[tokio::test]
+async fn cluster_four_node_four_drive_single_pool_smoke() -> TestResult {
+    crate::common::init_logging();
+
+    let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, 4)).await?;
+
+    let volumes = cluster.rustfs_volumes_arg();
+    assert_eq!(volumes.split(' ').count(), 16, "expected 16 explicit endpoints, got: {volumes}");
+    assert!(!volumes.contains('{'), "single-pool layout must not use ellipses: {volumes}");
+    assert!(cluster.nodes.iter().all(|node| node.data_dirs.len() == 4));
+
+    cluster.start().await?;
+    cluster.create_test_bucket(BUCKET).await?;
+
+    let payload = vec![0x3Cu8; 1024 * 1024];
+    put_get_roundtrip(&cluster, "multidrive-4/object", &payload).await?;
+    Ok(())
+}
+
 /// Two single-node pools, 2 drives each: the multi-pool layout boots and
 /// round-trips. Every pool is a distinct erasure pool (`pool_idx` 0 and 1).
 #[tokio::test]
@@ -103,3 +127,27 @@ async fn cluster_two_pool_smoke() -> TestResult {
     put_get_roundtrip(&cluster, "twopool/object", &payload).await?;
     Ok(())
 }
+
+/// A real cluster smoke for the volume FaultProxy wiring. The proxy target is
+/// not listening yet when it is created; cluster startup must still converge
+/// once the target node starts, and peer disk/RPC traffic must traverse it.
+#[tokio::test]
+async fn cluster_volume_fault_proxy_pass_smoke() -> TestResult {
+    crate::common::init_logging();
+
+    let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(2, 2)).await?;
+    let proxy = cluster.start_volume_proxy_for_node(0).await?;
+    let proxied = proxy.local_addr().to_string();
+    assert!(cluster.rustfs_volumes_arg().contains(&proxied));
+
+    let result: TestResult = async {
+        cluster.start().await?;
+        cluster.create_test_bucket(BUCKET).await?;
+        let payload = vec![0x6Du8; 256 * 1024];
+        put_get_roundtrip(&cluster, "volume-proxy/object", &payload).await
+    }
+    .await;
+
+    proxy.shutdown().await;
+    result
+}

crates/e2e_test/src/common.rs

@@ -34,6 +34,7 @@ use serde_json;
 use std::ffi::OsStr;
 use std::fs as stdfs;
 use std::io::ErrorKind;
+use std::net::SocketAddr;
 use std::path::{Path, PathBuf};
 use std::process::{Child, Command, Stdio};
 use std::sync::Once;
@@ -217,7 +218,37 @@ pub(crate) async fn signed_s3_request(
     access_key: &str,
     secret_key: &str,
 ) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
-    signed_s3_request_with_session_token(method, url, body, content_type, access_key, secret_key, None).await
+    signed_s3_request_with_headers(method, url, body, content_type, access_key, secret_key, &http::HeaderMap::new()).await
+}
+
+pub(crate) async fn signed_s3_request_with_headers(
+    method: http::Method,
+    url: &str,
+    body: Option<String>,
+    content_type: Option<&str>,
+    access_key: &str,
+    secret_key: &str,
+    extra_headers: &http::HeaderMap,
+) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
+    signed_s3_request_with_session_token(
+        method,
+        url,
+        body,
+        content_type,
+        SigningCredentials {
+            access_key,
+            secret_key,
+            session_token: None,
+        },
+        extra_headers,
+    )
+    .await
+}
+
+struct SigningCredentials<'a> {
+    access_key: &'a str,
+    secret_key: &'a str,
+    session_token: Option<&'a str>,
 }
 
 async fn signed_s3_request_with_session_token(
@@ -225,9 +256,8 @@ async fn signed_s3_request_with_session_token(
     url: &str,
     body: Option<String>,
     content_type: Option<&str>,
-    access_key: &str,
+    credentials: SigningCredentials<'_>,
-    secret_key: &str,
+    extra_headers: &http::HeaderMap,
-    session_token: Option<&str>,
 ) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
     let uri = url.parse::<http::Uri>()?;
     let authority = uri.authority().ok_or("S3 URL missing authority")?.to_string();
@@ -239,14 +269,17 @@ async fn signed_s3_request_with_session_token(
     if let Some(content_type) = content_type {
         request = request.header(CONTENT_TYPE, content_type);
     }
+    for (name, value) in extra_headers {
+        request = request.header(name, value);
+    }
 
     let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "S3 request body is too large")?;
     let signed = sign_v4(
         request.body(Body::empty())?,
         content_length,
-        access_key,
+        credentials.access_key,
-        secret_key,
+        credentials.secret_key,
-        session_token.unwrap_or_default(),
+        credentials.session_token.unwrap_or_default(),
         "us-east-1",
     );
 
@@ -283,8 +316,19 @@ pub(crate) async fn admin_request_with_session_token(
 ) -> Result<(StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
     let url = format!("{base_url}{path_and_query}");
     let content_type = body.as_ref().map(|_| "application/json");
-    let response =
+    let response = signed_s3_request_with_session_token(
-        signed_s3_request_with_session_token(method, &url, body, content_type, access_key, secret_key, session_token).await?;
+        method,
+        &url,
+        body,
+        content_type,
+        SigningCredentials {
+            access_key,
+            secret_key,
+            session_token,
+        },
+        &http::HeaderMap::new(),
+    )
+    .await?;
     let status = response.status();
     let body = response.text().await?;
     Ok((status, body))
@@ -1171,6 +1215,9 @@ pub struct RustFSTestClusterEnvironment {
     pub node_extra_env: Vec<Vec<(String, String)>>,
     pub node_capture_log_paths: Vec<Option<String>>,
     pub topology: ClusterTopology,
+    /// Optional socket proxies used for the corresponding node's volume
+    /// endpoints. Proxies must be installed before [`Self::start`].
+    volume_proxy_addresses: Vec<Option<SocketAddr>>,
 }
 
 impl RustFSTestClusterEnvironment {
@@ -1262,6 +1309,7 @@ impl RustFSTestClusterEnvironment {
             extra_env.push(("RUSTFS_UNSAFE_BYPASS_DISK_CHECK".to_string(), "true".to_string()));
         }
 
+        let node_count = topology.node_count;
         Ok(Self {
             nodes,
             temp_dir,
@@ -1271,6 +1319,7 @@ impl RustFSTestClusterEnvironment {
             node_extra_env: vec![Vec::new(); topology.node_count],
             node_capture_log_paths: vec![None; topology.node_count],
             topology,
+            volume_proxy_addresses: vec![None; node_count],
         })
     }
 
@@ -1338,6 +1387,34 @@ impl RustFSTestClusterEnvironment {
         self.build_volumes_arg()
     }
 
+    /// Start a socket proxy for one node's volume endpoints and route all
+    /// subsequent `RUSTFS_VOLUMES` references for that node through it.
+    ///
+    /// Call this before [`Self::start`], then use the returned proxy's
+    /// [`crate::fault_proxy::FaultProxy::set_mode`] to inject latency,
+    /// blackhole, or one-way partition faults. The node's own listen address
+    /// remains direct, so S3 clients can still reach it while peer disk/RPC
+    /// traffic is steered through the proxy.
+    pub async fn start_volume_proxy_for_node(
+        &mut self,
+        node_idx: usize,
+    ) -> Result<crate::fault_proxy::FaultProxy, Box<dyn std::error::Error + Send + Sync>> {
+        self.ensure_node_index(node_idx)?;
+        if self.volume_proxy_addresses[node_idx].is_some() {
+            return Err(format!("a volume proxy is already configured for node {node_idx}").into());
+        }
+        let target = self.nodes[node_idx].address.parse::<SocketAddr>()?;
+        let proxy = crate::fault_proxy::FaultProxy::start(target).await?;
+        self.volume_proxy_addresses[node_idx] = Some(proxy.local_addr());
+        Ok(proxy)
+    }
+
+    fn volume_address(&self, node_idx: usize) -> String {
+        self.volume_proxy_addresses[node_idx]
+            .map(|address| address.to_string())
+            .unwrap_or_else(|| self.nodes[node_idx].address.clone())
+    }
+
     fn build_volumes_arg(&self) -> String {
         let pools = self.topology.normalized_pools();
 
@@ -1346,7 +1423,11 @@ impl RustFSTestClusterEnvironment {
             return self
                 .nodes
                 .iter()
-                .flat_map(|n| n.data_dirs.iter().map(move |dir| format!("http://{}{}", n.address, dir)))
+                .enumerate()
+                .flat_map(|(node_idx, n)| {
+                    let address = self.volume_address(node_idx);
+                    n.data_dirs.iter().map(move |dir| format!("http://{}{}", address, dir))
+                })
                 .collect::<Vec<_>>()
                 .join(" ");
         }
@@ -1357,13 +1438,19 @@ impl RustFSTestClusterEnvironment {
         pools
             .iter()
             .map(|nodes| {
-                let node = &self.nodes[nodes[0]];
+                let node_idx = nodes[0];
+                let node = &self.nodes[node_idx];
                 let base = node
                     .data_dirs
                     .first()
                     .and_then(|d| d.rsplit_once('/').map(|(parent, _)| parent))
                     .unwrap_or(&node.data_dir);
-                format!("http://{}{}/drive{{0...{}}}", node.address, base, self.topology.drives_per_node - 1)
+                format!(
+                    "http://{}{}/drive{{0...{}}}",
+                    self.volume_address(node_idx),
+                    base,
+                    self.topology.drives_per_node - 1
+                )
             })
             .collect::<Vec<_>>()
             .join(" ")
@@ -1744,30 +1831,128 @@ pub(crate) async fn admin_create_user(
     username: &str,
     secret_key: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
+    admin_create_user_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, username, secret_key).await
-    let body = serde_json::json!({
+}
-        "secretKey": secret_key,
-        "status": "enabled"
-    });
-    let response = signed_request(
-        http::Method::PUT,
-        &url,
-        &env.access_key,
-        &env.secret_key,
-        Some(body.to_string().into_bytes()),
-        Some("application/json"),
-    )
-    .await?;
 
-    if response.status() != reqwest::StatusCode::OK {
+/// Transport used by the shared admin-API helpers: in-process SigV4 signing
-        let status = response.status();
+/// via [`signed_request`], or the external `awscurl` binary (an independent
-        let body = response.text().await.unwrap_or_default();
+/// SigV4 implementation exercised by the awscurl-gated suites).
-        return Err(format!("create user failed: {status} {body}").into());
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum AdminTransport {
+    Signed,
+    Awscurl,
+}
+
+/// Execute an admin-API request against `base_url` with admin credentials over
+/// the chosen transport, failing on any non-success response.
+pub(crate) async fn admin_execute_at(
+    transport: AdminTransport,
+    method: http::Method,
+    base_url: &str,
+    admin_access_key: &str,
+    admin_secret_key: &str,
+    path_and_query: &str,
+    body: Option<&str>,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let url = format!("{base_url}{path_and_query}");
+    match transport {
+        AdminTransport::Signed => {
+            let content_type = match body {
+                Some(body) if !body.is_empty() => Some("application/json"),
+                _ => None,
+            };
+            let response = signed_request(
+                method.clone(),
+                &url,
+                admin_access_key,
+                admin_secret_key,
+                body.map(|body| body.as_bytes().to_vec()),
+                content_type,
+            )
+            .await?;
+            if !response.status().is_success() {
+                let status = response.status();
+                let text = response.text().await.unwrap_or_default();
+                return Err(format!("{method} {path_and_query} failed: {status} {text}").into());
+            }
+        }
+        AdminTransport::Awscurl => {
+            execute_awscurl(&url, method.as_str(), body, admin_access_key, admin_secret_key).await?;
+        }
     }
-
     Ok(())
 }
 
+/// Create a new IAM user via the admin API over the chosen transport.
+pub(crate) async fn admin_create_user_via(
+    transport: AdminTransport,
+    base_url: &str,
+    admin_access_key: &str,
+    admin_secret_key: &str,
+    username: &str,
+    secret_key: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let path = format!("/rustfs/admin/v3/add-user?accessKey={username}");
+    let body = serde_json::json!({"secretKey": secret_key, "status": "enabled"}).to_string();
+    admin_execute_at(
+        transport,
+        http::Method::PUT,
+        base_url,
+        admin_access_key,
+        admin_secret_key,
+        &path,
+        Some(&body),
+    )
+    .await
+}
+
+/// Install a canned policy via the admin API over the chosen transport.
+pub(crate) async fn admin_add_canned_policy_via(
+    transport: AdminTransport,
+    base_url: &str,
+    admin_access_key: &str,
+    admin_secret_key: &str,
+    policy_name: &str,
+    policy_json: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let path = format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}");
+    admin_execute_at(
+        transport,
+        http::Method::PUT,
+        base_url,
+        admin_access_key,
+        admin_secret_key,
+        &path,
+        Some(policy_json),
+    )
+    .await
+}
+
+/// Attach a canned policy to a user via the admin API over the chosen transport.
+pub(crate) async fn admin_attach_user_policy_via(
+    transport: AdminTransport,
+    base_url: &str,
+    admin_access_key: &str,
+    admin_secret_key: &str,
+    policy_name: &str,
+    username: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let path = format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={username}&isGroup=false");
+    // `Some("")` preserves the historical wire shape on both transports: awscurl
+    // keeps sending `-d ''` and the signed path attaches an empty body with no
+    // content type.
+    admin_execute_at(
+        transport,
+        http::Method::PUT,
+        base_url,
+        admin_access_key,
+        admin_secret_key,
+        &path,
+        Some(""),
+    )
+    .await
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1859,7 +2044,7 @@ mod tests {
         }
         let multidrive = topology.drives_per_node > 1;
 
-        let nodes = (0..topology.node_count)
+        let nodes: Vec<ClusterNode> = (0..topology.node_count)
             .map(|i| {
                 let address = format!("127.0.0.1:{}", 9000 + i);
                 let data_dirs: Vec<String> = if multidrive {
@@ -1880,6 +2065,7 @@ mod tests {
             })
             .collect();
 
+        let node_count = nodes.len();
         RustFSTestClusterEnvironment {
             nodes,
             temp_dir,
@@ -1889,6 +2075,7 @@ mod tests {
             node_extra_env: vec![Vec::new(); topology.node_count],
             node_capture_log_paths: vec![None; topology.node_count],
             topology,
+            volume_proxy_addresses: vec![None; node_count],
         }
     }
 
@@ -1973,6 +2160,25 @@ mod tests {
         assert!(ClusterTopology::single_pool_multidrive(1, 1).validate().is_ok());
     }
 
+    #[tokio::test]
+    async fn volume_proxy_rewrites_cluster_volume_endpoint() {
+        let mut env = RustFSTestClusterEnvironment::new(1)
+            .await
+            .expect("cluster environment should allocate a node");
+        let direct = env.nodes[0].address.clone();
+        let proxy = env
+            .start_volume_proxy_for_node(0)
+            .await
+            .expect("volume proxy should bind before the target server starts");
+        let proxied = proxy.local_addr().to_string();
+        let volumes = env.rustfs_volumes_arg();
+
+        assert!(volumes.contains(&proxied), "volumes must use the proxy address: {volumes}");
+        assert!(!volumes.contains(&direct), "volumes must not retain the direct address: {volumes}");
+
+        proxy.shutdown().await;
+    }
+
     #[test]
     fn cluster_node_env_supports_per_node_overrides() {
         let mut env = fake_cluster(ClusterTopology::single_pool(4));

crates/e2e_test/src/existing_object_tag_policy_test.rs

@@ -16,37 +16,29 @@
 //! session policy** (`Policy` parameter) via `awscurl --service sts` with explicit
 //! `Content-Type: application/x-www-form-urlencoded` on `POST /`.
 
-use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging};
+use crate::common::{
-use aws_sdk_s3::config::{Credentials, Region};
+    AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via,
+    awscurl_delete, awscurl_post_sts_form_urlencoded, build_test_s3_config, init_logging,
+};
+use aws_sdk_s3::Client;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{Delete, ObjectIdentifier, Tag, Tagging};
-use aws_sdk_s3::{Client, Config};
 use tracing::info;
 use uuid::Uuid;
 
 fn user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
-    let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-existing-tag");
+    env.create_s3_client_with_credentials(access_key, secret_key)
-    let config = Config::builder()
-        .credentials_provider(credentials)
-        .region(Region::new("us-east-1"))
-        .endpoint_url(&env.url)
-        .force_path_style(true)
-        .behavior_version_latest()
-        .build();
-    Client::from_conf(config)
 }
 
 fn sts_session_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str, session_token: &str) -> Client {
-    let credentials = Credentials::new(access_key, secret_key, Some(session_token.into()), None, "e2e-sts-session");
+    Client::from_conf(build_test_s3_config(
-    let config = Config::builder()
+        &env.url,
-        .credentials_provider(credentials)
+        access_key,
-        .region(Region::new("us-east-1"))
+        secret_key,
-        .endpoint_url(&env.url)
+        Some(session_token),
-        .force_path_style(true)
+        "e2e-sts-session",
-        .behavior_version_latest()
+    ))
-        .build();
-    Client::from_conf(config)
 }
 
 fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> {
@@ -77,15 +69,16 @@ async fn assume_role_with_session_policy(
     parse_assume_role_credentials(&xml)
 }
 
+// This suite deliberately drives the admin API through the external `awscurl`
+// binary (an independent SigV4 implementation), so the wrappers below pin
+// `AdminTransport::Awscurl`.
+
 async fn admin_create_user(
     env: &RustFSTestEnvironment,
     username: &str,
     password: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let body = serde_json::json!({ "secretKey": password, "status": "enabled" }).to_string();
+    admin_create_user_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, username, password).await
-    let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
-    awscurl_put(&url, &body, &env.access_key, &env.secret_key).await?;
-    Ok(())
 }
 
 async fn admin_add_canned_policy(
@@ -93,9 +86,15 @@ async fn admin_add_canned_policy(
     policy_name: &str,
     policy_json: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
+    admin_add_canned_policy_via(
-    awscurl_put(&url, policy_json, &env.access_key, &env.secret_key).await?;
+        AdminTransport::Awscurl,
-    Ok(())
+        &env.url,
+        &env.access_key,
+        &env.secret_key,
+        policy_name,
+        policy_json,
+    )
+    .await
 }
 
 async fn admin_attach_policy_to_user(
@@ -103,12 +102,7 @@ async fn admin_attach_policy_to_user(
     policy_name: &str,
     username: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let url = format!(
+    admin_attach_user_policy_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, policy_name, username).await
-        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
-        env.url, policy_name, username
-    );
-    awscurl_put(&url, "", &env.access_key, &env.secret_key).await?;
-    Ok(())
 }
 
 async fn admin_remove_user(env: &RustFSTestEnvironment, username: &str) {

crates/e2e_test/src/group_delete_test.rs

@@ -15,20 +15,11 @@
 //! E2E tests for group management (fixes #2028).
 
 use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
-use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::Client;
-use aws_sdk_s3::{Client, Config};
 use tracing::info;
 
 fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
-    let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-group-test");
+    env.create_s3_client_with_credentials(access_key, secret_key)
-    let config = Config::builder()
-        .credentials_provider(credentials)
-        .region(Region::new("us-east-1"))
-        .endpoint_url(&env.url)
-        .force_path_style(true)
-        .behavior_version_latest()
-        .build();
-    Client::from_conf(config)
 }
 
 #[tokio::test(flavor = "multi_thread")]

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -16,13 +16,14 @@
 
 #[cfg(test)]
 mod tests {
-    use crate::chaos::signed_admin_post;
+    use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
-    use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging};
+    use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging};
     use aws_sdk_s3::primitives::ByteStream;
+    use http::Method;
     use std::collections::HashSet;
     use std::error::Error;
     use std::path::{Path, PathBuf};
-    use tokio::time::{Duration, sleep, timeout};
+    use tokio::time::{Duration, Instant, sleep, timeout};
     use tracing::info;
 
     fn has_file_under(path: &Path) -> bool {
@@ -48,6 +49,110 @@ mod tests {
         disk.join(bucket).join(key).join("xl.meta").is_file()
     }
 
+    // Healing may rewrite non-identity bookkeeping in xl.meta. The census
+    // therefore compares the canonical selected metadata fields plus every
+    // physical shard, while the payload seed makes object mix-ups observable.
+    #[derive(Debug)]
+    struct PhysicalObjectManifest {
+        key: String,
+        payload_seed: u8,
+        shard_census: VersionShardCensus,
+    }
+
+    fn deterministic_object_body(len: usize, seed: u8) -> Vec<u8> {
+        let mut value = seed;
+        std::iter::repeat_with(|| {
+            value = value.wrapping_mul(31).wrapping_add(17);
+            value
+        })
+        .take(len)
+        .collect()
+    }
+
+    fn matching_manifest_count(
+        disk: &Path,
+        bucket: &str,
+        expected_manifests: &[PhysicalObjectManifest],
+    ) -> Result<usize, Box<dyn Error + Send + Sync>> {
+        let mut matching = 0;
+        for expected in expected_manifests {
+            let actual = census_object_version_on_disk(disk, bucket, &expected.key, None)?;
+            if actual.matches_manifest(&expected.shard_census) {
+                matching += 1;
+            }
+        }
+        Ok(matching)
+    }
+
+    fn metadata_count(disk: &Path, bucket: &str, expected_manifests: &[PhysicalObjectManifest]) -> usize {
+        expected_manifests
+            .iter()
+            .filter(|expected| object_metadata_exists_on_disk(disk, bucket, &expected.key))
+            .count()
+    }
+
+    fn heal_task_status_diagnostic(body: &str) -> String {
+        let Ok(status) = serde_json::from_str::<serde_json::Value>(body) else {
+            return body.to_string();
+        };
+        let items = status["items"].as_array();
+        let mut unresolved_states = HashSet::new();
+        for item in items.into_iter().flatten() {
+            for drive in item["after"]["drives"].as_array().into_iter().flatten() {
+                if let Some(state) = drive["state"].as_str()
+                    && state != "ok"
+                {
+                    unresolved_states.insert(state.to_string());
+                }
+            }
+        }
+        let mut unresolved_states = unresolved_states.into_iter().collect::<Vec<_>>();
+        unresolved_states.sort();
+        format!(
+            "summary={:?}, detail={:?}, item_count={}, unresolved_drive_states={unresolved_states:?}",
+            status["summary"].as_str(),
+            status["detail"].as_str(),
+            items.map_or(0, Vec::len)
+        )
+    }
+
+    fn cluster_heal_is_idle(status: &serde_json::Value) -> bool {
+        let operations = &status["healOperations"];
+        status["clusterStatusComplete"] == serde_json::Value::Bool(true)
+            && status["state"].as_str() == Some("idle")
+            && operations["queueLength"].as_u64() == Some(0)
+            && operations["activeTasks"].as_u64() == Some(0)
+            && operations["retryingTasks"].as_u64() == Some(0)
+    }
+
+    fn only_admin_heal_is_active(status: &serde_json::Value) -> bool {
+        let operations = &status["healOperations"];
+        status["clusterStatusComplete"] == serde_json::Value::Bool(true)
+            && status["state"].as_str() == Some("active")
+            && operations["queueLength"].as_u64() == Some(0)
+            && operations["activeTasks"].as_u64() == Some(1)
+            && operations["retryingTasks"].as_u64() == Some(0)
+            && operations["activeBySource"]["admin"].as_u64() == Some(1)
+    }
+
+    async fn replacement_recovery_status(
+        cluster: &RustFSTestClusterEnvironment,
+    ) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
+        let (status, body) = admin_request(
+            &cluster.nodes[0].url,
+            Method::GET,
+            "/rustfs/admin/v4/heal/replacement-recovery",
+            None,
+            &cluster.access_key,
+            &cluster.secret_key,
+        )
+        .await?;
+        if !status.is_success() {
+            return Err(format!("replacement recovery status failed: {status} {body}").into());
+        }
+        serde_json::from_str(&body).map_err(|err| format!("replacement recovery status is not JSON ({err}): {body}").into())
+    }
+
     async fn assert_object_body(env: &RustFSTestEnvironment, bucket: &str, key: &str, expected: &[u8]) {
         let client = env.create_s3_client();
         let response = client
@@ -442,6 +547,380 @@ mod tests {
         .into())
     }
 
+    // Keep the original unformatted-disk scenario above. This case retains the
+    // format identity so only the explicit admin task can rebuild missing data.
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_cluster_root_heal_resumes_missing_remote_shards_after_node_restart() -> Result<(), Box<dyn Error + Send + Sync>>
+    {
+        init_logging();
+        info!(
+            event = "heal_restart_started",
+            component = "e2e_test",
+            subsystem = "heal",
+            "Starting root-heal restart test"
+        );
+
+        let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
+        cluster.set_env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true");
+        cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
+        cluster.set_env("RUSTFS_HEAL_AUTO_HEAL_ENABLE", "false");
+        cluster.set_env("RUSTFS_HEAL_MRF_ENABLE", "false");
+        cluster.set_env("RUSTFS_SCANNER_ENABLED", "false");
+        cluster.set_env("RUSTFS_HEAL_MAX_CONCURRENT_HEALS", "1");
+        cluster.set_env("RUSTFS_HEAL_MAX_CONCURRENT_PER_SET", "1");
+        cluster.set_env("RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY", "1");
+        cluster.set_env("RUSTFS_HEAL_PAGE_PARALLEL_ENABLE", "false");
+        // Keep all storage nodes' Heal runtimes enabled so their disk services
+        // complete normal registration after restart. Scanner, auto-heal and
+        // MRF are disabled; the pre-root idle barrier below drains the direct
+        // outage-object repair before the explicit admin task starts.
+        let server_rust_log = std::env::var("RUSTFS_HEAL_CHAOS_SERVER_RUST_LOG")
+            .unwrap_or_else(|_| "rustfs::heal::task=info,rustfs=error".to_string());
+        cluster.set_env("RUST_LOG", server_rust_log);
+        if let Ok(log_dir) = std::env::var("RUSTFS_HEAL_CHAOS_LOG_DIR") {
+            std::fs::create_dir_all(&log_dir)?;
+            for node_index in 0..cluster.nodes.len() {
+                cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
+            }
+        }
+        cluster.start().await?;
+        let clients = cluster.create_all_clients()?;
+
+        let bucket = "heal-restart-during-rebuild";
+        clients[0].create_bucket().bucket(bucket).send().await?;
+
+        let replaced_disk = PathBuf::from(&cluster.nodes[1].data_dir);
+        let replacement_format_path = replaced_disk.join(".rustfs.sys").join("format.json");
+        let replacement_format = std::fs::read(&replacement_format_path).map_err(|err| {
+            format!("failed to capture target format before replacement wipe at {replacement_format_path:?}: {err}")
+        })?;
+        let online_object_count = std::env::var("RUSTFS_HEAL_CHAOS_OBJECT_COUNT")
+            .ok()
+            .and_then(|value| value.parse::<usize>().ok())
+            .unwrap_or(24)
+            .clamp(8, 64);
+        let object_size_bytes = std::env::var("RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES")
+            .ok()
+            .and_then(|value| value.parse::<usize>().ok())
+            .unwrap_or(4 * 1024 * 1024)
+            .clamp(1024 * 1024, 16 * 1024 * 1024);
+        let mut expected_manifests = Vec::with_capacity(online_object_count);
+        for index in 0..online_object_count {
+            let key = format!("cluster/online/object-{index:04}.bin");
+            let payload_seed = u8::try_from(index + 1).expect("clamped object count must fit in u8");
+            timeout(
+                Duration::from_secs(30),
+                clients[0]
+                    .put_object()
+                    .bucket(bucket)
+                    .key(&key)
+                    .body(ByteStream::from(deterministic_object_body(object_size_bytes, payload_seed)))
+                    .send(),
+            )
+            .await??;
+            let shard_census = census_object_version_on_disk(&replaced_disk, bucket, &key, None)?;
+            assert!(
+                shard_census.is_complete(),
+                "node 1 should hold a complete baseline shard for {key}: {shard_census:?}"
+            );
+            assert!(
+                !shard_census.expected_part_numbers.is_empty(),
+                "chaos objects must use physical part shards rather than inline data: {shard_census:?}"
+            );
+            expected_manifests.push(PhysicalObjectManifest {
+                key,
+                payload_seed,
+                shard_census,
+            });
+        }
+
+        cluster.stop_node(1)?;
+        std::fs::remove_dir_all(&replaced_disk)?;
+        std::fs::create_dir_all(
+            replacement_format_path
+                .parent()
+                .ok_or("replacement format path has no parent")?,
+        )?;
+        std::fs::write(&replacement_format_path, replacement_format)?;
+        assert!(
+            replacement_format_path.is_file(),
+            "replacement target must retain only its preformatted topology identity"
+        );
+
+        let outage_key = "cluster/written-while-node-down.bin";
+        let outage_payload_seed = 0xf1;
+        timeout(
+            Duration::from_secs(30),
+            clients[2]
+                .put_object()
+                .bucket(bucket)
+                .key(outage_key)
+                .body(ByteStream::from(deterministic_object_body(object_size_bytes, outage_payload_seed)))
+                .send(),
+        )
+        .await??;
+
+        let mut outage_peer_erasure_indices = HashSet::new();
+        for (node_index, node) in cluster.nodes.iter().enumerate() {
+            if node_index == 1 {
+                continue;
+            }
+            let census = census_object_version_on_disk(Path::new(&node.data_dir), bucket, outage_key, None)?;
+            assert!(
+                census.is_complete(),
+                "online node {node_index} must hold a complete outage-object shard: {census:?}"
+            );
+            let erasure_index = census
+                .erasure_index
+                .ok_or_else(|| format!("online node {node_index} outage-object shard has no erasure index: {census:?}"))?;
+            assert!(
+                (1..=cluster.nodes.len()).contains(&erasure_index),
+                "online node {node_index} outage-object erasure index is out of range: {census:?}"
+            );
+            assert!(
+                outage_peer_erasure_indices.insert(erasure_index),
+                "outage-object erasure index {erasure_index} is duplicated across online nodes"
+            );
+        }
+        assert_eq!(
+            outage_peer_erasure_indices.len(),
+            cluster.nodes.len().saturating_sub(1),
+            "every online node must contribute one unique outage-object erasure index"
+        );
+        let expected_outage_target_erasure_index = (1..=cluster.nodes.len())
+            .find(|index| !outage_peer_erasure_indices.contains(index))
+            .ok_or("online outage-object shards leave no erasure index for the replacement target")?;
+
+        // The PUT path may have admitted a direct Internal object repair while
+        // node 1 was offline. Cancel the isolated bucket path before the target
+        // returns; otherwise it could rebuild the outage object and invalidate
+        // the explicit-root ownership assertion below.
+        let cancel_outage_heal_path = format!("/rustfs/admin/v3/heal/{bucket}?forceStop=true");
+        let (cancel_status, cancel_body) = admin_request(
+            &cluster.nodes[0].url,
+            Method::POST,
+            &cancel_outage_heal_path,
+            Some(
+                r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#
+                    .to_string(),
+            ),
+            &cluster.access_key,
+            &cluster.secret_key,
+        )
+        .await?;
+        if !cancel_status.is_success() {
+            return Err(format!("cancel outage heal failed: {cancel_status} {cancel_body}").into());
+        }
+
+        cluster.start_node(1).await?;
+
+        let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
+        let recovery_deadline = Instant::now() + Duration::from_secs(60);
+        loop {
+            let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
+            assert!(
+                !status_body.contains("MissingContentLength"),
+                "background heal status should not fail without an explicit Content-Length: {status_body}"
+            );
+            let recovered: serde_json::Value = serde_json::from_str(&status_body)
+                .map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
+            if cluster_heal_is_idle(&recovered) {
+                break;
+            }
+            if Instant::now() >= recovery_deadline {
+                return Err(format!("cluster heal operations did not become idle before root heal: {recovered}").into());
+            }
+            sleep(Duration::from_millis(250)).await;
+        }
+        assert_eq!(
+            matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?,
+            0,
+            "non-admin Heal is disabled, so the replacement target must remain empty before the explicit root heal"
+        );
+        assert!(
+            !census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?.has_xl_meta,
+            "the object written during the outage must be absent before the explicit root heal"
+        );
+        let pre_heal_replacement = replacement_recovery_status(&cluster).await?;
+        assert_eq!(
+            pre_heal_replacement["cluster"]["records"].as_array().map(Vec::len),
+            Some(0),
+            "isolated target must not retain an automatic replacement generation: {pre_heal_replacement}"
+        );
+
+        let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
+        let heal_url = format!("{}/rustfs/admin/v3/heal/?forceStart=true", cluster.nodes[0].url);
+        let heal_start_body = signed_admin_post(&heal_url, Some(heal_body), &cluster.access_key, &cluster.secret_key).await?;
+        let heal_start: serde_json::Value = serde_json::from_str(&heal_start_body)
+            .map_err(|err| format!("heal start response is not JSON ({err}): {heal_start_body}"))?;
+        let client_token = heal_start["clientToken"]
+            .as_str()
+            .filter(|token| !token.is_empty())
+            .ok_or_else(|| format!("heal start response has no client token: {heal_start}"))?;
+        let task_status_url = format!("{}/rustfs/admin/v3/heal/?clientToken={client_token}", cluster.nodes[0].url);
+
+        let partial_timeout_secs = std::env::var("RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS")
+            .ok()
+            .and_then(|value| value.parse::<u64>().ok())
+            .unwrap_or(60);
+        let partial_deadline = Instant::now() + Duration::from_secs(partial_timeout_secs);
+        let pre_interrupt_status = loop {
+            let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
+            let active_status: serde_json::Value = serde_json::from_str(&status_body)
+                .map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
+            if only_admin_heal_is_active(&active_status) {
+                break active_status;
+            }
+            if Instant::now() >= partial_deadline {
+                return Err(format!("root heal never became active within {partial_timeout_secs}s: {active_status}").into());
+            }
+            sleep(Duration::from_millis(50)).await;
+        };
+        let partial_count = loop {
+            let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
+            if matching > 0 && matching < expected_manifests.len() {
+                break matching;
+            }
+            if matching == expected_manifests.len() {
+                return Err(format!(
+                    "root heal rebuilt all {} baseline objects before the target could be interrupted",
+                    expected_manifests.len()
+                )
+                .into());
+            }
+            if Instant::now() >= partial_deadline {
+                return Err(format!(
+                    "root heal made no observable partial progress on the replacement target within {partial_timeout_secs}s"
+                )
+                .into());
+            }
+            sleep(Duration::from_millis(10)).await;
+        };
+        info!(
+            event = "heal_restart_checkpoint",
+            component = "e2e_test",
+            subsystem = "heal",
+            partial_count,
+            "Verified unique admin owner before target interruption"
+        );
+
+        cluster.stop_node(1)?;
+        let stopped_count = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
+        assert!(
+            stopped_count > 0 && stopped_count < expected_manifests.len(),
+            "the target must stop after a partial rebuild, observed before stop={partial_count}, after stop={stopped_count}, total={}",
+            expected_manifests.len()
+        );
+        let unclean_shutdown_marker = replaced_disk.join(".rustfs.sys").join("unclean-shutdown");
+        match std::fs::remove_file(&unclean_shutdown_marker) {
+            Ok(()) => {}
+            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
+            Err(error) => {
+                return Err(format!("failed to isolate unclean recovery marker {unclean_shutdown_marker:?}: {error}").into());
+            }
+        }
+        cluster.start_node(1).await?;
+
+        let heal_timeout_secs = std::env::var("RUSTFS_HEAL_REPLACED_DISK_TIMEOUT_SECS")
+            .ok()
+            .and_then(|value| value.parse::<u64>().ok())
+            .unwrap_or(180);
+        let heal_deadline = Instant::now() + Duration::from_secs(heal_timeout_secs);
+        loop {
+            if metadata_count(&replaced_disk, bucket, &expected_manifests) == expected_manifests.len()
+                && object_metadata_exists_on_disk(&replaced_disk, bucket, outage_key)
+            {
+                let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
+                let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
+                if matching == expected_manifests.len() && outage_census.is_complete() {
+                    break;
+                }
+            }
+            if Instant::now() >= heal_deadline {
+                let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
+                let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
+                let final_status = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key)
+                    .await
+                    .unwrap_or_else(|err| format!("status request failed: {err}"));
+                let task_status = match timeout(
+                    Duration::from_secs(5),
+                    signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key),
+                )
+                .await
+                {
+                    Ok(Ok(body)) => heal_task_status_diagnostic(&body),
+                    Ok(Err(err)) => format!("task status request failed: {err}"),
+                    Err(_) => "task status request exceeded 5s diagnostic budget".to_string(),
+                };
+                let replacement_status = match timeout(Duration::from_secs(5), replacement_recovery_status(&cluster)).await {
+                    Ok(Ok(status)) => status.to_string(),
+                    Ok(Err(err)) => format!("replacement status request failed: {err}"),
+                    Err(_) => "replacement status request exceeded 5s diagnostic budget".to_string(),
+                };
+                return Err(format!(
+                    "root heal did not resume after target restart within {heal_timeout_secs}s: baseline={matching}/{}, outage={outage_census:?}, status={final_status}, task_status={task_status}, pre_interrupt_status={pre_interrupt_status}, pre_heal_replacement={pre_heal_replacement}, replacement_status={replacement_status}",
+                    expected_manifests.len()
+                )
+                .into());
+            }
+            sleep(Duration::from_millis(250)).await;
+        }
+
+        for expected in &expected_manifests {
+            let actual = census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?;
+            assert!(
+                actual.matches_manifest(&expected.shard_census),
+                "rebuilt target shard differs from its baseline for {}: {actual:?}",
+                expected.key
+            );
+        }
+        let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
+        assert!(
+            outage_census.is_complete(),
+            "outage object must have a complete target shard: {outage_census:?}"
+        );
+        assert_eq!(
+            outage_census.erasure_index,
+            Some(expected_outage_target_erasure_index),
+            "the outage object must be rebuilt into its own missing erasure slot"
+        );
+
+        let target_client = cluster.create_s3_client(1)?;
+        for expected in &expected_manifests {
+            let response = target_client.get_object().bucket(bucket).key(&expected.key).send().await?;
+            let actual = response.body.collect().await?.into_bytes();
+            let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed);
+            assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key);
+        }
+        let response = target_client.get_object().bucket(bucket).key(outage_key).send().await?;
+        let actual = response.body.collect().await?.into_bytes();
+        let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed);
+        assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}");
+
+        let terminal_deadline = Instant::now() + Duration::from_secs(30);
+        loop {
+            let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
+            let status: serde_json::Value = serde_json::from_str(&status_body)
+                .map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
+            if cluster_heal_is_idle(&status) {
+                break;
+            }
+            if Instant::now() >= terminal_deadline {
+                return Err(format!("heal data rebuilt but operations did not converge to terminal idle: {status}").into());
+            }
+            sleep(Duration::from_millis(250)).await;
+        }
+
+        let task_status_body = signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key).await?;
+        let task_status: serde_json::Value = serde_json::from_str(&task_status_body)
+            .map_err(|err| format!("heal task status is not JSON ({err}): {task_status_body}"))?;
+        if task_status["summary"].as_str() != Some("finished") {
+            return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
+        }
+
+        Ok(())
+    }
+
     /// Issue #5850: `background-heal/status` must answer while a peer is down.
     ///
     /// Exercises the production path in `read_cluster_heal_status` end to end,

crates/e2e_test/src/inline_fast_path_cluster_test.rs

@@ -15,13 +15,13 @@
 //! Four-node EC regression gate for inline storage and the inline GET reader.
 //!
 //! The storage decision is based on shard bytes (256 KiB / 32 KiB objects for
-//! the default EC 2+2 geometry), while the GET fast path has its own object-size
+//! the default EC 2+2 geometry), and the GET fast path follows the persisted
-//! limits (128 KiB / 16 KiB). A local OTLP/HTTP collector observes the existing
+//! inline marker. A local OTLP/HTTP collector observes the existing reader-path
-//! reader-path counter without adding a scrape endpoint or production logging.
+//! counter without adding a scrape endpoint or production logging.
 //! One S3 GET can select readers on multiple EC nodes, so the counter tracks
 //! distributed reader selection rather than HTTP request count.
 
-use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, local_http_client};
+use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
@@ -30,7 +30,7 @@ use aws_sdk_s3::types::{
 };
 use bytes::Bytes;
 use flate2::read::GzDecoder;
-use http::header::{CONTENT_ENCODING, HOST};
+use http::header::CONTENT_ENCODING;
 use http::{Method, Request, Response, StatusCode};
 use http_body_util::{BodyExt, Full};
 use hyper::body::Incoming;
@@ -42,9 +42,6 @@ use opentelemetry_proto::tonic::metrics::v1::{
     Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, metric, number_data_point,
 };
 use prost::Message;
-use rustfs_signer::constants::UNSIGNED_PAYLOAD;
-use rustfs_signer::sign_v4;
-use s3s::Body;
 use std::collections::BTreeMap;
 use std::convert::Infallible;
 use std::error::Error;
@@ -92,6 +89,7 @@ const MPU_PART_1_SIZE: usize = 5 * 1024 * 1024;
 const MPU_PART_2_SIZE: usize = 16 * KIB;
 const TIER_BUCKET: &str = "inline-fallback-cold-tier";
 const TIER_PREFIX: &str = "tiered";
+const ALLOW_LOOPBACK_TIER_ENDPOINT_ENV: &str = "RUSTFS_TIER_RUSTFS_ALLOW_LOOPBACK_ENDPOINT";
 const MSGPACK_FALLBACK_CONTROL_SERIES: [(&str, &str); 4] = [
     (FALLBACK_REQUEST_DIRECTION, "ReadMultipleReq"),
     (FALLBACK_RESPONSE_DIRECTION, "ReadMultipleResp"),
@@ -794,12 +792,12 @@ fn metric_attribute(key: &str, value: &str) -> KeyValue {
 }
 
 fn boundary_cases(state: VersionState) -> Vec<BoundaryCase> {
-    let (fast_limit, storage_limit) = match state {
+    let storage_limit = match state {
-        VersionState::Enabled => (16 * KIB, 32 * KIB),
+        VersionState::Enabled => 32 * KIB,
-        VersionState::Unversioned => (128 * KIB, 256 * KIB),
+        VersionState::Unversioned => 256 * KIB,
         // A suspended bucket stores its null version using the unversioned
         // shard threshold, while ObjectInfo keeps version-aware GET semantics.
-        VersionState::Suspended => (16 * KIB, 256 * KIB),
+        VersionState::Suspended => 256 * KIB,
     };
     let mut sizes = vec![0, 16 * KIB - 1, 16 * KIB, 16 * KIB + 1, 32 * KIB - 1, 32 * KIB, 32 * KIB + 1];
     if !matches!(state, VersionState::Enabled) {
@@ -820,7 +818,7 @@ fn boundary_cases(state: VersionState) -> Vec<BoundaryCase> {
             stored_inline: size <= storage_limit,
             expected_reader_path: if size == 0 {
                 EMPTY
-            } else if size <= fast_limit {
+            } else if size <= storage_limit {
                 INLINE_DIRECT
             } else {
                 LEGACY_DUPLEX
@@ -1262,6 +1260,8 @@ async fn put_two_part_multipart(client: &Client, bucket: &str, key: &str) -> Tes
     Ok((body, part2, complete.e_tag().map(str::to_owned)))
 }
 
+/// Thin wrapper over [`crate::common::admin_request`], kept local so the call
+/// sites below keep their `Option<&str>` body shape.
 async fn signed_admin_request(
     base_url: &str,
     method: Method,
@@ -1270,30 +1270,7 @@ async fn signed_admin_request(
     access_key: &str,
     secret_key: &str,
 ) -> TestResult<(reqwest::StatusCode, String)> {
-    let url = format!("{base_url}{path}");
+    crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
-    let body_bytes = body.map(|value| value.as_bytes().to_vec()).unwrap_or_default();
-
-    let request = http::Request::builder()
-        .method(method.clone())
-        .uri(uri)
-        .header(HOST, authority)
-        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
-
-    let client = local_http_client();
-    let mut request_builder = client.request(method, url.as_str());
-    for (name, value) in signed.headers() {
-        request_builder = request_builder.header(name, value);
-    }
-    if !body_bytes.is_empty() {
-        request_builder = request_builder.body(body_bytes);
-    }
-    let response = request_builder.send().await?;
-    let status = response.status();
-    let text = response.text().await?;
-    Ok((status, text))
 }
 
 fn unique_tier_name() -> String {
@@ -2124,6 +2101,7 @@ async fn four_node_add_tier_converges() -> TestResult {
     cold.create_s3_client().create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestClusterEnvironment::new(4).await?;
+    hot.set_env(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV, "true");
     hot.start().await?;
 
     let tier_name = unique_tier_name();
@@ -2142,6 +2120,7 @@ async fn four_node_add_tier_converges_after_offline_node_restart_without_second_
     cold.create_s3_client().create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestClusterEnvironment::new(4).await?;
+    hot.set_env(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV, "true");
     hot.start().await?;
 
     let tier_name = unique_tier_name();
@@ -2238,6 +2217,7 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
     cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestClusterEnvironment::new(4).await?;
+    hot.set_env(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV, "true");
     hot.set_env("RUSTFS_SCANNER_ENABLED", "false");
     hot.set_env("RUSTFS_SCANNER_CYCLE", "3600");
     hot.set_env("RUSTFS_MAX_TRANSITION_WORKERS", "1");
@@ -2380,6 +2360,7 @@ async fn four_node_manual_transition_rollout_non_empty_restart_readback() -> Tes
     cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestClusterEnvironment::new(4).await?;
+    hot.set_env(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV, "true");
     hot.set_env("RUSTFS_SCANNER_ENABLED", "false");
     hot.set_env("RUSTFS_SCANNER_CYCLE", "3600");
     hot.set_env("RUSTFS_MAX_TRANSITION_WORKERS", "2");
@@ -2484,6 +2465,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
 
     let collector = OtlpMetricCollector::start().await?;
     let mut hot = RustFSTestClusterEnvironment::new(4).await?;
+    hot.set_env(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV, "true");
     configure_mixed_msgpack_cluster(&mut hot, &collector)?;
     hot.set_env("RUSTFS_SCANNER_CYCLE", "1");
     hot.set_env("RUSTFS_ILM_PROCESS_TIME", "1");
@@ -2595,6 +2577,7 @@ async fn four_node_transitioned_inline_fallback() -> TestResult {
 
     let collector = OtlpMetricCollector::start().await?;
     let mut hot = RustFSTestClusterEnvironment::new(4).await?;
+    hot.set_env(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV, "true");
     configure_reader_metric_cluster(&mut hot, &collector);
     hot.set_env("RUSTFS_SCANNER_CYCLE", "1");
     hot.set_env("RUSTFS_ILM_PROCESS_TIME", "1");

crates/e2e_test/src/kms/configured_roundtrip_test.rs

@@ -17,6 +17,7 @@
 
 use super::common::{
     LocalKMSTestEnvironment, VAULT_KEY_NAME, VaultTestEnvironment, configure_kms, get_kms_status, kms_admin_request, start_kms,
+    test_sse_kms_encryption,
 };
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{BucketVersioningStatus, ServerSideEncryption, VersioningConfiguration};
@@ -431,6 +432,38 @@ async fn test_configured_local_kms_admin_and_versioned_cleanup() -> TestResult {
     Ok(())
 }
 
+#[tokio::test]
+async fn test_admin_configured_local_kms_is_restored_after_restart() -> TestResult {
+    let mut env = LocalKMSTestEnvironment::new().await?;
+    env.base_env.start_rustfs_server(Vec::new()).await?;
+
+    let default_key_id = env.configure_local_kms().await?;
+    start_kms(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
+
+    env.base_env.restart_server_preserving_data(Vec::new(), &[]).await?;
+    assert_configured_status(
+        &env.base_env.url,
+        &env.base_env.access_key,
+        &env.base_env.secret_key,
+        "local",
+        &default_key_id,
+    )
+    .await?;
+
+    let bucket = format!("kms-restart-{}", Uuid::new_v4());
+    env.base_env.create_test_bucket(&bucket).await?;
+    let client = env.base_env.create_s3_client();
+    test_sse_kms_encryption(&client, &bucket).await?;
+    client
+        .delete_object()
+        .bucket(&bucket)
+        .key("test-sse-kms-object")
+        .send()
+        .await?;
+    env.base_env.delete_test_bucket(&bucket).await?;
+    Ok(())
+}
+
 #[tokio::test]
 async fn test_configured_vault_kms_admin_and_versioned_cleanup() -> TestResult {
     let mut env = VaultTestEnvironment::new().await?;

crates/e2e_test/src/kms/kms_ilm_sse_kms_test.rs

@@ -66,6 +66,7 @@ const SURVIVOR_KEY: &str = "keep/object.bin";
 const TIER_NAME: &str = "KMSCOLD";
 const TIER_BUCKET: &str = "kms-ilm-cold-tier";
 const TIER_PREFIX: &str = "tiered";
+const ALLOW_LOOPBACK_TIER_ENDPOINT_ENV: (&str, &str) = ("RUSTFS_TIER_RUSTFS_ALLOW_LOOPBACK_ENDPOINT", "true");
 const TRANSITION_BUCKET: &str = "kms-ilm-transition";
 const TRANSITION_KEY: &str = "tier/object.bin";
 
@@ -80,7 +81,7 @@ const ILM_DEADLINE: StdDuration = StdDuration::from_secs(90);
 /// `--kms-default-key-id`, insecure dev defaults). The lifecycle env matches
 /// `reliant/lifecycle.rs::fast_lifecycle_env` plus `RUSTFS_ILM_DEBUG_DAY_SECS=2`,
 /// so a `Days=1` rule is due about two seconds after the write.
-async fn start_enforcing_ilm_server(env: &mut LocalKMSTestEnvironment) -> TestResult {
+async fn start_enforcing_ilm_server(env: &mut LocalKMSTestEnvironment, extra_env: &[(&str, &str)]) -> TestResult {
     create_key_with_specific_id(&env.kms_keys_dir, SSE_KEY).await?;
 
     let key_dir = env.kms_keys_dir.clone();
@@ -94,13 +95,14 @@ async fn start_enforcing_ilm_server(env: &mut LocalKMSTestEnvironment) -> TestRe
         SSE_KEY,
     ];
 
-    let envs = [
+    let mut envs = vec![
         ("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"),
         ("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "true"),
         ("RUSTFS_SCANNER_CYCLE", "1"),
         ("RUSTFS_ILM_PROCESS_TIME", "1"),
         ("RUSTFS_ILM_DEBUG_DAY_SECS", "2"),
     ];
+    envs.extend_from_slice(extra_env);
 
     env.base_env.start_rustfs_server_with_env(args, &envs).await?;
     Ok(())
@@ -427,7 +429,7 @@ async fn ilm_expiration_on_sse_kms_bucket_under_enforcement() -> TestResult {
     init_logging();
 
     let mut env = LocalKMSTestEnvironment::new().await?;
-    start_enforcing_ilm_server(&mut env).await?;
+    start_enforcing_ilm_server(&mut env, &[]).await?;
     env.base_env.create_test_bucket(EXPIRY_BUCKET).await?;
 
     let client = env.base_env.create_s3_client();
@@ -499,7 +501,7 @@ async fn ilm_transition_on_sse_kms_bucket_under_enforcement_reads_back() -> Test
 
     // Hot server: Local KMS + enforcement + accelerated lifecycle clock.
     let mut env = LocalKMSTestEnvironment::new().await?;
-    start_enforcing_ilm_server(&mut env).await?;
+    start_enforcing_ilm_server(&mut env, &[ALLOW_LOOPBACK_TIER_ENDPOINT_ENV]).await?;
     let hot_client = env.base_env.create_s3_client();
 
     add_rustfs_tier(&env.base_env, &cold.base_env).await?;

crates/e2e_test/src/kms/mod.rs

@@ -57,6 +57,9 @@ mod copy_object_version_restore_sse_test;
 #[cfg(test)]
 mod configured_roundtrip_test;
 
+#[cfg(test)]
+mod select_sse_response_test;
+
 #[cfg(test)]
 mod kms_anonymous_enforcement_test;
 

crates/e2e_test/src/kms/select_sse_response_test.rs

@@ -0,0 +1,241 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! SelectObjectContent SSE response-header compatibility (backlog#1625).
+
+use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64, start_kms};
+use crate::common::signed_s3_request_with_headers;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::ServerSideEncryption;
+use base64_simd::STANDARD as BASE64;
+use http::{HeaderMap, Method};
+use std::error::Error;
+use uuid::Uuid;
+
+type TestResult<T = ()> = Result<T, Box<dyn Error + Send + Sync>>;
+
+const CSV_BODY: &[u8] = b"name\nalice\n";
+const SELECT_BODY: &str = r#"<SelectObjectContentRequest xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+<Expression>SELECT * FROM S3Object</Expression>
+<ExpressionType>SQL</ExpressionType>
+<InputSerialization><CSV><FileHeaderInfo>USE</FileHeaderInfo></CSV></InputSerialization>
+<OutputSerialization><CSV/></OutputSerialization>
+</SelectObjectContentRequest>"#;
+const KMS_CONTEXT: &str = "eyJ0ZW5hbnQiOiJzMy1zZWxlY3QifQ==";
+const SSE_ALGORITHM: &str = "x-amz-server-side-encryption";
+const SSE_KMS_KEY_ID: &str = "x-amz-server-side-encryption-aws-kms-key-id";
+const SSE_KMS_CONTEXT: &str = "x-amz-server-side-encryption-context";
+const SSE_C_ALGORITHM: &str = "x-amz-server-side-encryption-customer-algorithm";
+const SSE_C_KEY: &str = "x-amz-server-side-encryption-customer-key";
+const SSE_C_KEY_MD5: &str = "x-amz-server-side-encryption-customer-key-md5";
+const LOG_FLUSH_SENTINEL: &str = "select-sse-log-flush-sentinel.csv";
+
+async fn raw_select(
+    env: &crate::common::RustFSTestEnvironment,
+    bucket: &str,
+    object: &str,
+    request_headers: &HeaderMap,
+) -> TestResult<reqwest::Response> {
+    let url = format!("{}/{bucket}/{object}?select&select-type=2", env.url);
+    signed_s3_request_with_headers(
+        Method::POST,
+        &url,
+        Some(SELECT_BODY.to_string()),
+        Some("application/xml"),
+        &env.access_key,
+        &env.secret_key,
+        request_headers,
+    )
+    .await
+}
+
+async fn assert_success_headers(response: reqwest::Response, expected: &[(&str, &str)], absent: &[&str]) -> TestResult {
+    if response.status() != reqwest::StatusCode::OK {
+        let status = response.status();
+        let url = response.url().clone();
+        let body = response.text().await?;
+        panic!("Select request to {url} failed with {status}: {body}");
+    }
+    for (name, value) in expected {
+        assert_eq!(response.headers().get(*name).and_then(|header| header.to_str().ok()), Some(*value));
+    }
+    for name in absent {
+        assert!(response.headers().get(*name).is_none(), "successful Select response must omit {name}");
+    }
+    let body = response.bytes().await?;
+    assert!(
+        body.windows(b"alice".len()).any(|window| window == b"alice"),
+        "successful Select response must contain a Records event with the selected row"
+    );
+    assert!(
+        body.windows(b"End".len()).any(|window| window == b"End"),
+        "successful Select response must contain the terminal End event"
+    );
+    Ok(())
+}
+
+async fn assert_pre_stream_failure(response: reqwest::Response) -> TestResult {
+    assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST);
+    let body = response.text().await?;
+    assert!(body.contains("<Error>"), "pre-stream failure must return an S3 XML error: {body}");
+    assert!(
+        body.contains("<Code>InvalidRequest</Code>"),
+        "invalid SSE-C parameters must preserve the S3 error code: {body}"
+    );
+    Ok(())
+}
+
+fn put_object(
+    client: &aws_sdk_s3::Client,
+    bucket: &str,
+    object: &str,
+) -> aws_sdk_s3::operation::put_object::builders::PutObjectFluentBuilder {
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(object)
+        .body(ByteStream::from_static(CSV_BODY))
+}
+
+#[tokio::test]
+async fn select_projects_encryption_headers_and_rejects_invalid_sse_c_before_streaming() -> TestResult {
+    let mut kms = LocalKMSTestEnvironment::new().await?;
+    let log_path = format!("{}/server.log", kms.base_env.temp_dir);
+    kms.base_env.capture_log_path = Some(log_path.clone());
+    kms.base_env
+        .start_rustfs_server_with_env(Vec::new(), &[("RUST_LOG", "s3s=debug,rustfs=info")])
+        .await?;
+    let key_id = kms.configure_local_kms().await?;
+    start_kms(&kms.base_env.url, &kms.base_env.access_key, &kms.base_env.secret_key).await?;
+
+    let client = kms.base_env.create_s3_client();
+    let bucket = format!("select-sse-{}", Uuid::new_v4().simple());
+    client.create_bucket().bucket(&bucket).send().await?;
+
+    put_object(&client, &bucket, "plain.csv").send().await?;
+    put_object(&client, &bucket, "sse-s3.csv")
+        .server_side_encryption(ServerSideEncryption::Aes256)
+        .send()
+        .await?;
+    put_object(&client, &bucket, "sse-kms.csv")
+        .server_side_encryption(ServerSideEncryption::AwsKms)
+        .ssekms_key_id(&key_id)
+        .ssekms_encryption_context(KMS_CONTEXT)
+        .send()
+        .await?;
+
+    let customer_key = "01234567890123456789012345678901";
+    let customer_key_b64 = BASE64.encode_to_string(customer_key);
+    let customer_key_md5 = sse_customer_key_md5_base64(customer_key);
+    put_object(&client, &bucket, "sse-c.csv")
+        .sse_customer_algorithm("AES256")
+        .sse_customer_key(&customer_key_b64)
+        .sse_customer_key_md5(&customer_key_md5)
+        .send()
+        .await?;
+
+    assert_success_headers(
+        raw_select(&kms.base_env, &bucket, "plain.csv", &HeaderMap::new()).await?,
+        &[],
+        &[
+            SSE_ALGORITHM,
+            SSE_KMS_KEY_ID,
+            SSE_KMS_CONTEXT,
+            SSE_C_ALGORITHM,
+            SSE_C_KEY,
+            SSE_C_KEY_MD5,
+        ],
+    )
+    .await?;
+    assert_success_headers(
+        raw_select(&kms.base_env, &bucket, "sse-s3.csv", &HeaderMap::new()).await?,
+        &[(SSE_ALGORITHM, "AES256")],
+        &[SSE_KMS_KEY_ID, SSE_KMS_CONTEXT, SSE_C_ALGORITHM, SSE_C_KEY, SSE_C_KEY_MD5],
+    )
+    .await?;
+    assert_success_headers(
+        raw_select(&kms.base_env, &bucket, "sse-kms.csv", &HeaderMap::new()).await?,
+        &[
+            (SSE_ALGORITHM, "aws:kms"),
+            (SSE_KMS_KEY_ID, &key_id),
+            (SSE_KMS_CONTEXT, KMS_CONTEXT),
+        ],
+        &[SSE_C_ALGORITHM, SSE_C_KEY, SSE_C_KEY_MD5],
+    )
+    .await?;
+
+    let mut sse_c_headers = HeaderMap::new();
+    sse_c_headers.insert(SSE_C_ALGORITHM, "AES256".parse()?);
+    sse_c_headers.insert(SSE_C_KEY, customer_key_b64.parse()?);
+    sse_c_headers.insert(SSE_C_KEY_MD5, customer_key_md5.parse()?);
+    assert_success_headers(
+        raw_select(&kms.base_env, &bucket, "sse-c.csv", &sse_c_headers).await?,
+        &[(SSE_C_ALGORITHM, "AES256"), (SSE_C_KEY_MD5, &customer_key_md5)],
+        &[SSE_ALGORITHM, SSE_KMS_KEY_ID, SSE_KMS_CONTEXT, SSE_C_KEY],
+    )
+    .await?;
+
+    assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &HeaderMap::new()).await?).await?;
+
+    let mut missing_algorithm_headers = HeaderMap::new();
+    missing_algorithm_headers.insert(SSE_C_KEY, customer_key_b64.parse()?);
+    missing_algorithm_headers.insert(SSE_C_KEY_MD5, customer_key_md5.parse()?);
+    assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &missing_algorithm_headers).await?).await?;
+
+    let mut wrong_algorithm_headers = sse_c_headers.clone();
+    wrong_algorithm_headers.insert(SSE_C_ALGORITHM, "AES128".parse()?);
+    assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &wrong_algorithm_headers).await?).await?;
+
+    let wrong_md5 = sse_customer_key_md5_base64("99999999999999999999999999999999");
+    let mut wrong_md5_headers = sse_c_headers.clone();
+    wrong_md5_headers.insert(SSE_C_KEY_MD5, wrong_md5.parse()?);
+    assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &wrong_md5_headers).await?).await?;
+
+    let wrong_key = "99999999999999999999999999999999";
+    let wrong_key_b64 = BASE64.encode_to_string(wrong_key);
+    let mut wrong_key_headers = HeaderMap::new();
+    wrong_key_headers.insert(SSE_C_ALGORITHM, "AES256".parse()?);
+    wrong_key_headers.insert(SSE_C_KEY, wrong_key_b64.parse()?);
+    wrong_key_headers.insert(SSE_C_KEY_MD5, wrong_md5.parse()?);
+    assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &wrong_key_headers).await?).await?;
+
+    put_object(&client, &bucket, LOG_FLUSH_SENTINEL).send().await?;
+    assert_success_headers(
+        raw_select(&kms.base_env, &bucket, LOG_FLUSH_SENTINEL, &HeaderMap::new()).await?,
+        &[],
+        &[
+            SSE_ALGORITHM,
+            SSE_KMS_KEY_ID,
+            SSE_KMS_CONTEXT,
+            SSE_C_ALGORITHM,
+            SSE_C_KEY,
+            SSE_C_KEY_MD5,
+        ],
+    )
+    .await?;
+    let mut logs = String::new();
+    for _ in 0..100 {
+        logs = tokio::fs::read_to_string(&log_path).await?;
+        if logs.contains(LOG_FLUSH_SENTINEL) {
+            break;
+        }
+        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
+    }
+    assert!(logs.contains(LOG_FLUSH_SENTINEL), "timed out waiting for the log sink to flush");
+    for secret in [customer_key, customer_key_b64.as_str(), wrong_key, wrong_key_b64.as_str()] {
+        assert!(!logs.contains(secret), "Select request logging leaked SSE-C customer key material");
+    }
+
+    Ok(())
+}

crates/e2e_test/src/lib.rs

@@ -61,6 +61,9 @@ mod get_codec_streaming_compat_test;
 #[cfg(test)]
 mod version_id_regression_test;
 
+#[cfg(test)]
+mod select_request_root_alias_test;
+
 // Pinned previous-release -> current-build on-disk compatibility.
 #[cfg(test)]
 mod upgrade_compatibility_test;
@@ -164,6 +167,10 @@ mod delete_objects_versioning_test;
 #[cfg(test)]
 mod delete_object_no_content_length_test;
 
+// Regression test for signed empty PutObject requests without Content-Length.
+#[cfg(test)]
+mod put_object_no_content_length_test;
+
 // Delete-marker visibility baseline for data-movement migration proof.
 #[cfg(test)]
 mod delete_marker_migration_semantics_test;

crates/e2e_test/src/multipart_auth_test.rs

@@ -15,7 +15,7 @@
 //! Regression coverage for anonymous access on multipart control APIs.
 
 use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
-use async_compression::tokio::write::{BzEncoder, XzEncoder};
+use async_compression::tokio::write::{BzEncoder, Lz4Encoder, XzEncoder};
 use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
 use aws_sdk_s3::operation::head_object::HeadObjectOutput;
 use aws_sdk_s3::primitives::ByteStream;
@@ -23,7 +23,10 @@ use aws_sdk_s3::types::{
     ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
 };
 use chrono::{Duration as ChronoDuration, Utc};
-use flate2::{Compression, write::GzEncoder};
+use flate2::{
+    Compression,
+    write::{GzEncoder, ZlibEncoder},
+};
 use http::HeaderValue;
 use http::header::{CONTENT_TYPE, HOST};
 use md5::{Digest as Md5Digest, Md5};
@@ -187,6 +190,12 @@ fn gzip_bytes(data: &[u8]) -> Vec<u8> {
     encoder.finish().expect("gzip encoder should finish")
 }
 
+fn zlib_bytes(data: &[u8]) -> Vec<u8> {
+    let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
+    encoder.write_all(data).expect("zlib encoder should accept input");
+    encoder.finish().expect("zlib encoder should finish")
+}
+
 fn zstd_bytes(data: &[u8]) -> Vec<u8> {
     let mut encoder = zstd::Encoder::new(Vec::new(), 0).expect("zstd encoder should initialize");
     encoder.write_all(data).expect("zstd encoder should accept input");
@@ -209,6 +218,45 @@ async fn xz_bytes(data: &[u8]) -> Vec<u8> {
     encoder.into_inner().into_inner()
 }
 
+async fn lz4_bytes(data: &[u8]) -> Vec<u8> {
+    let cursor = Cursor::new(Vec::new());
+    let mut encoder = Lz4Encoder::new(cursor);
+    encoder.write_all(data).await.expect("LZ4 encoder should accept input");
+    encoder.shutdown().await.expect("LZ4 encoder should finish");
+    encoder.into_inner().into_inner()
+}
+
+/// Encode the S2 framed stream shape emitted by minio-go PutObjectsSnowball
+/// with `Compress: true`: 1 MiB independent blocks, better compression,
+/// masked CRC-32C, and the `S2sTwO` stream identifier.
+fn minio_go_snowball_s2_bytes(data: &[u8]) -> Vec<u8> {
+    const BLOCK_SIZE: usize = 1 << 20;
+    const CHECKSUM_SIZE: usize = 4;
+
+    let mut output = b"\xff\x06\x00\x00S2sTwO".to_vec();
+    let mut encoder = minlz::Encoder::new();
+    for block in data.chunks(BLOCK_SIZE) {
+        let compressed = encoder.encode_better(block);
+        let compressed_limit = block.len().saturating_sub(block.len() / 32).saturating_sub(5);
+        let (chunk_type, payload) = if compressed.len() <= compressed_limit {
+            (0x00, compressed.as_slice())
+        } else {
+            (0x01, block)
+        };
+        let chunk_len = payload.len() + CHECKSUM_SIZE;
+        assert!(chunk_len < 1 << 24, "S2 fixture chunk must fit the 24-bit frame length");
+        output.extend_from_slice(&[
+            chunk_type,
+            (chunk_len & 0xff) as u8,
+            ((chunk_len >> 8) & 0xff) as u8,
+            ((chunk_len >> 16) & 0xff) as u8,
+        ]);
+        output.extend_from_slice(&minlz::crc::crc(block).to_le_bytes());
+        output.extend_from_slice(payload);
+    }
+    output
+}
+
 fn assert_s3_error_code<T, E>(result: Result<T, SdkError<E>>, code: &str)
 where
     T: std::fmt::Debug,
@@ -3456,6 +3504,62 @@ async fn test_signed_put_object_extract_expands_tar_entries_with_prefix_headers(
     Ok(())
 }
 
+#[tokio::test]
+async fn test_signed_put_object_extract_ignore_dirs_skips_unauthorized_directory()
+-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "signed-extract-ignore-dirs-auth";
+    let archive_key = "bundle.tar";
+    let allowed_member = "allowed/member.txt";
+    let denied_directory = "denied/";
+    let username = "snowball-ignore-dirs";
+    let secret_key = "snowball-ignore-dirs-secret";
+    let expected_body = b"allowed-body";
+
+    let admin_client = env.create_s3_client();
+    admin_client.create_bucket().bucket(bucket).send().await?;
+    create_restricted_user(&env, username, secret_key).await?;
+
+    let policy = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Principal": { "AWS": [username] },
+            "Action": ["s3:PutObject"],
+            "Resource": [
+                format!("arn:aws:s3:::{bucket}/{archive_key}"),
+                format!("arn:aws:s3:::{bucket}/{allowed_member}")
+            ]
+        }]
+    })
+    .to_string();
+    admin_client.put_bucket_policy().bucket(bucket).policy(policy).send().await?;
+
+    let restricted_client = restricted_user_client(&env, username, secret_key);
+    let tar_bytes = make_tar(&[(allowed_member, expected_body)], &[denied_directory]).await;
+    restricted_client
+        .put_object()
+        .bucket(bucket)
+        .key(archive_key)
+        .body(ByteStream::from(tar_bytes))
+        .customize()
+        .mutate_request(|req| {
+            req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
+            req.headers_mut().insert("x-amz-meta-snowball-ignore-dirs", "true");
+        })
+        .send()
+        .await?;
+
+    let stored = admin_client.get_object().bucket(bucket).key(allowed_member).send().await?;
+    assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), expected_body);
+
+    Ok(())
+}
+
 #[tokio::test]
 async fn test_signed_put_object_extract_preserves_request_metadata_on_extracted_objects()
 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@@ -4185,6 +4289,60 @@ async fn test_signed_put_object_extract_returns_archive_etag() -> Result<(), Box
     Ok(())
 }
 
+#[tokio::test]
+async fn test_signed_put_object_extract_expands_s2_and_lz4_by_magic_with_raw_etags()
+-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "signed-extract-magic-codecs";
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
+
+    let s2_tar = make_tar(&[("s2/object.txt", b"s2-body")], &[]).await;
+    let s2_archive = minio_go_snowball_s2_bytes(&s2_tar);
+    let expected_s2_etag = format!("\"{}\"", md5_hex(&s2_archive));
+    let s2_response = client
+        .put_object()
+        .bucket(bucket)
+        // minio-go intentionally uploads a compressed S2 stream with a .tar key.
+        .key("snowball-upload-0123456789abcdef.tar")
+        .body(ByteStream::from(s2_archive))
+        .customize()
+        .mutate_request(|req| {
+            req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
+        })
+        .send()
+        .await?;
+    assert_eq!(s2_response.e_tag(), Some(expected_s2_etag.as_str()));
+
+    let s2_object = client.get_object().bucket(bucket).key("s2/object.txt").send().await?;
+    assert_eq!(s2_object.body.collect().await?.into_bytes().as_ref(), b"s2-body");
+
+    let lz4_tar = make_tar(&[("lz4/object.txt", b"lz4-body")], &[]).await;
+    let lz4_archive = lz4_bytes(&lz4_tar).await;
+    let expected_lz4_etag = format!("\"{}\"", md5_hex(&lz4_archive));
+    let lz4_response = client
+        .put_object()
+        .bucket(bucket)
+        .key("also-looks-like-a-plain.tar")
+        .body(ByteStream::from(lz4_archive))
+        .customize()
+        .mutate_request(|req| {
+            req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
+        })
+        .send()
+        .await?;
+    assert_eq!(lz4_response.e_tag(), Some(expected_lz4_etag.as_str()));
+
+    let lz4_object = client.get_object().bucket(bucket).key("lz4/object.txt").send().await?;
+    assert_eq!(lz4_object.body.collect().await?.into_bytes().as_ref(), b"lz4-body");
+
+    Ok(())
+}
+
 #[tokio::test]
 async fn test_signed_put_object_extract_preserves_entry_mtime() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
@@ -4309,9 +4467,15 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
     let context_archive_resources = [
         format!("arn:aws:s3:::{bucket}/tag-context.tar"),
         format!("arn:aws:s3:::{bucket}/lock-context.tar"),
+        format!("arn:aws:s3:::{bucket}/legal-hold-context.tar"),
+        format!("arn:aws:s3:::{bucket}/user-agent-bypass.tar"),
+        format!("arn:aws:s3:::{bucket}/sse-bypass.tar"),
     ];
     let tag_entry_resource = format!("arn:aws:s3:::{bucket}/tag-context-entry.txt");
     let lock_entry_resource = format!("arn:aws:s3:::{bucket}/lock-context-entry.txt");
+    let legal_hold_entry_resource = format!("arn:aws:s3:::{bucket}/legal-hold-context-entry.txt");
+    let user_agent_entry_resource = format!("arn:aws:s3:::{bucket}/user-agent-bypass-entry.txt");
+    let sse_entry_resource = format!("arn:aws:s3:::{bucket}/sse-bypass-entry.txt");
     let policy = serde_json::json!({
         "Version": "2012-10-17",
         "Statement": [
@@ -4371,7 +4535,7 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
                 "Sid": "PaxContextArchives",
                 "Effect": "Allow",
                 "Principal": { "AWS": [pax_context_user] },
-                "Action": ["s3:PutObject", "s3:PutObjectRetention", "s3:PutObjectTagging"],
+                "Action": ["s3:PutObject", "s3:PutObjectRetention", "s3:PutObjectLegalHold", "s3:PutObjectTagging"],
                 "Resource": context_archive_resources
             },
             {
@@ -4411,6 +4575,49 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
                 "Principal": { "AWS": [pax_context_user] },
                 "Action": ["s3:PutObjectRetention"],
                 "Resource": [lock_entry_resource]
+            },
+            {
+                "Sid": "PaxLegalHoldContextPut",
+                "Effect": "Allow",
+                "Principal": { "AWS": [pax_context_user] },
+                "Action": ["s3:PutObject"],
+                "Resource": [legal_hold_entry_resource.clone()]
+            },
+            {
+                "Sid": "PaxLegalHoldContextAction",
+                "Effect": "Allow",
+                "Principal": { "AWS": [pax_context_user] },
+                "Action": ["s3:PutObjectLegalHold"],
+                "Resource": [legal_hold_entry_resource],
+                "Condition": {
+                    "StringEquals": {
+                        "s3:object-lock-legal-hold": "OFF"
+                    }
+                }
+            },
+            {
+                "Sid": "MemberUserAgentCondition",
+                "Effect": "Allow",
+                "Principal": { "AWS": [pax_context_user] },
+                "Action": ["s3:PutObject"],
+                "Resource": [user_agent_entry_resource],
+                "Condition": {
+                    "StringEquals": {
+                        "aws:UserAgent": "trusted"
+                    }
+                }
+            },
+            {
+                "Sid": "MemberSseCondition",
+                "Effect": "Allow",
+                "Principal": { "AWS": [pax_context_user] },
+                "Action": ["s3:PutObject"],
+                "Resource": [sse_entry_resource],
+                "Condition": {
+                    "StringEquals": {
+                        "s3:x-amz-server-side-encryption": "AES256"
+                    }
+                }
             }
         ]
     })
@@ -4423,9 +4630,14 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
     let cases = [
         (
             "legal-hold.tar",
-            put_only_client,
+            put_only_client.clone(),
             HashMap::from([("minio.metadata.x-amz-object-lock-legal-hold", "ON".to_string())]),
         ),
+        (
+            "tagging.tar",
+            put_only_client,
+            HashMap::from([("minio.metadata.x-amz-tagging", "classification=restricted".to_string())]),
+        ),
         (
             "retention-condition.tar",
             conditional_client,
@@ -4512,6 +4724,57 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
     assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), b"condition-body");
 
     let pax_context_client = restricted_user_client(&env, pax_context_user, pax_context_secret);
+    for (archive_key, entry_key, pax_key, injected_value, outer_user_agent) in [
+        (
+            "user-agent-bypass.tar",
+            "user-agent-bypass-entry.txt",
+            "minio.metadata.user-agent",
+            "trusted",
+            Some("untrusted"),
+        ),
+        (
+            "sse-bypass.tar",
+            "sse-bypass-entry.txt",
+            "minio.metadata.x-amz-server-side-encryption",
+            "AES256",
+            None,
+        ),
+    ] {
+        let pax = HashMap::from([(pax_key, injected_value.to_string())]);
+        let archive = make_tar_with_pax_entry(entry_key, b"must-not-write", None, &pax).await;
+        let err = pax_context_client
+            .put_object()
+            .bucket(bucket)
+            .key(archive_key)
+            .body(ByteStream::from(archive))
+            .customize()
+            .mutate_request(move |req| {
+                req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
+                if let Some(user_agent) = outer_user_agent {
+                    req.headers_mut().insert("user-agent", user_agent);
+                }
+            })
+            .send()
+            .await
+            .expect_err("PAX metadata must not satisfy unrelated IAM request conditions");
+        assert_eq!(
+            err.as_service_error().and_then(|error| error.meta().code()),
+            Some("AccessDenied"),
+            "{archive_key}"
+        );
+        let err = admin_client
+            .head_object()
+            .bucket(bucket)
+            .key(entry_key)
+            .send()
+            .await
+            .expect_err("a denied PAX member must not be written");
+        assert!(matches!(
+            err.as_service_error().and_then(|error| error.meta().code()),
+            Some("NoSuchKey" | "NotFound")
+        ));
+    }
+
     let tag_pax = HashMap::from([("minio.metadata.x-amz-tagging", "classification=public".to_string())]);
     let archive = make_tar_with_pax_entry("tag-context-entry.txt", b"tag-context-body", None, &tag_pax).await;
     pax_context_client
@@ -4575,6 +4838,34 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
         pax_retain_until
     );
 
+    let legal_hold_pax = HashMap::from([("minio.metadata.x-amz-object-lock-legal-hold", "ON".to_string())]);
+    let archive = make_tar_with_pax_entry("legal-hold-context-entry.txt", b"must-not-write", None, &legal_hold_pax).await;
+    let err = pax_context_client
+        .put_object()
+        .bucket(bucket)
+        .key("legal-hold-context.tar")
+        .object_lock_legal_hold_status(aws_sdk_s3::types::ObjectLockLegalHoldStatus::Off)
+        .body(ByteStream::from(archive))
+        .customize()
+        .mutate_request(|req| {
+            req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
+        })
+        .send()
+        .await
+        .expect_err("PAX legal hold must replace the outer value in the member IAM condition context");
+    assert_eq!(err.as_service_error().and_then(|error| error.meta().code()), Some("AccessDenied"));
+    let err = admin_client
+        .head_object()
+        .bucket(bucket)
+        .key("legal-hold-context-entry.txt")
+        .send()
+        .await
+        .expect_err("a denied PAX legal-hold member must not be written");
+    assert!(matches!(
+        err.as_service_error().and_then(|error| error.meta().code()),
+        Some("NoSuchKey" | "NotFound")
+    ));
+
     Ok(())
 }
 
@@ -5050,8 +5341,8 @@ async fn test_signed_put_object_extract_expands_tzst_archive() -> Result<(), Box
 }
 
 #[tokio::test]
-async fn test_signed_put_object_extract_rejects_missing_archive_extension() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
+async fn test_signed_put_object_extract_uses_magic_without_requiring_or_trusting_extension()
-{
+-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
     let mut env = RustFSTestEnvironment::new().await?;
@@ -5064,8 +5355,7 @@ async fn test_signed_put_object_extract_rejects_missing_archive_extension() -> R
     admin_client.create_bucket().bucket(bucket).send().await?;
 
     let tar_bytes = make_tar(&[("plain.txt", b"plain-body")], &[]).await;
-
+    admin_client
-    let result = admin_client
         .put_object()
         .bucket(bucket)
         .key(archive_key)
@@ -5075,15 +5365,80 @@ async fn test_signed_put_object_extract_rejects_missing_archive_extension() -> R
             req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
         })
         .send()
-        .await;
+        .await?;
 
-    assert_s3_error_code(result, "InvalidArgument");
+    let plain = admin_client.get_object().bucket(bucket).key("plain.txt").send().await?;
+    assert_eq!(plain.body.collect().await?.into_bytes().as_ref(), b"plain-body");
+
+    let raw_with_gzip_suffix = make_tar(&[("raw-with-wrong-suffix.txt", b"raw-body")], &[]).await;
+    admin_client
+        .put_object()
+        .bucket(bucket)
+        .key("raw-but-named.tar.gz")
+        .body(ByteStream::from(raw_with_gzip_suffix))
+        .customize()
+        .mutate_request(|req| {
+            req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
+        })
+        .send()
+        .await?;
+
+    let raw = admin_client
+        .get_object()
+        .bucket(bucket)
+        .key("raw-with-wrong-suffix.txt")
+        .send()
+        .await?;
+    assert_eq!(raw.body.collect().await?.into_bytes().as_ref(), b"raw-body");
+
+    let gzip_with_tar_suffix = gzip_bytes(&make_tar(&[("gzip-with-wrong-suffix.txt", b"gzip-body")], &[]).await);
+    admin_client
+        .put_object()
+        .bucket(bucket)
+        .key("gzip-but-named.tar")
+        .body(ByteStream::from(gzip_with_tar_suffix))
+        .customize()
+        .mutate_request(|req| {
+            req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
+        })
+        .send()
+        .await?;
+
+    let gzip = admin_client
+        .get_object()
+        .bucket(bucket)
+        .key("gzip-with-wrong-suffix.txt")
+        .send()
+        .await?;
+    assert_eq!(gzip.body.collect().await?.into_bytes().as_ref(), b"gzip-body");
+
+    let zlib_archive = zlib_bytes(&make_tar(&[("zlib-extension.txt", b"zlib-body")], &[]).await);
+    admin_client
+        .put_object()
+        .bucket(bucket)
+        .key("bundle.zlib")
+        .body(ByteStream::from(zlib_archive))
+        .customize()
+        .mutate_request(|req| {
+            req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
+        })
+        .send()
+        .await?;
+
+    let zlib = admin_client
+        .get_object()
+        .bucket(bucket)
+        .key("zlib-extension.txt")
+        .send()
+        .await?;
+    assert_eq!(zlib.body.collect().await?.into_bytes().as_ref(), b"zlib-body");
 
     Ok(())
 }
 
 #[tokio::test]
-async fn test_signed_put_object_extract_rejects_invalid_tar_gz_payload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+async fn test_signed_put_object_extract_rejects_invalid_archive_payload() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
+{
     init_logging();
 
     let mut env = RustFSTestEnvironment::new().await?;

crates/e2e_test/src/negative_sigv4_test.rs

@@ -36,9 +36,10 @@
 use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
-use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER};
 use rustfs_signer::request_signature_v4::{SIGN_V4_ALGORITHM, get_scope, get_signature, get_signing_key};
 use std::fmt::Write as _;
+use std::io::Cursor;
 use time::macros::format_description;
 use time::{Duration, OffsetDateTime};
 use tracing::info;
@@ -98,15 +99,37 @@ impl SigV4 {
     /// header AND folded into the canonical request — pass the hash of the
     /// body you *claim* to send, which may differ from what you actually send.
     fn sign(&self, method: &str, path: &str, canonical_query: &str, content_sha256: &str) -> SignedHeaders {
-        let amz_date = amz_datetime(self.time);
+        self.sign_with_extra_headers(method, path, canonical_query, content_sha256, &[])
-        let signed_headers = "host;x-amz-content-sha256;x-amz-date";
+    }
 
-        let canonical_headers = format!(
+    /// Sign additional request headers while preserving SigV4's lowercase,
-            "host:{host}\nx-amz-content-sha256:{sha}\nx-amz-date:{date}\n",
+    /// lexicographically sorted canonical-header representation.
-            host = self.host,
+    fn sign_with_extra_headers(
-            sha = content_sha256,
+        &self,
-            date = amz_date,
+        method: &str,
-        );
+        path: &str,
+        canonical_query: &str,
+        content_sha256: &str,
+        extra_signed_headers: &[(&str, &str)],
+    ) -> SignedHeaders {
+        let amz_date = amz_datetime(self.time);
+        let mut canonical_header_values = vec![
+            ("host", self.host.as_str()),
+            ("x-amz-content-sha256", content_sha256),
+            ("x-amz-date", amz_date.as_str()),
+        ];
+        canonical_header_values.extend(extra_signed_headers.iter().copied());
+        canonical_header_values.sort_unstable_by(|left, right| left.0.cmp(right.0));
+
+        let signed_headers = canonical_header_values
+            .iter()
+            .map(|(name, _)| *name)
+            .collect::<Vec<_>>()
+            .join(";");
+        let mut canonical_headers = String::new();
+        for (name, value) in canonical_header_values {
+            let _ = writeln!(canonical_headers, "{name}:{value}");
+        }
         let canonical_request =
             format!("{method}\n{path}\n{canonical_query}\n{canonical_headers}\n{signed_headers}\n{content_sha256}");
 
@@ -179,6 +202,34 @@ async fn setup(env: &mut RustFSTestEnvironment) -> Result<(), Box<dyn std::error
     Ok(())
 }
 
+async fn build_single_member_archive(
+    member_key: &str,
+    member_body: &[u8],
+) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
+    let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
+    let mut header = tokio_tar::Header::new_gnu();
+    header.set_size(member_body.len() as u64);
+    header.set_mode(0o644);
+    header.set_cksum();
+    builder.append_data(&mut header, member_key, Cursor::new(member_body)).await?;
+    Ok(builder.into_inner().await?.into_inner())
+}
+
+fn sha256_base64(data: &[u8]) -> String {
+    use sha2::{Digest, Sha256};
+
+    base64_simd::STANDARD.encode_to_string(Sha256::digest(data))
+}
+
+fn encode_unsigned_aws_chunked_with_sha256_trailer(decoded: &[u8]) -> Vec<u8> {
+    let checksum = sha256_base64(decoded);
+    let mut encoded = format!("{:x}\r\n", decoded.len()).into_bytes();
+    encoded.extend_from_slice(decoded);
+    encoded.extend_from_slice(b"\r\n0\r\n\r\n");
+    encoded.extend_from_slice(format!("x-amz-checksum-sha256:{checksum}").as_bytes());
+    encoded
+}
+
 /// Positive control: a correctly hand-signed request must succeed. Without
 /// this, every negative assertion below could pass for the wrong reason (a
 /// broken signer that never produces a valid signature).
@@ -249,6 +300,128 @@ async fn tampered_signature_returns_signature_does_not_match() -> Result<(), Box
     Ok(())
 }
 
+/// `STREAMING-UNSIGNED-PAYLOAD-TRAILER` disables per-chunk signatures, not the
+/// seed/header SigV4 signature. A forged request must be rejected before the
+/// Snowball handler can publish any archive member.
+#[tokio::test]
+async fn snowball_streaming_unsigned_trailer_rejects_forged_signature() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    setup(&mut env).await?;
+
+    let archive_key = "forged-streaming-snowball.tar";
+    let member_key = "must-not-be-published.txt";
+    let archive = build_single_member_archive(member_key, b"forged request payload").await?;
+    let decoded_content_length = archive.len().to_string();
+    let encoded_body = encode_unsigned_aws_chunked_with_sha256_trailer(&archive);
+    let path = format!("/{BUCKET}/{archive_key}");
+
+    let mut signer = SigV4::new(&env);
+    signer.secret_key = "wrong-secret-for-forged-streaming-request".to_string();
+    let extra_signed_headers = [
+        ("content-encoding", "aws-chunked"),
+        ("x-amz-decoded-content-length", decoded_content_length.as_str()),
+        ("x-amz-meta-snowball-auto-extract", "true"),
+        ("x-amz-trailer", "x-amz-checksum-sha256"),
+    ];
+    let headers = signer.sign_with_extra_headers("PUT", &path, "", UNSIGNED_PAYLOAD_TRAILER, &extra_signed_headers);
+
+    let response = local_http_client()
+        .put(format!("{}{}", env.url, path))
+        .header("authorization", &headers.authorization)
+        .header("content-encoding", "aws-chunked")
+        .header("x-amz-content-sha256", &headers.content_sha256)
+        .header("x-amz-date", &headers.amz_date)
+        .header("x-amz-decoded-content-length", &decoded_content_length)
+        .header("x-amz-meta-snowball-auto-extract", "true")
+        .header("x-amz-trailer", "x-amz-checksum-sha256")
+        .body(encoded_body)
+        .send()
+        .await?;
+    let status = response.status();
+    let body = response.text().await?;
+    assert_eq!(status.as_u16(), 403, "forged streaming signature must be 403, body:\n{body}");
+    assert_error_code(&body, "SignatureDoesNotMatch");
+
+    let absent = env
+        .create_s3_client()
+        .get_object()
+        .bucket(BUCKET)
+        .key(member_key)
+        .send()
+        .await
+        .expect_err("a forged streaming request must not publish a Snowball member");
+    assert_eq!(absent.raw_response().map(|response| response.status().as_u16()), Some(404));
+    assert_eq!(absent.as_service_error().and_then(ProvideErrorMetadata::code), Some("NoSuchKey"));
+
+    env.stop_server();
+    Ok(())
+}
+
+/// Snowball must consume the complete aws-chunked body before reading the
+/// trailing checksum exported by s3s into the PutObject response.
+#[tokio::test]
+async fn snowball_streaming_unsigned_trailer_returns_sha256_checksum() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    setup(&mut env).await?;
+
+    let archive_key = "valid-streaming-snowball.tar";
+    let member_key = "streaming-checksum-member.txt";
+    let member_body = b"valid streaming Snowball payload";
+    let archive = build_single_member_archive(member_key, member_body).await?;
+    let expected_checksum = sha256_base64(&archive);
+    let decoded_content_length = archive.len().to_string();
+    let encoded_body = encode_unsigned_aws_chunked_with_sha256_trailer(&archive);
+    let path = format!("/{BUCKET}/{archive_key}");
+
+    let signer = SigV4::new(&env);
+    let extra_signed_headers = [
+        ("content-encoding", "aws-chunked"),
+        ("x-amz-decoded-content-length", decoded_content_length.as_str()),
+        ("x-amz-meta-snowball-auto-extract", "true"),
+        ("x-amz-sdk-checksum-algorithm", "SHA256"),
+        ("x-amz-trailer", "x-amz-checksum-sha256"),
+    ];
+    let headers = signer.sign_with_extra_headers("PUT", &path, "", UNSIGNED_PAYLOAD_TRAILER, &extra_signed_headers);
+
+    let response = local_http_client()
+        .put(format!("{}{}", env.url, path))
+        .header("authorization", &headers.authorization)
+        .header("content-encoding", "aws-chunked")
+        .header("x-amz-content-sha256", &headers.content_sha256)
+        .header("x-amz-date", &headers.amz_date)
+        .header("x-amz-decoded-content-length", &decoded_content_length)
+        .header("x-amz-meta-snowball-auto-extract", "true")
+        .header("x-amz-sdk-checksum-algorithm", "SHA256")
+        .header("x-amz-trailer", "x-amz-checksum-sha256")
+        .body(encoded_body)
+        .send()
+        .await?;
+    let status = response.status();
+    let response_checksum = response
+        .headers()
+        .get("x-amz-checksum-sha256")
+        .and_then(|value| value.to_str().ok())
+        .map(str::to_owned);
+    let response_body = response.text().await?;
+    assert_eq!(status.as_u16(), 200, "valid streaming Snowball PUT failed, body:\n{response_body}");
+    assert_eq!(response_checksum.as_deref(), Some(expected_checksum.as_str()));
+
+    let member = env
+        .create_s3_client()
+        .get_object()
+        .bucket(BUCKET)
+        .key(member_key)
+        .send()
+        .await?;
+    let stored = member.body.collect().await?.into_bytes();
+    assert_eq!(stored.as_ref(), member_body);
+
+    env.stop_server();
+    Ok(())
+}
+
 /// (b) A valid AccessKeyId paired with the wrong secret key must be rejected
 /// with SignatureDoesNotMatch / 403.
 #[tokio::test]

crates/e2e_test/src/notification_webhook_test.rs

@@ -38,14 +38,10 @@ use aws_sdk_s3::types::{
     NotificationConfiguration, NotificationConfigurationFilter, ObjectIdentifier, QueueConfiguration, S3KeyFilter,
     VersioningConfiguration,
 };
-use http::header::{CONTENT_TYPE, HOST};
 use local_ip_address::local_ip;
 use reqwest::StatusCode;
-use rustfs_signer::constants::UNSIGNED_PAYLOAD;
-use rustfs_signer::sign_v4;
 use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
 use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER};
-use s3s::Body;
 use serde_json::Value;
 use std::error::Error;
 use std::io::Cursor;
@@ -415,42 +411,16 @@ async fn collect_until(
 // Admin target configuration (signed admin HTTP)
 // ---------------------------------------------------------------------------
 
+/// Thin wrapper over [`crate::common::signed_request`] with this suite's
+/// root credentials; a `Some` body is always JSON here.
 async fn signed_admin_request(
     env: &RustFSTestEnvironment,
     method: http::Method,
     url: &str,
     body: Option<Vec<u8>>,
 ) -> Result<reqwest::Response, BoxError> {
-    let uri = url.parse::<http::Uri>()?;
+    let content_type = body.is_some().then_some("application/json");
-    let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
+    crate::common::signed_request(method, url, &env.access_key, &env.secret_key, body, content_type).await
-    let mut builder = http::Request::builder()
-        .method(method.clone())
-        .uri(uri)
-        .header(HOST, authority)
-        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if body.is_some() {
-        builder = builder.header(CONTENT_TYPE, "application/json");
-    }
-
-    let content_len = body.as_ref().map(|b| b.len() as i64).unwrap_or_default();
-    let signed = sign_v4(
-        builder.body(Body::empty())?,
-        content_len,
-        &env.access_key,
-        &env.secret_key,
-        "",
-        "us-east-1",
-    );
-
-    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
-    let mut request = crate::common::local_http_client().request(reqwest_method, url);
-    for (name, value) in signed.headers() {
-        request = request.header(name, value);
-    }
-    if let Some(body) = body {
-        request = request.body(body);
-    }
-    Ok(request.send().await?)
 }
 
 async fn enable_notify_module(env: &RustFSTestEnvironment) -> TestResult {

crates/e2e_test/src/policy/policy_variables_test.rs

@@ -15,28 +15,24 @@
 //! Tests for AWS IAM policy variables with single-value, multi-value, and nested scenarios
 
 use crate::common::{
-    RustFSTestEnvironment, awscurl_delete, awscurl_put, build_test_s3_config, build_test_sts_client, init_logging,
+    AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via,
+    awscurl_delete, awscurl_put, build_test_s3_config, build_test_sts_client, init_logging,
 };
 use aws_sdk_s3::Client;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use tracing::info;
 
-/// Helper function to create a regular user with given credentials
+/// Helper function to create a regular user with given credentials.
+///
+/// This suite deliberately drives the admin API through the external `awscurl`
+/// binary, so the shared helpers are pinned to `AdminTransport::Awscurl`.
 async fn create_user(
     env: &RustFSTestEnvironment,
     username: &str,
     password: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let create_user_body = serde_json::json!({
+    admin_create_user_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, username, password).await
-        "secretKey": password,
-        "status": "enabled"
-    })
-    .to_string();
-
-    let create_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
-    awscurl_put(&create_user_url, &create_user_body, &env.access_key, &env.secret_key).await?;
-    Ok(())
 }
 
 /// Helper function to create and attach a policy
@@ -46,18 +42,17 @@ async fn create_and_attach_policy(
     username: &str,
     policy_document: serde_json::Value,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let policy_string = policy_document.to_string();
+    admin_add_canned_policy_via(
-
+        AdminTransport::Awscurl,
-    // Create policy
+        &env.url,
-    let add_policy_url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
+        &env.access_key,
-    awscurl_put(&add_policy_url, &policy_string, &env.access_key, &env.secret_key).await?;
+        &env.secret_key,
-
+        policy_name,
-    // Attach policy to user
+        &policy_document.to_string(),
-    let attach_policy_url = format!(
+    )
-        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
+    .await?;
-        env.url, policy_name, username
+    admin_attach_user_policy_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, policy_name, username)
-    );
+        .await?;
-    awscurl_put(&attach_policy_url, "", &env.access_key, &env.secret_key).await?;
     Ok(())
 }
 

crates/e2e_test/src/protocols/webdav_core.rs

@@ -31,15 +31,11 @@
 //!
 //! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-3p3x-734c-h5vx>
 
-use crate::common::local_http_client;
 use crate::common::rustfs_binary_path_with_features;
+use crate::common::{AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via};
 use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
 use anyhow::Result;
-use http::header::{CONTENT_TYPE, HOST};
 use reqwest::Client;
-use rustfs_signer::constants::UNSIGNED_PAYLOAD;
-use rustfs_signer::sign_v4;
-use s3s::Body;
 use tokio::process::Command;
 use tracing::info;
 
@@ -67,92 +63,43 @@ fn basic_auth_header_for(access_key: &str, secret_key: &str) -> String {
     format!("Basic {}", encoded)
 }
 
-async fn signed_admin_request(
+async fn admin_create_user(base_url: &str, username: &str, secret_key: &str) -> Result<()> {
-    method: http::Method,
+    admin_create_user_via(
-    url: &str,
+        AdminTransport::Signed,
-    body: Option<Vec<u8>>,
+        base_url,
-    content_type: Option<&str>,
-) -> Result<reqwest::Response> {
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri
-        .authority()
-        .ok_or_else(|| anyhow::anyhow!("request URL missing authority"))?
-        .to_string();
-    let mut request = http::Request::builder().method(method.clone()).uri(uri);
-    request = request.header(HOST, authority);
-    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if let Some(content_type) = content_type {
-        request = request.header(CONTENT_TYPE, content_type);
-    }
-
-    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
-    let signed = sign_v4(
-        request.body(Body::empty())?,
-        content_len,
         DEFAULT_ACCESS_KEY,
         DEFAULT_SECRET_KEY,
-        "",
+        username,
-        "us-east-1",
+        secret_key,
-    );
+    )
-
+    .await
-    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    .map_err(|e| anyhow::anyhow!(e))
-    let mut request_builder = local_http_client().request(reqwest_method, url);
-    for (name, value) in signed.headers() {
-        request_builder = request_builder.header(name, value);
-    }
-    if let Some(body) = body {
-        request_builder = request_builder.body(body);
-    }
-
-    Ok(request_builder.send().await?)
-}
-
-async fn admin_create_user(base_url: &str, username: &str, secret_key: &str) -> Result<()> {
-    let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", base_url, username);
-    let body = serde_json::json!({
-        "secretKey": secret_key,
-        "status": "enabled"
-    });
-    let response =
-        signed_admin_request(http::Method::PUT, &url, Some(body.to_string().into_bytes()), Some("application/json")).await?;
-
-    if response.status() != reqwest::StatusCode::OK {
-        let status = response.status();
-        let body = response.text().await.unwrap_or_default();
-        anyhow::bail!("create user failed: {status} {body}");
-    }
-
-    Ok(())
 }
 
 async fn admin_add_canned_policy(base_url: &str, policy_name: &str, policy: &serde_json::Value) -> Result<()> {
-    let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", base_url, policy_name);
+    admin_add_canned_policy_via(
-    let response =
+        AdminTransport::Signed,
-        signed_admin_request(http::Method::PUT, &url, Some(policy.to_string().into_bytes()), Some("application/json")).await?;
+        base_url,
-
+        DEFAULT_ACCESS_KEY,
-    if response.status() != reqwest::StatusCode::OK {
+        DEFAULT_SECRET_KEY,
-        let status = response.status();
+        policy_name,
-        let body = response.text().await.unwrap_or_default();
+        &policy.to_string(),
-        anyhow::bail!("add canned policy failed: {status} {body}");
+    )
-    }
+    .await
-
+    .map_err(|e| anyhow::anyhow!(e))
-    Ok(())
 }
 
 async fn admin_attach_policy_to_user(base_url: &str, policy_name: &str, username: &str) -> Result<()> {
-    let url = format!(
+    admin_attach_user_policy_via(
-        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
+        AdminTransport::Signed,
-        base_url, policy_name, username
+        base_url,
-    );
+        DEFAULT_ACCESS_KEY,
-    let response = signed_admin_request(http::Method::PUT, &url, Some(Vec::new()), None).await?;
+        DEFAULT_SECRET_KEY,
-
+        policy_name,
-    if response.status() != reqwest::StatusCode::OK {
+        username,
-        let status = response.status();
+    )
-        let body = response.text().await.unwrap_or_default();
+    .await
-        anyhow::bail!("attach policy failed: {status} {body}");
+    .map_err(|e| anyhow::anyhow!(e))
-    }
-
-    Ok(())
 }
 
 /// Test WebDAV: MKCOL (create bucket), PUT, GET, DELETE, PROPFIND operations

crates/e2e_test/src/put_object_no_content_length_test.rs

@@ -0,0 +1,152 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression coverage for rustfs#6830: a signed empty `PutObject` request
+//! without `Content-Length` and without `Transfer-Encoding` is still a
+//! zero-length object upload.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use http::header::{CONTENT_LENGTH, HOST, TRANSFER_ENCODING};
+    use rustfs_signer::sign_v4;
+    use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+    use s3s::Body;
+    use std::error::Error;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    use tokio::net::TcpStream;
+    use tokio::time::{Duration, timeout};
+    use tracing::info;
+
+    const RAW_RESPONSE_TIMEOUT: Duration = Duration::from_secs(10);
+
+    fn parse_status(raw_response: &str) -> Option<u16> {
+        raw_response.lines().next()?.split_whitespace().nth(1)?.parse().ok()
+    }
+
+    async fn send_raw_signed_put(
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+        transfer_encoding: Option<&str>,
+        raw_body: &[u8],
+    ) -> Result<String, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let path_and_query = uri.path_and_query().ok_or("request URL missing path")?.as_str().to_string();
+
+        let mut request = http::Request::builder()
+            .method(http::Method::PUT)
+            .uri(uri)
+            .header(HOST, authority.clone())
+            .header("x-amz-content-sha256", EMPTY_STRING_SHA256_HASH);
+        if let Some(value) = transfer_encoding {
+            request = request.header(TRANSFER_ENCODING, value);
+        }
+
+        let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
+
+        let mut raw_request = format!("PUT {path_and_query} HTTP/1.1\r\nHost: {authority}\r\nConnection: close\r\n");
+        for (name, value) in signed.headers() {
+            if name == HOST || name == CONTENT_LENGTH {
+                continue;
+            }
+            raw_request.push_str(name.as_str());
+            raw_request.push_str(": ");
+            raw_request.push_str(value.to_str()?);
+            raw_request.push_str("\r\n");
+        }
+        raw_request.push_str("\r\n");
+
+        assert!(
+            !raw_request.to_ascii_lowercase().contains("\r\ncontent-length:"),
+            "raw regression request must omit Content-Length; request was:\n{raw_request}"
+        );
+
+        let mut stream = TcpStream::connect(&authority).await?;
+        stream.write_all(raw_request.as_bytes()).await?;
+        stream.write_all(raw_body).await?;
+        stream.flush().await?;
+
+        let mut response = Vec::new();
+        timeout(RAW_RESPONSE_TIMEOUT, stream.read_to_end(&mut response))
+            .await
+            .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out reading raw PUT response"))??;
+        Ok(String::from_utf8_lossy(&response).into_owned())
+    }
+
+    #[tokio::test]
+    async fn test_put_object_without_content_length_boundaries() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        info!("TEST: PutObject without Content-Length boundaries");
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let empty_bucket = "put-no-content-length";
+        let empty_key = "empty.bin";
+        let chunked_bucket = "put-chunked-no-length";
+        let chunked_key = "chunked.bin";
+
+        client.create_bucket().bucket(empty_bucket).send().await?;
+        client.create_bucket().bucket(chunked_bucket).send().await?;
+
+        let url = format!("{}/{}/{}", env.url, empty_bucket, empty_key);
+        let raw_response = send_raw_signed_put(&url, &env.access_key, &env.secret_key, None, b"").await?;
+        info!("raw empty PUT response:\n{}", raw_response);
+
+        assert_eq!(
+            parse_status(&raw_response),
+            Some(200),
+            "empty PutObject without Content-Length should succeed, got:\n{raw_response}"
+        );
+        assert!(
+            raw_response.to_ascii_lowercase().contains("\r\netag:"),
+            "successful PutObject should return an ETag header: {raw_response}"
+        );
+
+        let head = client.head_object().bucket(empty_bucket).key(empty_key).send().await?;
+        assert_eq!(head.content_length(), Some(0), "stored object must be zero length");
+
+        let url = format!("{}/{}/{}", env.url, chunked_bucket, chunked_key);
+        let raw_response = send_raw_signed_put(&url, &env.access_key, &env.secret_key, Some("chunked"), b"0\r\n\r\n").await?;
+        info!("raw chunked PUT response:\n{}", raw_response);
+
+        assert_eq!(
+            parse_status(&raw_response),
+            Some(411),
+            "unknown-length chunked PutObject must stay rejected, got:\n{raw_response}"
+        );
+        assert!(
+            raw_response.contains("<Code>MissingContentLength</Code>"),
+            "expected MissingContentLength, got:\n{raw_response}"
+        );
+
+        let missing = client
+            .head_object()
+            .bucket(chunked_bucket)
+            .key(chunked_key)
+            .send()
+            .await
+            .expect_err("rejected unknown-length PUT must not create an object");
+        assert_eq!(
+            missing.raw_response().map(|response| response.status().as_u16()),
+            Some(404),
+            "rejected unknown-length PUT absence probe must return HTTP 404, got {missing:?}"
+        );
+
+        Ok(())
+    }
+}

crates/e2e_test/src/reliant/mod.rs

@@ -21,5 +21,6 @@ mod head_tls_bodyless_test;
 mod lifecycle;
 mod lock;
 mod node_interact_test;
+mod s3_select_compression;
 mod sql;
 mod tiering;

crates/e2e_test/src/reliant/s3_select_compression.rs

@@ -0,0 +1,351 @@
+#![cfg(test)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::common::{RustFSTestEnvironment, init_logging};
+use async_compression::tokio::write::BzEncoder;
+use aws_sdk_s3::{
+    Client,
+    error::ProvideErrorMetadata,
+    operation::select_object_content::{SelectObjectContentOutput, builders::SelectObjectContentFluentBuilder},
+    types::{
+        CompressionType, CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput,
+        JsonType, OutputSerialization, SelectObjectContentEventStream,
+    },
+};
+use aws_smithy_types::event_stream::RawMessage;
+use bytes::Bytes;
+use flate2::{Compression, write::GzEncoder};
+use std::{error::Error, io::Cursor, time::Duration};
+use tokio::io::AsyncWriteExt;
+
+const BUCKET: &str = "s3-select-compression";
+const SELECT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(30);
+
+type TestResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
+
+async fn create_test_environment(extra_env: &[(&str, &str)]) -> TestResult<(RustFSTestEnvironment, Client)> {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server_with_env(vec![], extra_env).await?;
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(BUCKET).send().await?;
+    Ok((env, client))
+}
+
+async fn put_object(client: &Client, key: &str, body: &[u8]) -> TestResult<()> {
+    client
+        .put_object()
+        .bucket(BUCKET)
+        .key(key)
+        .body(Bytes::copy_from_slice(body).into())
+        .send()
+        .await?;
+    Ok(())
+}
+
+fn gzip(input: &[u8]) -> TestResult<Vec<u8>> {
+    let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
+    std::io::Write::write_all(&mut encoder, input)?;
+    Ok(encoder.finish()?)
+}
+
+async fn bzip2(input: &[u8]) -> TestResult<Vec<u8>> {
+    let mut encoder = BzEncoder::new(Cursor::new(Vec::new()));
+    encoder.write_all(input).await?;
+    encoder.shutdown().await?;
+    Ok(encoder.into_inner().into_inner())
+}
+
+fn csv_select_request(
+    client: &Client,
+    key: &str,
+    compression: CompressionType,
+    expression: &str,
+) -> SelectObjectContentFluentBuilder {
+    client
+        .select_object_content()
+        .bucket(BUCKET)
+        .key(key)
+        .expression(expression)
+        .expression_type(ExpressionType::Sql)
+        .input_serialization(
+            InputSerialization::builder()
+                .compression_type(compression)
+                .csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build())
+                .build(),
+        )
+        .output_serialization(OutputSerialization::builder().csv(CsvOutput::builder().build()).build())
+}
+
+fn json_select_request(
+    client: &Client,
+    key: &str,
+    compression: CompressionType,
+    json_type: JsonType,
+) -> SelectObjectContentFluentBuilder {
+    client
+        .select_object_content()
+        .bucket(BUCKET)
+        .key(key)
+        .expression("SELECT name FROM S3Object")
+        .expression_type(ExpressionType::Sql)
+        .input_serialization(
+            InputSerialization::builder()
+                .compression_type(compression)
+                .json(JsonInput::builder().set_type(Some(json_type)).build())
+                .build(),
+        )
+        .output_serialization(OutputSerialization::builder().json(JsonOutput::builder().build()).build())
+}
+
+async fn collect_success(
+    mut response: SelectObjectContentOutput,
+    compressed_bytes: usize,
+    processed_bytes: usize,
+) -> TestResult<Vec<u8>> {
+    tokio::time::timeout(SELECT_RESPONSE_TIMEOUT, async move {
+        let mut records = Vec::new();
+        let mut stats = None;
+        let mut saw_end = false;
+
+        while let Some(event) = response.payload.recv().await? {
+            assert!(!saw_end, "Select emitted an event after End");
+            match event {
+                SelectObjectContentEventStream::Records(event) => {
+                    assert!(stats.is_none(), "Select emitted Records after Stats");
+                    if let Some(payload) = event.payload {
+                        records.extend_from_slice(payload.as_ref());
+                    }
+                }
+                SelectObjectContentEventStream::Stats(event) => {
+                    assert!(stats.is_none(), "Select emitted more than one Stats event");
+                    stats = event.details;
+                }
+                SelectObjectContentEventStream::End(_) => {
+                    assert!(stats.is_some(), "Select emitted End before Stats");
+                    saw_end = true;
+                }
+                _ => assert!(stats.is_none(), "Select emitted a non-terminal event after Stats"),
+            }
+        }
+
+        let stats = stats.ok_or("Select response ended without a Stats event")?;
+        assert_eq!(stats.bytes_scanned(), Some(i64::try_from(compressed_bytes)?));
+        assert_eq!(stats.bytes_processed(), Some(i64::try_from(processed_bytes)?));
+        assert_eq!(stats.bytes_returned(), Some(i64::try_from(records.len())?));
+        assert!(saw_end, "Select response ended without an End event");
+        Ok::<_, Box<dyn Error + Send + Sync>>(records)
+    })
+    .await
+    .map_err(|_| -> Box<dyn Error + Send + Sync> { "Select response timed out".into() })?
+}
+
+async fn assert_truncated_stream_failure(mut response: SelectObjectContentOutput) -> TestResult<()> {
+    tokio::time::timeout(SELECT_RESPONSE_TIMEOUT, async move {
+        loop {
+            match response.payload.recv().await {
+                Err(error) => {
+                    // S3 Select request-level errors use `error` frames, which this SDK version exposes as raw response errors.
+                    if let Some(code) = error.code() {
+                        assert_eq!(code, "TruncatedInput", "unexpected modeled event-stream error: {error:?}");
+                    } else if let aws_sdk_s3::error::SdkError::ResponseError(context) = &error
+                        && let RawMessage::Decoded(message) = context.raw()
+                    {
+                        let header = |name: &str| {
+                            message
+                                .headers()
+                                .iter()
+                                .find(|header| header.name().as_str() == name)
+                                .and_then(|header| header.value().as_string().ok())
+                                .map(|value| value.as_str())
+                        };
+                        assert_eq!(header(":message-type"), Some("error"));
+                        assert_eq!(header(":error-code"), Some("TruncatedInput"));
+                    } else {
+                        panic!("unexpected event-stream error: {error:?}");
+                    }
+                    return Ok(());
+                }
+                Ok(Some(SelectObjectContentEventStream::Stats(_))) | Ok(Some(SelectObjectContentEventStream::End(_))) => {
+                    return Err("truncated compressed input reached a success terminal event".into());
+                }
+                Ok(Some(_)) => {}
+                Ok(None) => return Err("truncated compressed input ended without an error event".into()),
+            }
+        }
+    })
+    .await
+    .map_err(|_| -> Box<dyn Error + Send + Sync> { "truncated Select response timed out".into() })?
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn test_select_object_content_compressed_csv_and_json() -> TestResult<()> {
+    const CSV: &[u8] = b"name,age\nAlice,30\nBob,25\n";
+    const JSON_LINES: &[u8] = b"{\"name\":\"Alice\"}\n{\"name\":\"Bob\"}\n";
+    const JSON_DOCUMENT: &[u8] = br#"[{"name":"Alice"},{"name":"Bob"}]"#;
+
+    let (_env, client) = create_test_environment(&[]).await?;
+
+    let gzip_csv = gzip(CSV)?;
+    put_object(&client, "records.csv.gz", &gzip_csv).await?;
+    let gzip_csv_records = collect_success(
+        csv_select_request(&client, "records.csv.gz", CompressionType::Gzip, "SELECT * FROM S3Object")
+            .send()
+            .await?,
+        gzip_csv.len(),
+        CSV.len(),
+    )
+    .await?;
+    assert_eq!(gzip_csv_records, b"Alice,30\nBob,25\n");
+
+    let bzip_csv = bzip2(CSV).await?;
+    put_object(&client, "records.csv.bz2", &bzip_csv).await?;
+    let bzip_csv_records = collect_success(
+        csv_select_request(&client, "records.csv.bz2", CompressionType::Bzip2, "SELECT * FROM S3Object")
+            .send()
+            .await?,
+        bzip_csv.len(),
+        CSV.len(),
+    )
+    .await?;
+    assert_eq!(bzip_csv_records, gzip_csv_records);
+
+    let gzip_json_lines = gzip(JSON_LINES)?;
+    put_object(&client, "json-lines", &gzip_json_lines).await?;
+    let gzip_json_records = collect_success(
+        json_select_request(&client, "json-lines", CompressionType::Gzip, JsonType::Lines)
+            .send()
+            .await?,
+        gzip_json_lines.len(),
+        JSON_LINES.len(),
+    )
+    .await?;
+    assert_eq!(gzip_json_records, JSON_LINES);
+
+    let bzip_json_lines = bzip2(JSON_LINES).await?;
+    put_object(&client, "records.jsonl.bz2", &bzip_json_lines).await?;
+    let bzip_json_records = collect_success(
+        json_select_request(&client, "records.jsonl.bz2", CompressionType::Bzip2, JsonType::Lines)
+            .send()
+            .await?,
+        bzip_json_lines.len(),
+        JSON_LINES.len(),
+    )
+    .await?;
+    assert_eq!(bzip_json_records, gzip_json_records);
+
+    let gzip_json_document = gzip(JSON_DOCUMENT)?;
+    put_object(&client, "document.json.gz", &gzip_json_document).await?;
+    let document_records = collect_success(
+        json_select_request(&client, "document.json.gz", CompressionType::Gzip, JsonType::Document)
+            .send()
+            .await?,
+        gzip_json_document.len(),
+        JSON_DOCUMENT.len(),
+    )
+    .await?;
+    assert_eq!(document_records, JSON_LINES);
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn test_select_object_content_invalid_compressed_stream_fails() -> TestResult<()> {
+    const CSV: &[u8] = b"name\nAlice\n";
+
+    let (_env, client) = create_test_environment(&[]).await?;
+
+    put_object(&client, "invalid.csv.gz", CSV).await?;
+    let invalid = csv_select_request(&client, "invalid.csv.gz", CompressionType::Gzip, "SELECT * FROM S3Object")
+        .send()
+        .await
+        .expect_err("invalid GZIP header must fail before streaming");
+    assert_eq!(
+        invalid.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("InvalidCompressionFormat")
+    );
+
+    put_object(&client, "empty.csv.gz", b"").await?;
+    let empty = csv_select_request(&client, "empty.csv.gz", CompressionType::Gzip, "SELECT * FROM S3Object")
+        .send()
+        .await
+        .expect_err("empty GZIP input must fail as truncated");
+    assert_eq!(empty.as_service_error().and_then(ProvideErrorMetadata::code), Some("TruncatedInput"));
+
+    let mut truncated = bzip2(CSV).await?;
+    truncated.pop();
+    put_object(&client, "truncated.csv.bz2", &truncated).await?;
+    let truncated = csv_select_request(&client, "truncated.csv.bz2", CompressionType::Bzip2, "SELECT * FROM S3Object")
+        .send()
+        .await?;
+    assert_truncated_stream_failure(truncated).await?;
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn test_select_object_content_compressed_disconnect_releases_query() -> TestResult<()> {
+    const OBJECT: &str = "disconnect.csv.gz";
+    const ROWS: usize = 16 * 1024;
+    const RELEASE_ATTEMPTS: usize = 20;
+    const RELEASE_BACKOFF: Duration = Duration::from_millis(25);
+
+    let (_env, client) = create_test_environment(&[("RUSTFS_S3SELECT_MAX_CONCURRENT_QUERIES", "1")]).await?;
+    let row = format!("{}\n", "x".repeat(1023));
+    let mut body = Vec::with_capacity("value\n".len() + ROWS * row.len());
+    body.extend_from_slice(b"value\n");
+    for _ in 0..ROWS {
+        body.extend_from_slice(row.as_bytes());
+    }
+    let compressed = gzip(&body)?;
+    put_object(&client, OBJECT, &compressed).await?;
+
+    let first = csv_select_request(&client, OBJECT, CompressionType::Gzip, "SELECT * FROM S3Object")
+        .send()
+        .await?;
+    let saturated = csv_select_request(&client, OBJECT, CompressionType::Gzip, "SELECT * FROM S3Object")
+        .send()
+        .await
+        .expect_err("the unread compressed response should retain the only query permit");
+    assert_eq!(saturated.as_service_error().and_then(ProvideErrorMetadata::code), Some("SlowDown"));
+
+    drop(first);
+    let second = tokio::time::timeout(Duration::from_secs(5), async {
+        for attempt in 0..RELEASE_ATTEMPTS {
+            match csv_select_request(&client, OBJECT, CompressionType::Gzip, "SELECT * FROM S3Object")
+                .send()
+                .await
+            {
+                Ok(response) => return Ok::<_, Box<dyn Error + Send + Sync>>(response),
+                Err(error)
+                    if error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("SlowDown")
+                        && attempt + 1 < RELEASE_ATTEMPTS =>
+                {
+                    tokio::time::sleep(RELEASE_BACKOFF).await;
+                }
+                Err(error) if error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("SlowDown") => {
+                    return Err("disconnected compressed Select retained its query permit".into());
+                }
+                Err(error) => return Err(format!("unexpected Select error after disconnect: {error}").into()),
+            }
+        }
+        Err("query permit release retry loop ended unexpectedly".into())
+    })
+    .await
+    .map_err(|_| -> Box<dyn Error + Send + Sync> { "compressed Select did not release its query permit".into() })??;
+    drop(second);
+
+    Ok(())
+}

crates/e2e_test/src/reliant/sql.rs

@@ -17,7 +17,8 @@ use crate::common::{RustFSTestEnvironment, init_logging};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::types::{
-    CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput, JsonType, OutputSerialization,
+    CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput, JsonType,
+    OutputSerialization, RequestProgress,
 };
 use bytes::Bytes;
 use std::error::Error;
@@ -26,6 +27,9 @@ use std::time::Duration;
 const BUCKET: &str = "test-sql-bucket";
 const CSV_OBJECT: &str = "test-data.csv";
 const JSON_OBJECT: &str = "test-data.json";
+const JSON_DOCUMENT_OBJECT: &str = "nested-data.json";
+const JSON_ROOT_ARRAY_OBJECT: &str = "root-array.json";
+const JSON_ROOT_SCALAR_ARRAY_OBJECT: &str = "root-scalars.json";
 const SELECT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(30);
 
 type TestResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
@@ -73,6 +77,69 @@ async fn upload_test_json(client: &Client) -> TestResult<()> {
     Ok(())
 }
 
+async fn upload_nested_json_document(client: &Client) -> TestResult<()> {
+    let json_data = r#"{"departments":[{"employees":[{"name":"Alice","active":true},{"name":"Bob","active":false}]},{"employees":[{"name":"Charlie","active":true}]}]}"#;
+
+    client
+        .put_object()
+        .bucket(BUCKET)
+        .key(JSON_DOCUMENT_OBJECT)
+        .body(Bytes::from_static(json_data.as_bytes()).into())
+        .send()
+        .await?;
+    client
+        .put_object()
+        .bucket(BUCKET)
+        .key(JSON_ROOT_ARRAY_OBJECT)
+        .body(Bytes::from_static(br#"[{"name":"Alice"},{"name":"Bob"}]"#).into())
+        .send()
+        .await?;
+    client
+        .put_object()
+        .bucket(BUCKET)
+        .key(JSON_ROOT_SCALAR_ARRAY_OBJECT)
+        .body(Bytes::from_static(b"[1,2]").into())
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn select_json_document(client: &Client, key: &str, expression: &str) -> TestResult<String> {
+    let response = client
+        .select_object_content()
+        .bucket(BUCKET)
+        .key(key)
+        .expression(expression)
+        .expression_type(ExpressionType::Sql)
+        .input_serialization(
+            InputSerialization::builder()
+                .json(JsonInput::builder().set_type(Some(JsonType::Document)).build())
+                .build(),
+        )
+        .output_serialization(OutputSerialization::builder().json(JsonOutput::builder().build()).build())
+        .send()
+        .await?;
+    process_select_response(response).await
+}
+
+fn csv_select_request(
+    client: &Client,
+    key: &str,
+) -> aws_sdk_s3::operation::select_object_content::builders::SelectObjectContentFluentBuilder {
+    client
+        .select_object_content()
+        .bucket(BUCKET)
+        .key(key)
+        .expression("SELECT * FROM S3Object")
+        .expression_type(ExpressionType::Sql)
+        .input_serialization(
+            InputSerialization::builder()
+                .csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build())
+                .build(),
+        )
+        .output_serialization(OutputSerialization::builder().csv(CsvOutput::builder().build()).build())
+}
+
 async fn process_select_response(
     mut event_stream: aws_sdk_s3::operation::select_object_content::SelectObjectContentOutput,
 ) -> TestResult<String> {
@@ -104,6 +171,209 @@ async fn process_select_response(
     .map_err(|_| -> Box<dyn Error + Send + Sync> { "Select response timed out".into() })?
 }
 
+async fn assert_input_byte_stats(
+    client: &Client,
+    object: &str,
+    body: &[u8],
+    expression: &str,
+    input_serialization: InputSerialization,
+    output_serialization: OutputSerialization,
+    progress_enabled: bool,
+) -> TestResult<()> {
+    client
+        .put_object()
+        .bucket(BUCKET)
+        .key(object)
+        .body(Bytes::copy_from_slice(body).into())
+        .send()
+        .await?;
+
+    let mut request = client
+        .select_object_content()
+        .bucket(BUCKET)
+        .key(object)
+        .expression(expression)
+        .expression_type(ExpressionType::Sql)
+        .input_serialization(input_serialization)
+        .output_serialization(output_serialization);
+    if progress_enabled {
+        request = request.request_progress(RequestProgress::builder().enabled(true).build());
+    }
+    let response = request.send().await?;
+
+    let mut payload = response.payload;
+    let mut records_len = 0_u64;
+    let mut last_progress: Option<aws_sdk_s3::types::Progress> = None;
+    let mut stats = None;
+    let mut saw_end = false;
+    tokio::time::timeout(SELECT_RESPONSE_TIMEOUT, async {
+        // The AWS SDK validates both event-stream CRCs before yielding an event.
+        while let Some(event) = payload.recv().await? {
+            assert!(!saw_end, "Select emitted an event after End");
+            match event {
+                aws_sdk_s3::types::SelectObjectContentEventStream::Records(records) => {
+                    assert!(stats.is_none(), "Select emitted Records after Stats");
+                    if let Some(bytes) = records.payload {
+                        records_len = records_len.saturating_add(u64::try_from(bytes.as_ref().len())?);
+                    }
+                }
+                aws_sdk_s3::types::SelectObjectContentEventStream::Progress(event) => {
+                    assert!(stats.is_none(), "Select emitted Progress after Stats");
+                    let details = event.details.ok_or("Progress event did not contain details")?;
+                    if let Some(previous) = last_progress.as_ref() {
+                        assert!(details.bytes_scanned() >= previous.bytes_scanned());
+                        assert!(details.bytes_processed() >= previous.bytes_processed());
+                        assert!(details.bytes_returned() >= previous.bytes_returned());
+                    }
+                    last_progress = Some(details);
+                }
+                aws_sdk_s3::types::SelectObjectContentEventStream::Stats(event) => {
+                    assert!(stats.is_none(), "Select emitted more than one Stats event");
+                    stats = event.details;
+                }
+                aws_sdk_s3::types::SelectObjectContentEventStream::End(_) => {
+                    assert!(stats.is_some(), "Select emitted End before Stats");
+                    saw_end = true;
+                }
+                _ => assert!(stats.is_none(), "Select emitted a non-terminal event after Stats"),
+            }
+        }
+        Ok::<(), Box<dyn Error + Send + Sync>>(())
+    })
+    .await
+    .map_err(|_| -> Box<dyn Error + Send + Sync> { "Select response timed out".into() })??;
+
+    let stats = stats.ok_or("Select response ended without a Stats event")?;
+    let input_len = i64::try_from(body.len())?;
+    assert_eq!(stats.bytes_scanned(), Some(input_len));
+    assert_eq!(stats.bytes_processed(), Some(input_len));
+    assert_eq!(stats.bytes_returned(), Some(i64::try_from(records_len)?));
+    if progress_enabled {
+        if let Some(progress) = last_progress {
+            assert!(stats.bytes_scanned() >= progress.bytes_scanned());
+            assert!(stats.bytes_processed() >= progress.bytes_processed());
+            assert!(stats.bytes_returned() >= progress.bytes_returned());
+        }
+    } else {
+        assert!(last_progress.is_none(), "disabled request progress emitted a Progress event");
+    }
+    assert!(saw_end, "Select response ended without an End event");
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn test_select_object_content_http_event_order_crc_and_input_byte_stats() -> TestResult<()> {
+    const CSV_BODY: &[u8] = b"name,age\nAlice,30\nBob,25\n";
+    const JSON_LINES_BODY: &[u8] = b"{\"name\":\"Alice\"}\n{\"name\":\"Bob\"}\n";
+    const JSON_DOCUMENT_BODY: &[u8] = b"[{\"name\":\"Alice\"},{\"name\":\"Bob\"}]";
+
+    let (_env, client) = create_test_environment().await?;
+    setup_test_bucket(&client).await?;
+    assert_input_byte_stats(
+        &client,
+        "input-metrics.csv",
+        CSV_BODY,
+        "SELECT name FROM S3Object",
+        InputSerialization::builder()
+            .csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build())
+            .build(),
+        OutputSerialization::builder().csv(CsvOutput::builder().build()).build(),
+        true,
+    )
+    .await?;
+    assert_input_byte_stats(
+        &client,
+        "input-metrics.jsonl",
+        JSON_LINES_BODY,
+        "SELECT name FROM S3Object",
+        InputSerialization::builder()
+            .json(JsonInput::builder().set_type(Some(JsonType::Lines)).build())
+            .build(),
+        OutputSerialization::builder().json(JsonOutput::builder().build()).build(),
+        true,
+    )
+    .await?;
+    assert_input_byte_stats(
+        &client,
+        "input-metrics.json",
+        JSON_DOCUMENT_BODY,
+        "SELECT name FROM S3Object",
+        InputSerialization::builder()
+            .json(JsonInput::builder().set_type(Some(JsonType::Document)).build())
+            .build(),
+        OutputSerialization::builder().json(JsonOutput::builder().build()).build(),
+        true,
+    )
+    .await?;
+    assert_input_byte_stats(
+        &client,
+        "input-metrics-without-progress.csv",
+        CSV_BODY,
+        "SELECT name FROM S3Object",
+        InputSerialization::builder()
+            .csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build())
+            .build(),
+        OutputSerialization::builder().csv(CsvOutput::builder().build()).build(),
+        false,
+    )
+    .await?;
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn test_select_object_content_http_disconnect_releases_query() -> TestResult<()> {
+    const OBJECT: &str = "disconnect.csv";
+    const ROWS: usize = 16 * 1024;
+    const RELEASE_BACKOFF: Duration = Duration::from_millis(25);
+
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server_with_env(vec![], &[("RUSTFS_S3SELECT_MAX_CONCURRENT_QUERIES", "1")])
+        .await?;
+    let client = env.create_s3_client();
+    setup_test_bucket(&client).await?;
+
+    let row = format!("{}\n", "x".repeat(1023));
+    let mut body = Vec::with_capacity("value\n".len() + ROWS * row.len());
+    body.extend_from_slice(b"value\n");
+    for _ in 0..ROWS {
+        body.extend_from_slice(row.as_bytes());
+    }
+    client
+        .put_object()
+        .bucket(BUCKET)
+        .key(OBJECT)
+        .body(Bytes::from(body).into())
+        .send()
+        .await?;
+
+    // Leaving this response body unread fills the bounded HTTP/event channels before the query can finish.
+    let first = csv_select_request(&client, OBJECT).send().await?;
+    let saturated = csv_select_request(&client, OBJECT)
+        .send()
+        .await
+        .expect_err("the first HTTP stream should retain the only query permit");
+    assert_eq!(saturated.as_service_error().and_then(ProvideErrorMetadata::code), Some("SlowDown"));
+
+    drop(first);
+    let second = tokio::time::timeout(Duration::from_secs(5), async {
+        loop {
+            match csv_select_request(&client, OBJECT).send().await {
+                Ok(response) => return Ok::<_, Box<dyn Error + Send + Sync>>(response),
+                Err(error) if error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("SlowDown") => {
+                    tokio::time::sleep(RELEASE_BACKOFF).await;
+                }
+                Err(error) => return Err(format!("unexpected Select error after disconnect: {error}").into()),
+            }
+        }
+    })
+    .await
+    .map_err(|_| -> Box<dyn Error + Send + Sync> { "disconnected Select did not release its query permit".into() })??;
+    drop(second);
+
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
 async fn test_select_object_content_csv_basic() -> TestResult<()> {
     let (_env, client) = create_test_environment().await?;
@@ -228,6 +498,107 @@ async fn test_select_object_content_json_basic() -> TestResult<()> {
     Ok(())
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn test_select_object_content_nested_json_source_path() -> TestResult<()> {
+    let (_env, client) = create_test_environment().await?;
+    setup_test_bucket(&client).await?;
+    upload_nested_json_document(&client).await?;
+
+    let result = select_json_document(
+        &client,
+        JSON_DOCUMENT_OBJECT,
+        "SELECT e.name FROM S3Object[*].departments[*].employees[*] AS e WHERE e.active = true",
+    )
+    .await?;
+    let names: Vec<String> = result
+        .lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| -> TestResult<String> {
+            let value: serde_json::Value = serde_json::from_str(line)?;
+            Ok(value["name"].as_str().ok_or("missing name field")?.to_string())
+        })
+        .collect::<TestResult<_>>()?;
+
+    assert_eq!(names, vec!["Alice", "Charlie"]);
+
+    let terminal_scalars = select_json_document(
+        &client,
+        JSON_DOCUMENT_OBJECT,
+        "SELECT NAME FROM S3Object[*].DEPARTMENTS[*].employees[*].NAME",
+    )
+    .await?;
+    let scalar_names: Vec<String> = terminal_scalars
+        .lines()
+        .map(|line| -> TestResult<String> {
+            let value: serde_json::Value = serde_json::from_str(line)?;
+            Ok(value["name"].as_str().ok_or("missing scalar name field")?.to_string())
+        })
+        .collect::<TestResult<_>>()?;
+    assert_eq!(scalar_names, vec!["Alice", "Bob", "Charlie"]);
+
+    let aliased_scalars = select_json_document(
+        &client,
+        JSON_DOCUMENT_OBJECT,
+        "SELECT v FROM S3Object[*].departments[*].employees[*].name AS v",
+    )
+    .await?;
+    let aliased_names: Vec<String> = aliased_scalars
+        .lines()
+        .map(|line| -> TestResult<String> {
+            let value: serde_json::Value = serde_json::from_str(line)?;
+            Ok(value["v"].as_str().ok_or("missing aliased scalar field")?.to_string())
+        })
+        .collect::<TestResult<_>>()?;
+    assert_eq!(aliased_names, vec!["Alice", "Bob", "Charlie"]);
+
+    let root_array = select_json_document(&client, JSON_ROOT_ARRAY_OBJECT, "SELECT c.name FROM S3Object[*][*] AS c").await?;
+    let root_names: Vec<String> = root_array
+        .lines()
+        .map(|line| -> TestResult<String> {
+            let value: serde_json::Value = serde_json::from_str(line)?;
+            Ok(value["name"].as_str().ok_or("missing root-array name field")?.to_string())
+        })
+        .collect::<TestResult<_>>()?;
+    assert_eq!(root_names, vec!["Alice", "Bob"]);
+
+    let root_index = select_json_document(&client, JSON_ROOT_ARRAY_OBJECT, "SELECT c.name FROM S3Object[*][0] AS c").await?;
+    let root_index_value: serde_json::Value = serde_json::from_str(root_index.trim())?;
+    assert_eq!(root_index_value["name"], "Alice");
+
+    let root_scalars = select_json_document(&client, JSON_ROOT_SCALAR_ARRAY_OBJECT, "SELECT V FROM S3Object AS V").await?;
+    let scalar_values: Vec<i64> = root_scalars
+        .lines()
+        .map(|line| -> TestResult<i64> {
+            let value: serde_json::Value = serde_json::from_str(line)?;
+            Ok(value["v"].as_i64().ok_or("missing root scalar value")?)
+        })
+        .collect::<TestResult<_>>()?;
+    assert_eq!(scalar_values, vec![1, 2]);
+
+    let implicit_root_scalars =
+        select_json_document(&client, JSON_ROOT_SCALAR_ARRAY_OBJECT, "SELECT S3Object FROM S3Object").await?;
+    let implicit_scalar_values: Vec<i64> = implicit_root_scalars
+        .lines()
+        .map(|line| -> TestResult<i64> {
+            let value: serde_json::Value = serde_json::from_str(line)?;
+            Ok(value["s3object"].as_i64().ok_or("missing implicit root scalar value")?)
+        })
+        .collect::<TestResult<_>>()?;
+    assert_eq!(implicit_scalar_values, vec![1, 2]);
+
+    let quoted_root_scalars =
+        select_json_document(&client, JSON_ROOT_SCALAR_ARRAY_OBJECT, "SELECT \"S3Object\" FROM \"S3Object\"").await?;
+    let quoted_scalar_values: Vec<i64> = quoted_root_scalars
+        .lines()
+        .map(|line| -> TestResult<i64> {
+            let value: serde_json::Value = serde_json::from_str(line)?;
+            Ok(value["S3Object"].as_i64().ok_or("missing quoted root scalar value")?)
+        })
+        .collect::<TestResult<_>>()?;
+    assert_eq!(quoted_scalar_values, vec![1, 2]);
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
 async fn test_select_object_content_csv_limit() -> TestResult<()> {
     let (_env, client) = create_test_environment().await?;

crates/e2e_test/src/reliant/tiering.rs

@@ -23,9 +23,9 @@
 //!
 //! There are no containers, no external S3 backend and no `awscurl`: the
 //! `AddTier` admin call is signed in-process with `rustfs_signer`, exactly like
-//! the other admin-API e2e suites in this crate. The RustFS warm backend has no
+//! the other admin-API e2e suites in this crate. The source server uses the
-//! loopback/SSRF restriction (that guard is replication-only), so `hot` can tier
+//! explicit test-only loopback opt-in to tier to `cold` over
-//! to `cold` over `http://127.0.0.1:<port>`.
+//! `http://127.0.0.1:<port>` while production keeps the SSRF guard enabled.
 //!
 //! The hermetic tests drive the transition and restore paths and pin the
 //! chains required by ilm-7 and the restore follow-up:
@@ -46,7 +46,7 @@
 //!      retry serves the object locally until expiry, and expiry leaves the
 //!      remote object available for a second restore.
 
-use crate::common::{RustFSTestEnvironment, local_http_client};
+use crate::common::RustFSTestEnvironment;
 use aws_sdk_s3::Client;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
@@ -56,10 +56,6 @@ use aws_sdk_s3::types::{
     VersioningConfiguration,
 };
 use http::Method;
-use http::header::HOST;
-use rustfs_signer::constants::UNSIGNED_PAYLOAD;
-use rustfs_signer::sign_v4;
-use s3s::Body;
 use serde::Deserialize;
 use std::time::{Duration as StdDuration, Instant};
 use time::{OffsetDateTime, format_description::well_known::Rfc3339};
@@ -100,6 +96,7 @@ const MANUAL_ACTIVE_CANCEL_OBJECTS: usize = 512;
 const MANUAL_RESTART_CANCEL_OBJECTS: usize = 512;
 const MANUAL_ACTIVE_CANCEL_RUNNING_TIMEOUT: StdDuration = StdDuration::from_secs(15);
 const MANUAL_TRANSITION_CANCEL_BARRIER_ENV: &str = "RUSTFS_E2E_MANUAL_TRANSITION_CANCEL_BARRIER";
+const ALLOW_LOOPBACK_TIER_ENDPOINT_ENV: (&str, &str) = ("RUSTFS_TIER_RUSTFS_ALLOW_LOOPBACK_ENDPOINT", "true");
 const MANUAL_ASYNC_CONFLICT_TERMINAL_TIMEOUT: StdDuration = StdDuration::from_secs(90);
 const MANUAL_RESTART_RECOVERY_TIMEOUT: StdDuration = StdDuration::from_secs(80);
 const OBJECT_KEY: &str = "tier/鲁A12345/report.bin";
@@ -116,6 +113,20 @@ const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-reques
 const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime";
 const TIER_MUTATION_RECOVERY_CHANGED: &str = "Remote tier mutation recovery changed before publish";
 
+async fn start_tier_source(hot: &mut RustFSTestEnvironment, extra_env: &[(&str, &str)]) -> TestResult {
+    let mut env = Vec::with_capacity(extra_env.len() + 1);
+    env.push(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV);
+    env.extend_from_slice(extra_env);
+    hot.start_rustfs_server_with_env(vec![], &env).await
+}
+
+async fn restart_tier_source(hot: &mut RustFSTestEnvironment, extra_env: &[(&str, &str)]) -> TestResult {
+    let mut env = Vec::with_capacity(extra_env.len() + 1);
+    env.push(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV);
+    env.extend_from_slice(extra_env);
+    hot.restart_server_preserving_data(vec![], &env).await
+}
+
 /// 5 MiB — the S3 minimum size for a non-final multipart part; the object's only
 /// internal part boundary sits at this offset.
 const PART0_SIZE: usize = 5 * 1024 * 1024;
@@ -131,9 +142,8 @@ fn payload() -> Vec<u8> {
 
 /// Sign and send an admin request in-process (no `awscurl`).
 ///
-/// Mirrors the shared admin-API e2e pattern: the SigV4 signature is computed
+/// Thin wrapper over [`crate::common::admin_request`], kept local so the call
-/// over `UNSIGNED_PAYLOAD`, so the JSON body rides on the wire without being
+/// sites below keep their `Option<&str>` body shape.
-/// pre-hashed. Returns the response status and body text.
 async fn signed_admin_request(
     base_url: &str,
     method: Method,
@@ -142,30 +152,7 @@ async fn signed_admin_request(
     access_key: &str,
     secret_key: &str,
 ) -> Result<(reqwest::StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
-    let url = format!("{base_url}{path}");
+    crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
-    let body_bytes = body.map(|b| b.as_bytes().to_vec()).unwrap_or_default();
-
-    let request = http::Request::builder()
-        .method(method.clone())
-        .uri(uri)
-        .header(HOST, authority)
-        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
-
-    let client = local_http_client();
-    let mut request_builder = client.request(method, url.as_str());
-    for (name, value) in signed.headers() {
-        request_builder = request_builder.header(name, value);
-    }
-    if !body_bytes.is_empty() {
-        request_builder = request_builder.body(body_bytes);
-    }
-    let response = request_builder.send().await?;
-    let status = response.status();
-    let text = response.text().await?;
-    Ok((status, text))
 }
 
 /// Wire `hot` -> `cold` as a `TierType::RustFS` remote tier via `AddTier`.
@@ -223,19 +210,27 @@ async fn add_rustfs_tier(hot: &RustFSTestEnvironment, cold: &RustFSTestEnvironme
     }
 }
 
-async fn remove_rustfs_tier_force(hot: &RustFSTestEnvironment) -> TestResult {
+fn clear_tiers_confirmation_token(now: OffsetDateTime) -> String {
-    let path = format!("/rustfs/admin/v3/tier/{TIER_NAME}?force=true");
+    let mut rand = "AGD1R25GI3I1GJGUGJFD7FBS4DFAASDF".to_string();
+    rand.insert_str(3, &now.day().to_string());
+    rand.insert_str(17, &now.month().to_string());
+    rand.insert_str(23, &now.year().to_string());
+    rand
+}
+
+async fn clear_rustfs_tiers_force(hot: &RustFSTestEnvironment) -> TestResult {
     let deadline = Instant::now() + StdDuration::from_secs(30);
     loop {
-        let (status, resp) =
+        let rand = clear_tiers_confirmation_token(OffsetDateTime::now_utc());
-            signed_admin_request(&hot.url, Method::DELETE, &path, None, &hot.access_key, &hot.secret_key).await?;
+        let path = format!("/rustfs/admin/v3/tier/clear?rand={rand}&force=true");
+        let (status, resp) = signed_admin_request(&hot.url, Method::POST, &path, None, &hot.access_key, &hot.secret_key).await?;
         if status.is_success() {
             return Ok(());
         }
         if (!resp.contains("TierNameBackendInUse") && !resp.contains(TIER_MUTATION_RECOVERY_CHANGED))
             || Instant::now() >= deadline
         {
-            return Err(format!("RemoveTier(RustFS) failed: status={status}, body={resp}").into());
+            return Err(format!("ClearTier(RustFS) failed: status={status}, body={resp}").into());
         }
         // Tier mutation cleanup and startup recovery are asynchronous.
         tokio::time::sleep(StdDuration::from_millis(100)).await;
@@ -888,8 +883,7 @@ async fn test_hermetic_transition_main_path() -> TestResult {
     // Hot/source server. A 1s scanner cycle is a backstop; transition is
     // primarily driven immediately by the multipart completion path.
     let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_CYCLE", "1")])
+    start_tier_source(&mut hot, &[("RUSTFS_SCANNER_CYCLE", "1")]).await?;
-        .await?;
     let hot_client = hot.create_s3_client();
 
     // Wire the RustFS remote tier (real connectivity probe, no force).
@@ -987,8 +981,7 @@ async fn test_hermetic_transition_restore_failure_expiry_and_retry() -> TestResu
     cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_ILM_DEBUG_DAY_SECS", "5")])
+    start_tier_source(&mut hot, &[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_ILM_DEBUG_DAY_SECS", "5")]).await?;
-        .await?;
     let hot_client = hot.create_s3_client();
     add_rustfs_tier(&hot, &cold).await?;
 
@@ -1116,8 +1109,7 @@ async fn test_manual_transition_run_black_box_semantics() -> TestResult {
     cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
+    start_tier_source(&mut hot, &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]).await?;
-        .await?;
     let hot_client = hot.create_s3_client();
     add_rustfs_tier(&hot, &cold).await?;
     let due_mtime = OffsetDateTime::now_utc() - time::Duration::hours(25);
@@ -1222,8 +1214,7 @@ async fn test_manual_transition_async_job_status_polling() -> TestResult {
     cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
+    start_tier_source(&mut hot, &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]).await?;
-        .await?;
     let hot_client = hot.create_s3_client();
     add_rustfs_tier(&hot, &cold).await?;
 
@@ -1321,8 +1312,7 @@ async fn test_manual_transition_async_limit_reports_terminal_partial() -> TestRe
     cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
+    start_tier_source(&mut hot, &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]).await?;
-        .await?;
     let hot_client = hot.create_s3_client();
     add_rustfs_tier(&hot, &cold).await?;
 
@@ -1483,8 +1473,8 @@ async fn test_manual_transition_async_scope_conflicts_report_active_job() -> Tes
     cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(
+    start_tier_source(
-        vec![],
+        &mut hot,
         &[
             ("RUSTFS_SCANNER_ENABLED", "false"),
             ("RUSTFS_SCANNER_CYCLE", "3600"),
@@ -1593,8 +1583,7 @@ async fn test_manual_transition_async_different_buckets_admit_concurrently() ->
     cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
+    start_tier_source(&mut hot, &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]).await?;
-        .await?;
     let hot_client = hot.create_s3_client();
     add_rustfs_tier(&hot, &cold).await?;
 
@@ -1714,8 +1703,7 @@ async fn test_manual_transition_async_tier_failure_reports_terminal_partial() ->
     cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
+    start_tier_source(&mut hot, &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]).await?;
-        .await?;
     let hot_client = hot.create_s3_client();
     add_rustfs_tier(&hot, &cold).await?;
 
@@ -1728,7 +1716,7 @@ async fn test_manual_transition_async_tier_failure_reports_terminal_partial() ->
         0,
     )
     .await?;
-    remove_rustfs_tier_force(&hot).await?;
+    clear_rustfs_tiers_force(&hot).await?;
 
     let due_mtime = OffsetDateTime::now_utc() - time::Duration::hours(25);
     put_backdated_single_part_object(
@@ -1807,8 +1795,7 @@ async fn test_manual_transition_async_worker_failure_reports_terminal_partial()
     cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
+    start_tier_source(&mut hot, &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]).await?;
-        .await?;
     let hot_client = hot.create_s3_client();
     add_rustfs_tier(&hot, &cold).await?;
     cold.stop_server();
@@ -1900,8 +1887,8 @@ async fn test_manual_transition_async_active_cancel_reports_terminal_cancelled()
     cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(
+    start_tier_source(
-        vec![],
+        &mut hot,
         &[
             ("RUSTFS_SCANNER_ENABLED", "false"),
             ("RUSTFS_SCANNER_CYCLE", "3600"),
@@ -2006,7 +1993,7 @@ async fn test_manual_transition_async_cancel_after_process_restart_recovers_term
         ("RUSTFS_TRANSITION_QUEUE_CAPACITY", "512"),
     ];
     let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(vec![], &restart_env).await?;
+    start_tier_source(&mut hot, &restart_env).await?;
     let hot_client = hot.create_s3_client();
     add_rustfs_tier(&hot, &cold).await?;
 
@@ -2034,7 +2021,7 @@ async fn test_manual_transition_async_cancel_after_process_restart_recovers_term
         .ok_or("async response must include status_endpoint")?;
     assert_eq!(accepted.cancel_endpoint.as_deref(), Some(status_endpoint));
 
-    hot.restart_server_preserving_data(vec![], &restart_env).await?;
+    restart_tier_source(&mut hot, &restart_env).await?;
 
     let restarted = manual_transition_job_status(&hot, status_endpoint).await?;
     assert_eq!(restarted.job_id, job_id);
@@ -2158,8 +2145,7 @@ async fn test_manual_transition_run_contract_no_status_cancel_fields() -> TestRe
     cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
+    start_tier_source(&mut hot, &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]).await?;
-        .await?;
     let hot_client = hot.create_s3_client();
     add_rustfs_tier(&hot, &cold).await?;
 
@@ -2197,8 +2183,7 @@ async fn test_manual_transition_run_continuation_token_resumes_without_raw_marke
     cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
+    start_tier_source(&mut hot, &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]).await?;
-        .await?;
     let hot_client = hot.create_s3_client();
     add_rustfs_tier(&hot, &cold).await?;
 
@@ -2238,8 +2223,7 @@ async fn test_manual_transition_run_continuation_token_resumes_without_raw_marke
         "continuation token must not expose the raw object prefix: {continuation}"
     );
 
-    hot.restart_server_preserving_data(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
+    restart_tier_source(&mut hot, &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]).await?;
-        .await?;
 
     let second = manual_transition_run_with_max_and_continuation(
         &hot,
@@ -2272,8 +2256,8 @@ async fn test_manual_transition_run_queue_pressure_partial() -> TestResult {
     cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(
+    start_tier_source(
-        vec![],
+        &mut hot,
         &[
             ("RUSTFS_SCANNER_ENABLED", "false"),
             ("RUSTFS_SCANNER_CYCLE", "3600"),

crates/e2e_test/src/replication_extension_test.rs

@@ -13,8 +13,9 @@
 // limitations under the License.
 
 use crate::common::{
-    RustFSTestEnvironment, admin_create_user, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
+    AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user,
-    replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client, signed_request_with_session_token,
+    awscurl_post_sts_form_urlencoded, init_logging, local_http_client, replication_fast_env, rustfs_binary_path, signed_request,
+    signed_request_with_client, signed_request_with_session_token,
 };
 use crate::fake_s3_target::{
     FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation,
@@ -25,7 +26,7 @@ use crate::kms::common::{
     sse_customer_key_md5_base64,
 };
 use crate::storage_api::replication_extension::BucketTargetSys;
-use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::Client;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
 use aws_sdk_s3::primitives::ByteStream;
@@ -33,7 +34,6 @@ use aws_sdk_s3::types::{
     BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
     VersioningConfiguration,
 };
-use aws_sdk_s3::{Client, Config};
 use base64_simd::STANDARD as BASE64_STANDARD;
 use bytes::Bytes;
 use flate2::read::GzDecoder;
@@ -895,15 +895,7 @@ async fn wait_for_replicated_object_over_https(
 }
 
 fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
-    let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-site-replication");
+    env.create_s3_client_with_credentials(access_key, secret_key)
-    let config = Config::builder()
-        .credentials_provider(credentials)
-        .region(Region::new("us-east-1"))
-        .endpoint_url(&env.url)
-        .force_path_style(true)
-        .behavior_version_latest()
-        .build();
-    Client::from_conf(config)
 }
 
 async fn admin_add_canned_policy(
@@ -911,24 +903,15 @@ async fn admin_add_canned_policy(
     policy_name: &str,
     policy: &serde_json::Value,
 ) -> Result<(), Box<dyn Error + Send + Sync>> {
-    let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
+    admin_add_canned_policy_via(
-    let response = signed_request(
+        AdminTransport::Signed,
-        http::Method::PUT,
+        &env.url,
-        &url,
         &env.access_key,
         &env.secret_key,
-        Some(policy.to_string().into_bytes()),
+        policy_name,
-        Some("application/json"),
+        &policy.to_string(),
     )
-    .await?;
+    .await
-
-    if response.status() != StatusCode::OK {
-        let status = response.status();
-        let body = response.text().await.unwrap_or_default();
-        return Err(format!("add canned policy failed: {status} {body}").into());
-    }
-
-    Ok(())
 }
 
 async fn admin_attach_policy_to_user(
@@ -936,19 +919,7 @@ async fn admin_attach_policy_to_user(
     policy_name: &str,
     username: &str,
 ) -> Result<(), Box<dyn Error + Send + Sync>> {
-    let url = format!(
+    admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, policy_name, username).await
-        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
-        env.url, policy_name, username
-    );
-    let response = signed_request(http::Method::PUT, &url, &env.access_key, &env.secret_key, Some(Vec::new()), None).await?;
-
-    if response.status() != StatusCode::OK {
-        let status = response.status();
-        let body = response.text().await.unwrap_or_default();
-        return Err(format!("attach policy to user failed: {status} {body}").into());
-    }
-
-    Ok(())
 }
 
 async fn admin_update_group_members(
@@ -1941,6 +1912,21 @@ async fn site_replication_info(env: &RustFSTestEnvironment) -> Result<SiteReplic
     Ok(serde_json::from_slice(&response.bytes().await?)?)
 }
 
+async fn site_replication_rotate_svc_acct(
+    env: &RustFSTestEnvironment,
+) -> Result<ReplicateEditStatus, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/site-replication/rotate-svc-acct", env.url);
+    let response = signed_request(http::Method::POST, &url, &env.access_key, &env.secret_key, None, None).await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication rotate-svc-acct failed: {status} {body}").into());
+    }
+
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
 async fn site_replication_resync_op(
     env: &RustFSTestEnvironment,
     operation: &str,
@@ -6339,6 +6325,112 @@ async fn test_site_replication_remove_all_real_dual_node() -> Result<(), Box<dyn
     Ok(())
 }
 
+#[tokio::test]
+async fn test_site_replication_rotate_svc_acct_completes_and_replication_survives_real_dual_node()
+-> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env
+        .start_rustfs_server_with_env(vec![], LOOPBACK_REPLICATION_TARGET_ENV)
+        .await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env
+        .start_rustfs_server_without_cleanup_with_env(LOOPBACK_REPLICATION_TARGET_ENV)
+        .await?;
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+    let bucket = "site-repl-rotate-svc-acct";
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+                ..Default::default()
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+                ..Default::default()
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {add_status:?}");
+
+    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+
+    source_client.create_bucket().bucket(bucket).send().await?;
+    enable_bucket_versioning(&source_env, bucket).await?;
+    wait_for_bucket_on_target(&target_client, bucket).await?;
+    let baseline_payload = b"before rotation".to_vec();
+    source_client
+        .put_object()
+        .bucket(bucket)
+        .key("before-rotate.txt")
+        .body(ByteStream::from(baseline_payload.clone()))
+        .send()
+        .await?;
+    let replicated_baseline = wait_for_object_on_target(&target_client, bucket, "before-rotate.txt").await?;
+    assert_eq!(replicated_baseline, baseline_payload);
+
+    // A single rotation call must finish the whole hand-over. Before the fix
+    // the join push could only sign with the freshly installed secret, every
+    // peer rejected it, the rotation stayed pending forever, and both
+    // replication directions were dead until an operator retried.
+    let rotate_status = site_replication_rotate_svc_acct(&source_env).await?;
+    assert!(rotate_status.success, "rotation did not complete in one call: {rotate_status:?}");
+
+    for env in [&source_env, &target_env] {
+        let deadline = std::time::Instant::now() + Duration::from_secs(30);
+        loop {
+            let info = site_replication_info(env).await?;
+            if info.enabled && info.pending_operation.is_none() {
+                break;
+            }
+            if std::time::Instant::now() > deadline {
+                return Err(format!("rotation left {} with a pending operation: {:?}", env.url, info.pending_operation).into());
+            }
+            tokio::time::sleep(Duration::from_millis(500)).await;
+        }
+    }
+
+    // Replication must actually flow again in both directions with the
+    // rotated service-account secret.
+    let forward_payload = b"after rotation from source".to_vec();
+    source_client
+        .put_object()
+        .bucket(bucket)
+        .key("after-rotate-forward.txt")
+        .body(ByteStream::from(forward_payload.clone()))
+        .send()
+        .await?;
+    let replicated_forward = wait_for_object_on_target(&target_client, bucket, "after-rotate-forward.txt").await?;
+    assert_eq!(replicated_forward, forward_payload);
+
+    let reverse_payload = b"after rotation from target".to_vec();
+    target_client
+        .put_object()
+        .bucket(bucket)
+        .key("after-rotate-reverse.txt")
+        .body(ByteStream::from(reverse_payload.clone()))
+        .send()
+        .await?;
+    let replicated_reverse = wait_for_object_on_target(&source_client, bucket, "after-rotate-reverse.txt").await?;
+    assert_eq!(replicated_reverse, reverse_payload);
+
+    Ok(())
+}
+
 #[tokio::test]
 async fn test_site_replication_state_edit_fresh_and_stale_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
     init_logging();

crates/e2e_test/src/select_request_root_alias_test.rs

@@ -0,0 +1,84 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Raw HTTP regression coverage for the Select request root alias (backlog#1626).
+
+use crate::common::{RustFSTestEnvironment, signed_s3_request};
+use aws_sdk_s3::primitives::ByteStream;
+use http::Method;
+use std::error::Error;
+use uuid::Uuid;
+
+type TestResult<T = ()> = Result<T, Box<dyn Error + Send + Sync>>;
+
+const CSV_BODY: &[u8] = b"name\nGatewayJ-root-alias\nignored\n";
+const EXPECTED_RECORD: &[u8] = b"GatewayJ-root-alias";
+
+fn select_request(root: &str) -> String {
+    format!(
+        r#"<{root} xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+<Expression>SELECT s.name FROM S3Object s WHERE s.name = 'GatewayJ-root-alias'</Expression>
+<ExpressionType>SQL</ExpressionType>
+<InputSerialization><CSV><FileHeaderInfo>USE</FileHeaderInfo></CSV></InputSerialization>
+<OutputSerialization><CSV/></OutputSerialization>
+</{root}>"#
+    )
+}
+
+async fn raw_select(env: &RustFSTestEnvironment, bucket: &str, object: &str, root: &str) -> TestResult {
+    let response = signed_s3_request(
+        Method::POST,
+        &format!("{}/{bucket}/{object}?select&select-type=2", env.url),
+        Some(select_request(root)),
+        Some("application/xml"),
+        &env.access_key,
+        &env.secret_key,
+    )
+    .await?;
+    let status = response.status();
+    let body = response.bytes().await?.to_vec();
+    assert_eq!(
+        status,
+        reqwest::StatusCode::OK,
+        "{root} root was rejected: {}",
+        String::from_utf8_lossy(&body)
+    );
+    assert!(
+        body.windows(EXPECTED_RECORD.len()).any(|window| window == EXPECTED_RECORD),
+        "{root} root did not return the projected record"
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn select_request_root_alias_reaches_select_endpoint() -> TestResult {
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(Vec::new()).await?;
+    let client = env.create_s3_client();
+    let bucket = format!("select-root-{}", Uuid::new_v4().simple());
+    let object = "input.csv";
+
+    client.create_bucket().bucket(&bucket).send().await?;
+    client
+        .put_object()
+        .bucket(&bucket)
+        .key(object)
+        .body(ByteStream::from_static(CSV_BODY))
+        .send()
+        .await?;
+
+    raw_select(&env, &bucket, object, "SelectObjectContentRequest").await?;
+    raw_select(&env, &bucket, object, "SelectRequest").await?;
+    Ok(())
+}

crates/e2e_test/src/snowball_auto_extract_test.rs

@@ -17,8 +17,56 @@ mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
     use aws_sdk_s3::error::ProvideErrorMetadata;
     use aws_sdk_s3::primitives::ByteStream;
+    use flate2::{Compression, write::GzEncoder};
     use std::error::Error;
-    use std::io::Cursor;
+    use std::io::{Cursor, Write};
+
+    fn pax_record(key: &str, value: &str) -> Vec<u8> {
+        let payload = format!("{key}={value}\n");
+        let mut len = payload.len() + 3;
+        loop {
+            let record = format!("{len} {payload}");
+            if record.len() == len {
+                return record.into_bytes();
+            }
+            len = record.len();
+        }
+    }
+
+    async fn append_pax_header(
+        builder: &mut tokio_tar::Builder<Cursor<Vec<u8>>>,
+        entry_type: tokio_tar::EntryType,
+        records: &[(&str, &str)],
+    ) -> Result<(), Box<dyn Error + Send + Sync>> {
+        let mut payload = Vec::new();
+        for (key, value) in records {
+            payload.extend(pax_record(key, value));
+        }
+        let mut header = tokio_tar::Header::new_ustar();
+        header.set_entry_type(entry_type);
+        header.set_size(u64::try_from(payload.len()).expect("PAX payload length should fit in u64"));
+        header.set_mode(0o644);
+        header.set_cksum();
+        builder
+            .append_data(&mut header, "PaxHeaders.X/snowball", Cursor::new(payload))
+            .await?;
+        Ok(())
+    }
+
+    async fn append_typed_entry(
+        builder: &mut tokio_tar::Builder<Cursor<Vec<u8>>>,
+        path: &str,
+        entry_type: tokio_tar::EntryType,
+        body: &[u8],
+    ) -> Result<(), Box<dyn Error + Send + Sync>> {
+        let mut header = tokio_tar::Header::new_gnu();
+        header.set_entry_type(entry_type);
+        header.set_size(u64::try_from(body.len()).expect("TAR member length should fit in u64"));
+        header.set_mode(0o644);
+        header.set_cksum();
+        builder.append_data(&mut header, path, Cursor::new(body)).await?;
+        Ok(())
+    }
 
     async fn build_test_archive() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
         let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
@@ -69,12 +117,50 @@ mod tests {
         Ok(builder.into_inner().await?.into_inner())
     }
 
-    fn build_archive_with_parent_dir_entry(victim_bucket: &str) -> Vec<u8> {
+    async fn build_archive_with_invalid_checksum() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
-        let path = format!("../{victim_bucket}/evil-injected.txt");
+        let mut archive = build_test_archive().await?;
-        let data = b"injected-body";
+        archive[0] ^= 1;
+        Ok(archive)
+    }
+
+    async fn build_archive_with_negative_gnu_mtime() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+        let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
+        let mut header = tokio_tar::Header::new_gnu();
+        header.set_size(b"negative-mtime-body".len() as u64);
+        header.set_mode(0o644);
+        header.as_old_mut().mtime.fill(0xff);
+        builder
+            .append_data(&mut header, "negative-mtime.txt", Cursor::new(b"negative-mtime-body".as_slice()))
+            .await?;
+        Ok(builder.into_inner().await?.into_inner())
+    }
+
+    fn gzip_member(payload: &[u8]) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+        let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
+        encoder.write_all(payload)?;
+        Ok(encoder.finish()?)
+    }
+
+    async fn build_concatenated_gzip_archive() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+        let archive = build_test_archive().await?;
+        let split_at = archive.len() / 2;
+        let mut encoded = gzip_member(&archive[..split_at])?;
+        encoded.extend(gzip_member(&archive[split_at..])?);
+        Ok(encoded)
+    }
+
+    async fn build_gzip_archive_with_invalid_crc() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+        let mut encoded = gzip_member(&build_test_archive().await?)?;
+        let crc_offset = encoded.len().checked_sub(8).expect("gzip fixture must contain a trailer");
+        encoded[crc_offset] ^= 1;
+        Ok(encoded)
+    }
+
+    fn append_raw_tar_entry_with_type(archive: &mut Vec<u8>, path: &[u8], data: &[u8], entry_type: u8) {
+        assert!(path.len() <= 100, "raw TAR fixture path must fit in the name field");
         let mut header = [0u8; 512];
 
-        header[..path.len()].copy_from_slice(path.as_bytes());
+        header[..path.len()].copy_from_slice(path);
         header[100..108].copy_from_slice(b"0000644\0");
         header[108..116].copy_from_slice(b"0000000\0");
         header[116..124].copy_from_slice(b"0000000\0");
@@ -82,7 +168,7 @@ mod tests {
         header[124..136].copy_from_slice(size.as_bytes());
         header[136..148].copy_from_slice(b"00000000000\0");
         header[148..156].fill(b' ');
-        header[156] = b'0';
+        header[156] = entry_type;
         header[257..263].copy_from_slice(b"ustar\0");
         header[263..265].copy_from_slice(b"00");
 
@@ -90,11 +176,87 @@ mod tests {
         let checksum = format!("{:06o}\0 ", checksum);
         header[148..156].copy_from_slice(checksum.as_bytes());
 
-        let mut archive = Vec::new();
         archive.extend_from_slice(&header);
         archive.extend_from_slice(data);
         let padding = (512 - (data.len() % 512)) % 512;
         archive.extend(std::iter::repeat_n(0, padding));
+    }
+
+    fn append_raw_tar_entry(archive: &mut Vec<u8>, path: &[u8], data: &[u8]) {
+        append_raw_tar_entry_with_type(archive, path, data, b'0');
+    }
+
+    fn build_archive_with_parent_dir_entry(victim_bucket: &str) -> Vec<u8> {
+        let path = format!("../{victim_bucket}/evil-injected.txt");
+        let mut archive = Vec::new();
+        append_raw_tar_entry(&mut archive, path.as_bytes(), b"injected-body");
+        archive.extend_from_slice(&[0u8; 1024]);
+        archive
+    }
+
+    async fn build_member_semantics_archive() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+        let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
+        append_pax_header(
+            &mut builder,
+            tokio_tar::EntryType::XGlobalHeader,
+            &[
+                ("minio.metadata.x-amz-meta-owner", "global"),
+                ("minio.metadata.x-amz-meta-snowball-auto-extract", "true"),
+            ],
+        )
+        .await?;
+        append_pax_header(
+            &mut builder,
+            tokio_tar::EntryType::XHeader,
+            &[("minio.metadata.x-amz-meta-owner", "local")],
+        )
+        .await?;
+        append_typed_entry(&mut builder, "regular.txt", tokio_tar::EntryType::Regular, b"regular-body").await?;
+        for (path, entry_type) in [
+            ("char", tokio_tar::EntryType::Char),
+            ("block", tokio_tar::EntryType::Block),
+            ("fifo", tokio_tar::EntryType::Fifo),
+        ] {
+            append_typed_entry(&mut builder, path, entry_type, b"").await?;
+        }
+        let mut directory = tokio_tar::Header::new_gnu();
+        directory.set_entry_type(tokio_tar::EntryType::Directory);
+        directory.set_size(0);
+        directory.set_mode(0o755);
+        directory.set_cksum();
+        builder
+            .append_data(&mut directory, "directory/", Cursor::new(Vec::new()))
+            .await?;
+        for (path, entry_type) in [
+            ("hard-link", tokio_tar::EntryType::Link),
+            ("symlink", tokio_tar::EntryType::Symlink),
+            ("continuous", tokio_tar::EntryType::Continuous),
+            ("unknown", tokio_tar::EntryType::Other(b'9')),
+        ] {
+            append_typed_entry(&mut builder, path, entry_type, b"").await?;
+        }
+        Ok(builder.into_inner().await?.into_inner())
+    }
+
+    async fn build_versioned_member_archive(path: &str, version_id: &str) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+        let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
+        append_pax_header(&mut builder, tokio_tar::EntryType::XHeader, &[("minio.versionId", version_id)]).await?;
+        append_typed_entry(&mut builder, path, tokio_tar::EntryType::Regular, b"versioned-body").await?;
+        Ok(builder.into_inner().await?.into_inner())
+    }
+
+    fn build_archive_with_invalid_utf8_entry() -> Vec<u8> {
+        let mut archive = Vec::new();
+        append_raw_tar_entry(&mut archive, b"invalid-\xff.txt", b"ignored-body");
+        append_raw_tar_entry(&mut archive, b"valid.txt", b"valid-body");
+        archive.extend_from_slice(&[0u8; 1024]);
+        archive
+    }
+
+    fn build_archive_with_invalid_utf8_symlink() -> Vec<u8> {
+        let mut archive = Vec::new();
+        append_raw_tar_entry_with_type(&mut archive, b"invalid-\xff-link", b"", b'2');
+        append_raw_tar_entry(&mut archive, b"valid.txt", b"valid-body");
         archive.extend_from_slice(&[0u8; 1024]);
         archive
     }
@@ -135,6 +297,147 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test]
+    async fn snowball_auto_extract_applies_member_semantics_and_metadata_precedence() -> Result<(), Box<dyn Error + Send + Sync>>
+    {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "snowball-member-semantics";
+        client.create_bucket().bucket(bucket).send().await?;
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("fixture.tar")
+            .metadata("Snowball-Auto-Extract", "true")
+            .metadata("Minio-Snowball-Prefix", "members")
+            .metadata("owner", "outer")
+            .body(ByteStream::from(build_member_semantics_archive().await?))
+            .send()
+            .await?;
+
+        let regular = client.head_object().bucket(bucket).key("members/regular.txt").send().await?;
+        let regular_metadata = regular.metadata().expect("regular member should expose metadata");
+        assert_eq!(regular_metadata.get("owner").map(String::as_str), Some("local"));
+        assert!(!regular_metadata.contains_key("snowball-auto-extract"));
+        assert!(!regular_metadata.contains_key("minio-snowball-prefix"));
+
+        for key in ["char", "block", "fifo"] {
+            let head = client
+                .head_object()
+                .bucket(bucket)
+                .key(format!("members/{key}"))
+                .send()
+                .await?;
+            assert_eq!(head.content_length(), Some(0), "{key} should be materialized as an empty object");
+            assert_eq!(
+                head.metadata().and_then(|metadata| metadata.get("owner")).map(String::as_str),
+                Some("outer"),
+                "{key} should not inherit global PAX metadata"
+            );
+        }
+        let directory = client.head_object().bucket(bucket).key("members/directory/").send().await?;
+        assert_eq!(directory.content_length(), Some(0));
+
+        for key in ["hard-link", "symlink", "continuous", "unknown"] {
+            let error = client
+                .head_object()
+                .bucket(bucket)
+                .key(format!("members/{key}"))
+                .send()
+                .await
+                .expect_err("unsupported TAR entry type must be skipped");
+            assert_eq!(error.into_service_error().code(), Some("NotFound"), "{key}");
+        }
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn snowball_auto_extract_validates_pax_version_id_against_bucket_state() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "snowball-version-semantics";
+        client.create_bucket().bucket(bucket).send().await?;
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("null.tar")
+            .metadata("Snowball-Auto-Extract", "true")
+            .body(ByteStream::from(build_versioned_member_archive("null.txt", "null").await?))
+            .send()
+            .await?;
+        let null_member = client.get_object().bucket(bucket).key("null.txt").send().await?;
+        assert_eq!(null_member.body.collect().await?.into_bytes().as_ref(), b"versioned-body");
+
+        for (archive_key, member_key, version_id) in [
+            ("uuid.tar", "uuid.txt", uuid::Uuid::new_v4().to_string()),
+            ("uppercase-null.tar", "uppercase-null.txt", "NULL".to_string()),
+        ] {
+            let error = client
+                .put_object()
+                .bucket(bucket)
+                .key(archive_key)
+                .metadata("Snowball-Auto-Extract", "true")
+                .body(ByteStream::from(build_versioned_member_archive(member_key, &version_id).await?))
+                .send()
+                .await
+                .expect_err("invalid or unversioned UUID import must be rejected");
+            assert_eq!(error.into_service_error().code(), Some("InvalidArgument"), "{archive_key}");
+            let missing = client
+                .head_object()
+                .bucket(bucket)
+                .key(member_key)
+                .send()
+                .await
+                .expect_err("rejected version import must not create an object");
+            assert_eq!(missing.into_service_error().code(), Some("NotFound"), "{member_key}");
+        }
+
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                aws_sdk_s3::types::VersioningConfiguration::builder()
+                    .status(aws_sdk_s3::types::BucketVersioningStatus::Enabled)
+                    .build(),
+            )
+            .send()
+            .await?;
+        let imported_version_id = uuid::Uuid::new_v4().to_string();
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("versioned-uuid.tar")
+            .metadata("Snowball-Auto-Extract", "true")
+            .body(ByteStream::from(
+                build_versioned_member_archive("versioned-uuid.txt", &imported_version_id).await?,
+            ))
+            .send()
+            .await?;
+        let imported = client
+            .get_object()
+            .bucket(bucket)
+            .key("versioned-uuid.txt")
+            .version_id(&imported_version_id)
+            .send()
+            .await?;
+        assert_eq!(imported.version_id(), Some(imported_version_id.as_str()));
+        assert_eq!(imported.body.collect().await?.into_bytes().as_ref(), b"versioned-body");
+
+        env.stop_server();
+        Ok(())
+    }
+
     #[tokio::test]
     async fn snowball_auto_extract_supports_standard_headers_with_combined_extract_options()
     -> Result<(), Box<dyn Error + Send + Sync>> {
@@ -263,6 +566,113 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test]
+    async fn snowball_auto_extract_accepts_negative_gnu_mtime() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "snowball-negative-mtime";
+        client.create_bucket().bucket(bucket).send().await?;
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("fixture.tar")
+            .metadata("Snowball-Auto-Extract", "true")
+            .body(ByteStream::from(build_archive_with_negative_gnu_mtime().await?))
+            .send()
+            .await?;
+
+        let object = client.get_object().bucket(bucket).key("negative-mtime.txt").send().await?;
+        assert_eq!(object.body.collect().await?.into_bytes().as_ref(), b"negative-mtime-body");
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn snowball_auto_extract_consumes_concatenated_gzip_members() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "snowball-concatenated-gzip";
+        client.create_bucket().bucket(bucket).send().await?;
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("fixture.tar.gz")
+            .metadata("Snowball-Auto-Extract", "true")
+            .body(ByteStream::from(build_concatenated_gzip_archive().await?))
+            .send()
+            .await?;
+
+        let object = client.get_object().bucket(bucket).key("root.txt").send().await?;
+        assert_eq!(object.body.collect().await?.into_bytes().as_ref(), b"root payload\n");
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn snowball_auto_extract_rejects_gzip_crc_error_when_ignore_errors_enabled() -> Result<(), Box<dyn Error + Send + Sync>>
+    {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "snowball-gzip-crc-ignore-errors";
+        client.create_bucket().bucket(bucket).send().await?;
+        let err = client
+            .put_object()
+            .bucket(bucket)
+            .key("fixture.tar.gz")
+            .metadata("Snowball-Auto-Extract", "true")
+            .metadata("Minio-Snowball-Ignore-Errors", "true")
+            .body(ByteStream::from(build_gzip_archive_with_invalid_crc().await?))
+            .send()
+            .await
+            .expect_err("gzip integrity failures must remain fatal under ignore-errors");
+
+        assert_eq!(err.into_service_error().code(), Some("InvalidArgument"));
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn snowball_auto_extract_rejects_mismatched_content_md5() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "snowball-content-md5";
+        client.create_bucket().bucket(bucket).send().await?;
+        let err = client
+            .put_object()
+            .bucket(bucket)
+            .key("fixture.tar")
+            .metadata("Snowball-Auto-Extract", "true")
+            .content_md5("AAAAAAAAAAAAAAAAAAAAAA==")
+            .body(ByteStream::from(build_test_archive().await?))
+            .send()
+            .await
+            .expect_err("mismatched Content-MD5 must fail after the raw body reaches EOF");
+
+        assert_eq!(err.into_service_error().code(), Some("BadDigest"));
+
+        env.stop_server();
+        Ok(())
+    }
+
     #[tokio::test]
     async fn snowball_auto_extract_ignores_invalid_entries_when_requested() -> Result<(), Box<dyn Error + Send + Sync>> {
         init_logging();
@@ -299,7 +709,100 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn snowball_auto_extract_rejects_parent_dir_entry_without_cross_bucket_write()
+    async fn snowball_auto_extract_skips_non_utf8_symlink_without_ignore_errors() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "snowball-invalid-utf8-link";
+        client.create_bucket().bucket(bucket).send().await?;
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("fixture.tar")
+            .metadata("Snowball-Auto-Extract", "true")
+            .body(ByteStream::from(build_archive_with_invalid_utf8_symlink()))
+            .send()
+            .await?;
+
+        let valid = client.get_object().bucket(bucket).key("valid.txt").send().await?;
+        assert_eq!(valid.body.collect().await?.into_bytes().as_ref(), b"valid-body");
+        let listed = client.list_objects_v2().bucket(bucket).send().await?;
+        let keys: Vec<_> = listed.contents().iter().filter_map(|entry| entry.key()).collect();
+        assert_eq!(keys, vec!["valid.txt"]);
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn snowball_auto_extract_skips_non_utf8_member_without_lossy_key_collision() -> Result<(), Box<dyn Error + Send + Sync>>
+    {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "snowball-invalid-utf8";
+        client.create_bucket().bucket(bucket).send().await?;
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("fixture.tar")
+            .metadata("Snowball-Auto-Extract", "true")
+            .metadata("Minio-Snowball-Ignore-Errors", "true")
+            .body(ByteStream::from(build_archive_with_invalid_utf8_entry()))
+            .send()
+            .await?;
+
+        let valid = client.get_object().bucket(bucket).key("valid.txt").send().await?;
+        assert_eq!(valid.body.collect().await?.into_bytes().as_ref(), b"valid-body");
+        let listed = client.list_objects_v2().bucket(bucket).send().await?;
+        let keys: Vec<_> = listed.contents().iter().filter_map(|entry| entry.key()).collect();
+        assert_eq!(keys, vec!["valid.txt"]);
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn snowball_auto_extract_rejects_corrupt_tar_when_ignore_errors_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "snowball-corrupt-ignore-errors";
+        let archive = build_archive_with_invalid_checksum().await?;
+        client.create_bucket().bucket(bucket).send().await?;
+
+        let err = client
+            .put_object()
+            .bucket(bucket)
+            .key("fixture.tar")
+            .metadata("Snowball-Auto-Extract", "true")
+            .metadata("Minio-Snowball-Ignore-Errors", "true")
+            .body(ByteStream::from(archive))
+            .send()
+            .await
+            .expect_err("corrupt TAR structure must remain fatal under ignore-errors");
+        assert_eq!(err.into_service_error().code(), Some("InvalidArgument"));
+
+        let listed = client.list_objects_v2().bucket(bucket).send().await?;
+        assert!(listed.contents().is_empty(), "corrupt archive must not produce objects");
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn snowball_auto_extract_rejects_parent_dir_entry_even_when_ignore_errors_enabled()
     -> Result<(), Box<dyn Error + Send + Sync>> {
         init_logging();
 
@@ -319,6 +822,7 @@ mod tests {
             .bucket(attacker_bucket)
             .key("fixture.tar")
             .metadata("Snowball-Auto-Extract", "true")
+            .metadata("Minio-Snowball-Ignore-Errors", "true")
             .body(ByteStream::from(archive))
             .send()
             .await

crates/ecstore/src/api/mod.rs

@@ -194,9 +194,10 @@ pub mod bucket {
             BucketReplicationResyncStatus, BucketReplicationStat, BucketReplicationStats, BucketStats,
             DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, InQueueMetric,
             MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, OperatorRuleContract,
-            REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
+            REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_READ_ONLY_HISTORICAL_FIELDS,
-            REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
+            REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE,
-            REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
+            REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
+            ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
             ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
             ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
             ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
@@ -405,7 +406,7 @@ pub mod notification {
     pub use crate::services::notification_sys::{
         CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
         acquire_cross_pool_fence_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
-        new_global_notification_sys, start_remote_version_state_fleet_probe,
+        new_global_notification_sys, scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
     };
 }
 
@@ -415,9 +416,10 @@ pub mod object {
         GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, NamespaceLockFence, ObjectEncryptionResolver,
         ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
         RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
-        SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, StreamConsumer, get_object_body_cache_plaintext_len,
+        SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
-        lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
+        ScannerPublicationCommitState, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
-        unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
+        register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
+        unregister_object_mutation_hook,
     };
     pub use crate::store::{
         PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
@@ -459,8 +461,8 @@ pub mod rpc {
         tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
         verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
         verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
-        verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
+        verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
-        verify_tonic_rpc_signature_with_bootstrap,
+        verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
     };
 }
 
@@ -487,9 +489,9 @@ pub mod storage {
     pub use crate::core::pools::HealLifecycleExpiryContext;
     pub use crate::store::HealWalkVersion;
     pub use crate::store::{
-        ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
+        ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, all_local_disk_path,
-        init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
+        find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
-        prewarm_local_disk_id_map_with_instance_ctx,
+        prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
     };
 }
 

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -22,7 +22,9 @@ use crate::bucket::target::{self, BucketTarget, BucketTargets, Credentials};
 use crate::bucket::versioning_sys::BucketVersioningSys;
 use crate::runtime::sources as runtime_sources;
 use aws_credential_types::Credentials as SdkCredentials;
+use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future};
 use aws_sdk_s3::config::Region as SdkRegion;
+use aws_sdk_s3::config::RequestChecksumCalculation;
 use aws_sdk_s3::config::SharedHttpClient;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::error::SdkError;
@@ -38,6 +40,7 @@ use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::Tagging as SdkTagging;
 use aws_sdk_s3::types::{
     ChecksumMode, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
+    ServerSideEncryption,
 };
 use aws_sdk_s3::{Client as S3Client, Config as S3Config, operation::head_object::HeadObjectOutput};
 use aws_sdk_s3::{config::SharedCredentialsProvider, types::BucketVersioningStatus};
@@ -77,7 +80,7 @@ use std::str::FromStr as _;
 use std::sync::Arc;
 use std::sync::OnceLock;
 use std::sync::Weak;
-use std::time::{Duration, Instant};
+use std::time::{Duration, Instant, SystemTime};
 use time::{OffsetDateTime, format_description::well_known::Rfc3339};
 use tokio::sync::Mutex;
 use tokio::sync::RwLock;
@@ -89,6 +92,71 @@ use uuid::Uuid;
 
 const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
 const REDACTED_CREDENTIAL: &str = "<redacted>";
+const EXPIRED_REMOTE_TARGET_CREDENTIALS: &str = "remote target credentials have expired";
+
+#[derive(Clone)]
+struct RemoteTargetCredentialsProvider {
+    credentials: SdkCredentials,
+}
+
+impl RemoteTargetCredentialsProvider {
+    fn resolve_at(&self, now: SystemTime) -> aws_credential_types::provider::Result {
+        if self.credentials.expiry().is_some_and(|expiration| expiration <= now) {
+            return Err(CredentialsError::provider_error(std::io::Error::other(EXPIRED_REMOTE_TARGET_CREDENTIALS)));
+        }
+        Ok(self.credentials.clone())
+    }
+}
+
+impl fmt::Debug for RemoteTargetCredentialsProvider {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("RemoteTargetCredentialsProvider")
+            .field("temporary", &self.credentials.session_token().is_some())
+            .field("expiration", &self.credentials.expiry())
+            .finish()
+    }
+}
+
+impl ProvideCredentials for RemoteTargetCredentialsProvider {
+    fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a>
+    where
+        Self: 'a,
+    {
+        future::ProvideCredentials::ready(self.resolve_at(SystemTime::now()))
+    }
+
+    fn fallback_on_interrupt(&self) -> Option<SdkCredentials> {
+        self.resolve_at(SystemTime::now()).ok()
+    }
+}
+
+fn remote_target_sdk_credentials(
+    credentials: &Credentials,
+    account_id: &str,
+    now: SystemTime,
+) -> Result<SdkCredentials, &'static str> {
+    let session_token = credentials.effective_session_token();
+    let expiration = credentials.effective_expiration().map(SystemTime::from);
+    if expiration.is_some() && session_token.is_none() {
+        return Err("remote target credential expiration requires a session token");
+    }
+    if expiration.is_some_and(|expiration| expiration <= now) {
+        return Err(EXPIRED_REMOTE_TARGET_CREDENTIALS);
+    }
+
+    let mut builder = SdkCredentials::builder()
+        .access_key_id(credentials.access_key.clone())
+        .secret_access_key(credentials.secret_key.clone())
+        .account_id(account_id.to_string())
+        .provider_name("bucket_target_sys");
+    if let Some(session_token) = session_token {
+        builder = builder.session_token(session_token.to_string());
+    }
+    if let Some(expiration) = expiration {
+        builder = builder.expiry(expiration);
+    }
+    Ok(builder.build())
+}
 
 pub type HeadObjectSdkError = Box<SdkError<HeadObjectError>>;
 pub type GetObjectSdkError = Box<SdkError<GetObjectError>>;
@@ -352,6 +420,28 @@ pub struct BucketTargetSys {
     heartbeat_started: OnceLock<()>,
 }
 
+/// Build the bucket-target health-check HTTP client without panicking when
+/// the host has no system CA bundle (issue #6734).
+///
+/// `BucketTargetSys::get()` initializes lazily on the startup path (bucket
+/// metadata install calls it on the main thread), and `reqwest::Client::new()`
+/// panics when the TLS backend cannot load any system trust root — the state
+/// of a minimal container image. Fall back to a client with an explicit empty
+/// trust store: HTTP health checks keep working, and HTTPS targets fail closed
+/// at the TLS handshake with a clear certificate error instead of aborting
+/// the whole process at startup.
+fn build_health_check_client() -> HttpClient {
+    HttpClient::builder().build().unwrap_or_else(|error| {
+        warn!(
+            "bucket target health-check HTTP client could not load system TLS roots ({error}); continuing with an empty trust store — HTTPS target health checks will fail until a CA bundle is installed"
+        );
+        HttpClient::builder()
+            .tls_certs_only(std::iter::empty::<reqwest::Certificate>())
+            .build()
+            .expect("HTTP client construction must succeed with an explicit empty trust store")
+    })
+}
+
 impl BucketTargetSys {
     pub fn get() -> &'static Self {
         GLOBAL_BUCKET_TARGET_SYS.get_or_init(Self::new)
@@ -364,7 +454,7 @@ impl BucketTargetSys {
             targets_map: Arc::new(RwLock::new(HashMap::new())),
             h_mutex: Arc::new(RwLock::new(HashMap::new())),
             target_h_mutex: Arc::new(RwLock::new(HashMap::new())),
-            hc_client: Arc::new(HttpClient::new()),
+            hc_client: Arc::new(build_health_check_client()),
             a_mutex: Arc::new(Mutex::new(HashMap::new())),
             arn_errs_map: Arc::new(RwLock::new(HashMap::new())),
             target_update_mutexes: Arc::new(Mutex::new(HashMap::new())),
@@ -823,13 +913,26 @@ impl BucketTargetSys {
         Ok(BucketTargets { targets: new_targets })
     }
 
+    async fn mark_refresh_attempt(&self, arn: &str) {
+        // Rate-limit a failed config fetch as well as a failed client build.
+        // A successful rebuild replaces this timestamp during publication.
+        self.arn_remotes_map
+            .write()
+            .await
+            .entry(arn.to_string())
+            .or_default()
+            .last_refresh = OffsetDateTime::now_utc();
+    }
+
     pub async fn mark_refresh_in_progress(&self, bucket: &str, arn: &str) {
         let mut arn_errs = self.arn_errs_map.write().await;
-        arn_errs.entry(arn.to_string()).or_insert_with(|| ArnErrs {
+        let err = arn_errs.entry(arn.to_string()).or_insert_with(|| ArnErrs {
-            bucket: bucket.to_string(),
-            update_in_progress: true,
             count: 1,
+            bucket: bucket.to_string(),
+            ..Default::default()
         });
+        err.update_in_progress = true;
+        err.bucket = bucket.to_string();
     }
 
     pub async fn mark_refresh_done(&self, bucket: &str, arn: &str) {
@@ -841,15 +944,21 @@ impl BucketTargetSys {
     }
 
     pub async fn is_reloading_target(&self, _bucket: &str, arn: &str) -> bool {
-        let arn_errs = self.arn_errs_map.read().await;
+        self.arn_errs_map
-        arn_errs.get(arn).map(|err| err.update_in_progress).unwrap_or(false)
+            .read()
+            .await
+            .get(arn)
+            .is_some_and(|err| err.update_in_progress)
     }
 
-    pub async fn inc_arn_errs(&self, _bucket: &str, arn: &str) {
+    pub async fn inc_arn_errs(&self, bucket: &str, arn: &str) {
         let mut arn_errs = self.arn_errs_map.write().await;
-        if let Some(err) = arn_errs.get_mut(arn) {
+        let err = arn_errs.entry(arn.to_string()).or_insert_with(|| ArnErrs {
-            err.count += 1;
+            bucket: bucket.to_string(),
-        }
+            ..Default::default()
+        });
+        err.count += 1;
+        err.bucket = bucket.to_string();
     }
 
     pub async fn get_remote_target_client(&self, bucket: &str, arn: &str) -> Option<Arc<TargetClient>> {
@@ -862,15 +971,15 @@ impl BucketTargetSys {
                 .unwrap_or((None, None))
         };
 
-        if let Some(cli) = cli {
+        let credentials_expired = cli
+            .as_ref()
+            .is_some_and(|client| client.credentials_expired_at(jiff::Timestamp::now()));
+        if let Some(cli) = cli
+            && !credentials_expired
+        {
             return Some(cli);
         }
 
-        // TODO(backlog): spawn an async task to proactively reload the replication target
-        if self.is_reloading_target(bucket, arn).await {
-            return None;
-        }
-
         if let Some(last_refresh) = last_refresh {
             let now = OffsetDateTime::now_utc();
             if now - last_refresh < Duration::from_secs(60 * 5) {
@@ -878,16 +987,24 @@ impl BucketTargetSys {
             }
         }
 
+        // The existing per-bucket publication lock is also the reload claim:
+        // try-locking keeps the request path non-blocking, is cancellation-safe,
+        // and prevents a stale reload from publishing after a credential update.
+        let update_mutex = self.target_update_mutex(bucket).await;
+        let Ok(update_guard) = update_mutex.try_lock() else {
+            return None;
+        };
+        self.mark_refresh_attempt(arn).await;
+
         match get_bucket_targets_config(bucket).await {
             Ok(bucket_targets) => {
-                self.mark_refresh_in_progress(bucket, arn).await;
+                self.update_all_targets_locked(bucket, Some(&bucket_targets)).await;
-                self.update_all_targets(bucket, Some(&bucket_targets)).await;
-                self.mark_refresh_done(bucket, arn).await;
             }
             Err(e) => {
                 error!("get bucket targets config error:{}", e);
             }
         };
+        drop(update_guard);
 
         let cli = self
             .arn_remotes_map
@@ -895,8 +1012,10 @@ impl BucketTargetSys {
             .await
             .get(arn)
             .and_then(|target| target.client.clone());
-        if cli.is_some() {
+        if let Some(cli) = cli
-            return cli;
+            && !cli.credentials_expired_at(jiff::Timestamp::now())
+        {
+            return Some(cli);
         }
 
         self.inc_arn_errs(bucket, arn).await;
@@ -926,12 +1045,13 @@ impl BucketTargetSys {
             });
         };
 
-        let creds = SdkCredentials::builder()
+        let creds = remote_target_sdk_credentials(credentials, &target.reset_id, SystemTime::now()).map_err(|error| {
-            .access_key_id(credentials.access_key.clone())
+            BucketTargetError::RemoteTargetConnectionErr {
-            .secret_access_key(credentials.secret_key.clone())
+                bucket: target.target_bucket.clone(),
-            .account_id(target.reset_id.clone())
+                access_key: credentials.access_key.clone(),
-            .provider_name("bucket_target_sys")
+                error: error.to_string(),
-            .build();
+            }
+        })?;
 
         let endpoint = if target.secure {
             format!("https://{}", target.endpoint)
@@ -951,9 +1071,10 @@ impl BucketTargetSys {
 
         let mut config_builder = S3Config::builder()
             .endpoint_url(endpoint.clone())
-            .credentials_provider(SharedCredentialsProvider::new(creds))
+            .credentials_provider(SharedCredentialsProvider::new(RemoteTargetCredentialsProvider { credentials: creds }))
             .region(SdkRegion::new(target.region.clone()))
-            .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest());
+            .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
+            .request_checksum_calculation(replication_request_checksum_calculation());
 
         if should_force_path_style(target) {
             config_builder = config_builder.force_path_style(true);
@@ -1025,6 +1146,13 @@ impl BucketTargetSys {
         let update_mutex = self.target_update_mutex(bucket).await;
         let _update_guard = update_mutex.lock().await;
 
+        self.update_all_targets_locked(bucket, targets).await;
+    }
+
+    /// Builds and publishes one bucket snapshot while its update mutex is held.
+    /// Keeping persisted-config reads under the same mutex prevents a stale
+    /// reload from overwriting a concurrent credential rotation.
+    async fn update_all_targets_locked(&self, bucket: &str, targets: Option<&BucketTargets>) {
         let mut clients = Vec::new();
         if let Some(new_targets) = targets {
             for target in &new_targets.targets {
@@ -1056,6 +1184,17 @@ impl BucketTargetSys {
             && !new_targets.is_empty()
         {
             for (target, client) in clients {
+                // Keep a timestamped placeholder for configured targets whose
+                // client cannot be built. Replication records these attempts as
+                // failed, while the placeholder prevents every object from
+                // triggering another metadata reload/client build for five minutes.
+                arn_remotes_map.insert(
+                    target.arn.clone(),
+                    ArnTarget {
+                        client: None,
+                        last_refresh: OffsetDateTime::now_utc(),
+                    },
+                );
                 match client {
                     Ok(client) => {
                         arn_remotes_map.insert(
@@ -1068,11 +1207,6 @@ impl BucketTargetSys {
                         health_map.insert(client.arn.clone(), target_health(&client));
                         self.update_bandwidth_limit(bucket, &target.arn, target.bandwidth_limit);
                     }
-                    // The target stays in `targets_map`, so it keeps showing up in
-                    // `bucket remote ls` while no client exists to replicate through it —
-                    // replication then drops every object for this ARN. Without this the
-                    // rejection (loopback endpoint, bad CA, unparseable URL) left no trace
-                    // anywhere.
                     Err(err) => warn!(
                         bucket = %bucket,
                         arn = %target.arn,
@@ -1236,6 +1370,25 @@ fn loopback_replication_targets_allowed() -> bool {
         .unwrap_or(false)
 }
 
+const REPLICATION_STREAMING_CHECKSUMS_ENV: &str = "RUSTFS_REPLICATION_STREAMING_CHECKSUMS";
+
+/// Streaming trailer checksums make the SDK frame request bodies as
+/// `aws-chunked`; a target that does not decode that framing stores the frames
+/// verbatim, silently corrupting every replica while the transfer itself
+/// succeeds (#6853). Plain signed payloads are the compatible default; the env
+/// knob restores trailer checksums for fleets whose targets are all known to
+/// decode them.
+fn replication_request_checksum_calculation() -> RequestChecksumCalculation {
+    if std::env::var(REPLICATION_STREAMING_CHECKSUMS_ENV)
+        .map(|v| v.eq_ignore_ascii_case("true") || v == "1")
+        .unwrap_or(false)
+    {
+        RequestChecksumCalculation::WhenSupported
+    } else {
+        RequestChecksumCalculation::WhenRequired
+    }
+}
+
 fn validate_replication_target_endpoint(url: &Url) -> Result<(), OutboundUrlError> {
     validate_replication_target_endpoint_inner(url, loopback_replication_targets_allowed())
 }
@@ -1615,6 +1768,17 @@ impl Default for AdvancedPutOptions {
     }
 }
 
+/// The subset of the target's PutObject response replication audits.
+#[derive(Debug, Clone)]
+pub struct RemotePutObjectResponse {
+    /// Version id the target assigned (`x-amz-version-id`).
+    pub version_id: Option<String>,
+    /// ETag of what the target stored; `None` when the target withheld it or
+    /// when its encryption mode (SSE-KMS / SSE-C) makes it incomparable to
+    /// the source ETag. `None` is therefore "not decidable", never evidence.
+    pub etag: Option<String>,
+}
+
 #[derive(Clone)]
 pub struct PutObjectOptions {
     pub user_metadata: HashMap<String, String>,
@@ -1940,6 +2104,13 @@ pub struct TargetClient {
 }
 
 impl TargetClient {
+    fn credentials_expired_at(&self, now: jiff::Timestamp) -> bool {
+        self.credentials
+            .as_ref()
+            .and_then(Credentials::effective_expiration)
+            .is_some_and(|expiration| expiration <= now)
+    }
+
     pub fn to_url(&self) -> Url {
         Url::parse(&self.endpoint).unwrap()
     }
@@ -2153,7 +2324,9 @@ impl TargetClient {
 
     /// On success returns the version id the target assigned (from
     /// `x-amz-version-id`), letting callers audit the version-identity
-    /// contract — a target that adopts the source version echoes it back.
+    /// contract — a target that adopts the source version echoes it back —
+    /// together with the ETag of what the target actually stored, so callers
+    /// can detect a target that persisted transformed bytes (#6853).
     pub async fn put_object(
         &self,
         bucket: &str,
@@ -2161,7 +2334,7 @@ impl TargetClient {
         size: i64,
         body: ByteStream,
         opts: &PutObjectOptions,
-    ) -> Result<Option<String>, S3ClientError> {
+    ) -> Result<RemotePutObjectResponse, S3ClientError> {
         let mut headers = opts.header();
 
         let builder = self.client.put_object();
@@ -2196,7 +2369,25 @@ impl TargetClient {
             .send()
             .await
         {
-            Ok(output) => Ok(output.version_id().map(ToOwned::to_owned)),
+            Ok(output) => {
+                // Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
+                // of the stored plaintext, so it cannot be compared against the
+                // source ETag; withhold it rather than let a caller conclude
+                // corruption from an opaque value.
+                let etag_comparable = output.sse_customer_algorithm().is_none()
+                    && !matches!(
+                        output.server_side_encryption(),
+                        Some(ServerSideEncryption::AwsKms) | Some(ServerSideEncryption::AwsKmsDsse)
+                    );
+                Ok(RemotePutObjectResponse {
+                    version_id: output.version_id().map(ToOwned::to_owned),
+                    etag: if etag_comparable {
+                        output.e_tag().map(ToOwned::to_owned)
+                    } else {
+                        None
+                    },
+                })
+            }
             Err(e) => match e {
                 SdkError::ServiceError(service_err) => {
                     let err = service_err.into_err();
@@ -2344,6 +2535,21 @@ impl TargetClient {
         }
     }
 
+    pub async fn abort_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str) -> Result<(), S3ClientError> {
+        match self
+            .client
+            .abort_multipart_upload()
+            .bucket(bucket)
+            .key(object)
+            .upload_id(upload_id)
+            .send()
+            .await
+        {
+            Ok(_) => Ok(()),
+            Err(e) => Err(e.into()),
+        }
+    }
+
     pub async fn remove_object(
         &self,
         bucket: &str,
@@ -2490,6 +2696,18 @@ mod tests {
     use super::*;
     use rcgen::generate_simple_self_signed;
 
+    // The startup panic fix for hosts without a CA bundle (issue #6734) rests
+    // on two properties: the health-check client constructor never panics, and
+    // its degraded fallback — an explicit empty trust store — always builds.
+    #[test]
+    fn health_check_client_construction_never_panics() {
+        let _ = build_health_check_client();
+        HttpClient::builder()
+            .tls_certs_only(std::iter::empty::<reqwest::Certificate>())
+            .build()
+            .expect("empty-trust-store client build must succeed without touching system roots");
+    }
+
     #[derive(Clone, Debug)]
     struct RecordingHttpConnector {
         request_uris: Arc<std::sync::Mutex<Vec<String>>>,
@@ -2508,6 +2726,165 @@ mod tests {
         }
     }
 
+    type RecordedHeaders = Arc<std::sync::Mutex<Vec<Vec<(String, String)>>>>;
+
+    /// Records full request headers and answers with canned response headers,
+    /// for asserting wire framing and response parsing.
+    #[derive(Clone, Debug)]
+    struct RecordingHeaderConnector {
+        request_headers: RecordedHeaders,
+        response_headers: Vec<(String, String)>,
+    }
+
+    impl SmithyHttpConnector for RecordingHeaderConnector {
+        fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
+            self.request_headers
+                .lock()
+                .expect("recorded header lock should not be poisoned")
+                .push(
+                    request
+                        .headers()
+                        .iter()
+                        .map(|(k, v)| (k.to_string(), v.to_string()))
+                        .collect(),
+                );
+            let mut response = HttpResponse::new(
+                aws_smithy_runtime_api::http::StatusCode::try_from(200_u16).expect("200 should be a valid response status"),
+                SdkBody::empty(),
+            );
+            for (name, value) in &self.response_headers {
+                response.headers_mut().insert(name.clone(), value.clone());
+            }
+            HttpConnectorFuture::ready(Ok(response))
+        }
+    }
+
+    fn header_recording_target_client(response_headers: Vec<(String, String)>) -> (TargetClient, RecordedHeaders) {
+        let request_headers: RecordedHeaders = Arc::new(std::sync::Mutex::new(Vec::new()));
+        let connector = SharedHttpConnector::new(RecordingHeaderConnector {
+            request_headers: Arc::clone(&request_headers),
+            response_headers,
+        });
+        let http_client = http_client_fn(move |_settings, _components| connector.clone());
+        let client = s3_client_for_test(443, Some(http_client));
+        (
+            TargetClient {
+                endpoint: "https://localhost:443".to_string(),
+                credentials: None,
+                bucket: "target-bucket".to_string(),
+                storage_class: String::new(),
+                disable_proxy: false,
+                arn: "arn:rustfs:replication:us-east-1:target:bucket".to_string(),
+                reset_id: String::new(),
+                secure: true,
+                health_check_duration: Duration::from_secs(5),
+                replicate_sync: false,
+                client: Arc::new(client),
+            },
+            request_headers,
+        )
+    }
+
+    fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
+        let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
+        let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
+        ByteStream::new(SdkBody::from_body_1_x(body))
+    }
+
+    #[test]
+    fn replication_checksums_default_to_plain_payloads() {
+        assert!(matches!(
+            replication_request_checksum_calculation(),
+            RequestChecksumCalculation::WhenRequired
+        ));
+    }
+
+    #[tokio::test]
+    async fn replication_put_object_sends_plain_signed_payloads_by_default() {
+        let (client, recorded) = header_recording_target_client(Vec::new());
+        client
+            .put_object("target-bucket", "object", 4, streaming_test_body(b"data"), &PutObjectOptions::default())
+            .await
+            .expect("recorded put_object should succeed");
+
+        let recorded = recorded.lock().expect("recorded header lock should not be poisoned");
+        let headers = &recorded[0];
+        let header = |name: &str| {
+            headers
+                .iter()
+                .find(|(k, _)| k.eq_ignore_ascii_case(name))
+                .map(|(_, v)| v.as_str())
+        };
+        // The #6853 regression shape: trailer checksums force aws-chunked
+        // framing, which a non-decoding target stores verbatim as the object.
+        assert_eq!(header("x-amz-trailer"), None, "streaming uploads must not carry a trailer checksum");
+        assert!(
+            header("content-encoding").is_none_or(|v| !v.contains("aws-chunked")),
+            "streaming uploads must not be aws-chunked framed"
+        );
+        assert_eq!(header("x-amz-decoded-content-length"), None);
+        assert_eq!(header("content-length"), Some("4"));
+    }
+
+    #[tokio::test]
+    async fn put_object_returns_the_etag_the_target_stored() {
+        let (client, _) =
+            header_recording_target_client(vec![("etag".to_string(), "\"9a0364b9e99bb480dd25e1f0284c8555\"".to_string())]);
+        let response = client
+            .put_object(
+                "target-bucket",
+                "object",
+                4,
+                ByteStream::from_static(b"data"),
+                &PutObjectOptions::default(),
+            )
+            .await
+            .expect("recorded put_object should succeed");
+        assert_eq!(response.etag.as_deref(), Some("\"9a0364b9e99bb480dd25e1f0284c8555\""));
+    }
+
+    #[tokio::test]
+    async fn put_object_withholds_the_etag_under_target_side_kms() {
+        let (client, _) = header_recording_target_client(vec![
+            ("etag".to_string(), "\"9a0364b9e99bb480dd25e1f0284c8555\"".to_string()),
+            ("x-amz-server-side-encryption".to_string(), "aws:kms".to_string()),
+        ]);
+        let response = client
+            .put_object(
+                "target-bucket",
+                "object",
+                4,
+                ByteStream::from_static(b"data"),
+                &PutObjectOptions::default(),
+            )
+            .await
+            .expect("recorded put_object should succeed");
+        assert!(
+            response.etag.is_none(),
+            "a KMS-encrypted replica's etag is not the content MD5 and must be withheld"
+        );
+    }
+
+    #[derive(Clone, Debug)]
+    struct RecordingAuthConnector {
+        signed_requests: Arc<std::sync::Mutex<Vec<(bool, bool)>>>,
+    }
+
+    impl SmithyHttpConnector for RecordingAuthConnector {
+        fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
+            let has_expected_token = request.headers().get("x-amz-security-token") == Some("temporary-session-token");
+            let has_authorization = request.headers().contains_key("authorization");
+            self.signed_requests
+                .lock()
+                .expect("recorded auth request lock should not be poisoned")
+                .push((has_expected_token, has_authorization));
+            HttpConnectorFuture::ready(Ok(HttpResponse::new(
+                aws_smithy_runtime_api::http::StatusCode::try_from(200_u16).expect("200 should be a valid response status"),
+                SdkBody::empty(),
+            )))
+        }
+    }
+
     fn recording_target_client() -> (TargetClient, Arc<std::sync::Mutex<Vec<String>>>) {
         let request_uris = Arc::new(std::sync::Mutex::new(Vec::new()));
         let connector = SharedHttpConnector::new(RecordingHttpConnector {
@@ -2533,6 +2910,150 @@ mod tests {
         )
     }
 
+    #[test]
+    fn remote_target_sdk_credentials_preserve_temporary_credential_fields() {
+        let now = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000);
+        let expiration = SystemTime::UNIX_EPOCH + Duration::from_secs(2_000);
+        let credentials = Credentials {
+            access_key: "access".to_string(),
+            secret_key: "secret".to_string(),
+            session_token: Some("temporary-session-token".to_string()),
+            expiration: Some(jiff::Timestamp::try_from(expiration).expect("test expiration should convert")),
+        };
+
+        let sdk_credentials =
+            remote_target_sdk_credentials(&credentials, "account", now).expect("unexpired temporary credentials should build");
+
+        assert_eq!(sdk_credentials.session_token(), Some("temporary-session-token"));
+        assert_eq!(sdk_credentials.expiry(), Some(expiration));
+        assert_eq!(sdk_credentials.account_id().map(|id| id.as_str()), Some("account"));
+    }
+
+    #[test]
+    fn remote_target_sdk_credentials_normalize_go_zero_expiration() {
+        let credentials = Credentials {
+            access_key: "access".to_string(),
+            secret_key: "secret".to_string(),
+            session_token: None,
+            expiration: Some("0001-01-01T00:00:00Z".parse().expect("Go zero time should parse")),
+        };
+
+        let sdk_credentials = remote_target_sdk_credentials(&credentials, "", SystemTime::now())
+            .expect("Go zero expiration should remain compatible with static credentials");
+
+        assert!(sdk_credentials.session_token().is_none());
+        assert!(sdk_credentials.expiry().is_none());
+    }
+
+    #[test]
+    fn remote_target_sdk_credentials_reject_invalid_expiration_boundaries() {
+        let expiration = SystemTime::UNIX_EPOCH + Duration::from_secs(2_000);
+        let mut credentials = Credentials {
+            access_key: "access".to_string(),
+            secret_key: "secret".to_string(),
+            session_token: None,
+            expiration: Some(jiff::Timestamp::try_from(expiration).expect("test expiration should convert")),
+        };
+
+        assert_eq!(
+            remote_target_sdk_credentials(&credentials, "", SystemTime::UNIX_EPOCH + Duration::from_secs(1_000))
+                .expect_err("expiration without a session token must fail"),
+            "remote target credential expiration requires a session token"
+        );
+
+        credentials.session_token = Some("temporary-session-token".to_string());
+        assert_eq!(
+            remote_target_sdk_credentials(&credentials, "", expiration)
+                .expect_err("credentials expire at the exact expiration boundary"),
+            EXPIRED_REMOTE_TARGET_CREDENTIALS
+        );
+    }
+
+    #[test]
+    fn remote_target_credentials_provider_fails_closed_after_expiration() {
+        let expiration = SystemTime::UNIX_EPOCH + Duration::from_secs(2_000);
+        let provider = RemoteTargetCredentialsProvider {
+            credentials: SdkCredentials::new(
+                "access",
+                "secret",
+                Some("temporary-session-token".to_string()),
+                Some(expiration),
+                "test",
+            ),
+        };
+
+        assert!(provider.resolve_at(expiration - Duration::from_nanos(1)).is_ok());
+        let err = provider
+            .resolve_at(expiration)
+            .expect_err("expired credentials must not be returned");
+        assert_eq!(err.source().map(ToString::to_string).as_deref(), Some(EXPIRED_REMOTE_TARGET_CREDENTIALS));
+        assert!(!format!("{provider:?}").contains("temporary-session-token"));
+        assert!(!format!("{provider:?}").contains("secret"));
+    }
+
+    #[test]
+    fn target_client_detects_expiration_for_cache_refresh() {
+        let expiration: jiff::Timestamp = "2099-01-01T00:00:00Z".parse().expect("expiration should parse");
+        let (mut client, _) = recording_target_client();
+        client.credentials = Some(Credentials {
+            access_key: "access".to_string(),
+            secret_key: "secret".to_string(),
+            session_token: Some("temporary-session-token".to_string()),
+            expiration: Some(expiration),
+        });
+
+        assert!(!client.credentials_expired_at("2098-12-31T23:59:59Z".parse().expect("pre-expiration timestamp should parse")));
+        assert!(client.credentials_expired_at(expiration));
+
+        client.credentials.as_mut().expect("credentials should exist").expiration =
+            Some("0001-01-01T00:00:00Z".parse().expect("Go zero time should parse"));
+        assert!(!client.credentials_expired_at(jiff::Timestamp::now()));
+    }
+
+    #[tokio::test]
+    async fn temporary_credentials_add_security_token_to_sigv4_requests() {
+        let signed_requests = Arc::new(std::sync::Mutex::new(Vec::new()));
+        let connector = SharedHttpConnector::new(RecordingAuthConnector {
+            signed_requests: Arc::clone(&signed_requests),
+        });
+        let http_client = http_client_fn(move |_settings, _components| connector.clone());
+        let credentials = Credentials {
+            access_key: "access".to_string(),
+            secret_key: "secret".to_string(),
+            session_token: Some("temporary-session-token".to_string()),
+            expiration: Some("2099-01-01T00:00:00Z".parse().expect("future expiration should parse")),
+        };
+        let sdk_credentials = remote_target_sdk_credentials(&credentials, "", SystemTime::now())
+            .expect("unexpired temporary credentials should build");
+        let client = S3Client::from_conf(
+            S3Config::builder()
+                .endpoint_url("https://target.example")
+                .credentials_provider(SharedCredentialsProvider::new(RemoteTargetCredentialsProvider {
+                    credentials: sdk_credentials,
+                }))
+                .region(SdkRegion::new("us-east-1"))
+                .http_client(http_client)
+                .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
+                .build(),
+        );
+
+        client
+            .head_bucket()
+            .bucket("target-bucket")
+            .send()
+            .await
+            .expect("recording connector should accept the signed request");
+
+        assert_eq!(
+            signed_requests
+                .lock()
+                .expect("recorded auth request lock should not be poisoned")
+                .as_slice(),
+            &[(true, true)],
+            "SigV4 request must include both authorization and the session-token header"
+        );
+    }
+
     fn spawn_https_server(cert: &rcgen::CertifiedKey<rcgen::KeyPair>, requests: usize) -> (u16, std::thread::JoinHandle<()>) {
         use std::io::{Read, Write};
 
@@ -2640,7 +3161,10 @@ mod tests {
             .credentials_provider(SharedCredentialsProvider::new(credentials))
             .region(SdkRegion::new("us-east-1"))
             .force_path_style(true)
-            .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest());
+            .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
+            // Mirror the production remote-target builder so recorded requests
+            // exercise the same checksum/framing behavior (#6853).
+            .request_checksum_calculation(replication_request_checksum_calculation());
         if let Some(http_client) = http_client {
             config = config.http_client(http_client);
         }
@@ -3464,6 +3988,29 @@ mod tests {
         assert!(mutexes.contains_key("second"));
     }
 
+    #[tokio::test]
+    async fn target_refresh_attempt_updates_retry_timestamp_and_error_count() {
+        let sys = BucketTargetSys::default();
+
+        sys.mark_refresh_attempt("arn:reload").await;
+        let last_refresh = sys.arn_remotes_map.read().await["arn:reload"].last_refresh;
+        assert!(OffsetDateTime::now_utc() - last_refresh < Duration::from_secs(5));
+
+        sys.inc_arn_errs("bucket", "arn:reload").await;
+        sys.inc_arn_errs("bucket", "arn:reload").await;
+        let errors = sys.arn_errs_map.read().await;
+        assert_eq!(errors["arn:reload"].count, 2);
+        assert_eq!(errors["arn:reload"].bucket, "bucket");
+        drop(errors);
+
+        sys.mark_refresh_in_progress("bucket", "arn:reload").await;
+        assert!(sys.is_reloading_target("bucket", "arn:reload").await);
+        sys.mark_refresh_done("bucket", "arn:reload").await;
+        assert!(!sys.is_reloading_target("bucket", "arn:reload").await);
+        sys.mark_refresh_in_progress("bucket", "arn:reload").await;
+        assert!(sys.is_reloading_target("bucket", "arn:reload").await);
+    }
+
     #[tokio::test]
     async fn update_all_targets_publishes_disable_proxy_on_target_client() {
         // The read-proxy selector (replication_proxy::get_proxy_targets) skips
@@ -3502,6 +4049,88 @@ mod tests {
         assert!(opted_out.disable_proxy, "disable_proxy must reach the published TargetClient");
     }
 
+    #[tokio::test]
+    async fn update_all_targets_keeps_failed_client_placeholder() {
+        let sys = BucketTargetSys::default();
+        let target = BucketTarget {
+            arn: "arn:expired".to_string(),
+            endpoint: "192.168.1.10:9000".to_string(),
+            target_bucket: "target-bucket".to_string(),
+            region: "us-east-1".to_string(),
+            credentials: Some(Credentials {
+                access_key: "access".to_string(),
+                secret_key: "secret".to_string(),
+                session_token: Some("temporary-session-token".to_string()),
+                expiration: Some("2000-01-01T00:00:00Z".parse().expect("expired timestamp should parse")),
+            }),
+            ..Default::default()
+        };
+        let targets = BucketTargets { targets: vec![target] };
+
+        sys.update_all_targets("bucket", Some(&targets)).await;
+
+        let remotes = sys.arn_remotes_map.read().await;
+        let placeholder = remotes
+            .get("arn:expired")
+            .expect("configured target should retain a cache entry");
+        assert!(placeholder.client.is_none());
+        assert!(OffsetDateTime::now_utc() - placeholder.last_refresh < Duration::from_secs(5));
+        drop(remotes);
+        assert!(sys.get_remote_target_client("bucket", "arn:expired").await.is_none());
+    }
+
+    #[tokio::test]
+    async fn credential_rotation_atomically_replaces_published_client() {
+        let sys = BucketTargetSys::default();
+        let target = |session_token: &str| BucketTarget {
+            arn: "arn:rotating".to_string(),
+            endpoint: "192.168.1.10:9000".to_string(),
+            target_bucket: "target-bucket".to_string(),
+            region: "us-east-1".to_string(),
+            credentials: Some(Credentials {
+                access_key: "access".to_string(),
+                secret_key: "secret".to_string(),
+                session_token: Some(session_token.to_string()),
+                expiration: None,
+            }),
+            ..Default::default()
+        };
+
+        sys.update_all_targets(
+            "bucket",
+            Some(&BucketTargets {
+                targets: vec![target("old-session-token")],
+            }),
+        )
+        .await;
+        let old_client = sys
+            .get_remote_target_client("bucket", "arn:rotating")
+            .await
+            .expect("initial client should be published");
+
+        sys.update_all_targets(
+            "bucket",
+            Some(&BucketTargets {
+                targets: vec![target("new-session-token")],
+            }),
+        )
+        .await;
+        let new_client = sys
+            .get_remote_target_client("bucket", "arn:rotating")
+            .await
+            .expect("rotated client should be published");
+
+        assert!(!Arc::ptr_eq(&old_client, &new_client));
+        assert_eq!(
+            old_client.credentials.as_ref().and_then(Credentials::effective_session_token),
+            Some("old-session-token")
+        );
+        assert_eq!(
+            new_client.credentials.as_ref().and_then(Credentials::effective_session_token),
+            Some("new-session-token")
+        );
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn target_updates_serialize_client_build_through_publication_per_bucket() {
         let sys = Arc::new(BucketTargetSys::default());

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -490,7 +490,7 @@ impl ExpiryStats {
     }
 
     fn add_nonnegative(counter: &AtomicI64, delta: i64) {
-        let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_add(delta).max(0)));
+        let _ = counter.try_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_add(delta).max(0)));
     }
 
     fn increment_missed_expiry_tasks(&self) {
@@ -12195,7 +12195,7 @@ mod tests {
     #[tokio::test]
     #[serial]
     async fn tier_free_version_recovery_continues_after_deleted_marker_bucket() {
-        let (_paths, ecstore) = setup_test_env().await;
+        let (disk_paths, ecstore) = setup_test_env().await;
         let suffix = Uuid::new_v4().simple();
         let earlier_bucket = format!("zzzz-recovery-{suffix}-a");
         let deleted_marker = format!("zzzz-recovery-{suffix}-m");
@@ -12203,11 +12203,7 @@ mod tests {
         let later_object = "a-before-stale-marker";
         create_test_bucket(&ecstore, &earlier_bucket).await;
         create_test_bucket(&ecstore, &later_bucket).await;
-        let mut reader = PutObjReader::from_vec(b"cursor reset probe".to_vec());
+        seed_recoverable_free_version(&disk_paths, &later_bucket, later_object, None, None).await;
-        ecstore
-            .put_object(&later_bucket, later_object, &mut reader, &ObjectOptions::default())
-            .await
-            .expect("successor bucket object should be created");
 
         let page = list_tier_free_versions(
             Arc::clone(&ecstore),
@@ -12220,14 +12216,10 @@ mod tests {
         .expect("recovery should resume at the first bucket after a deleted marker bucket");
 
         assert_eq!(page.buckets_scanned, 1, "the later bucket must not be skipped");
-        assert_eq!(
+        assert_eq!(page.items.len(), 1, "the successor bucket's recoverable object must be returned");
-            page.scanned_entries, 1,
+        assert_eq!(page.items[0].bucket, later_bucket);
-            "the deleted bucket's object marker must not skip objects in the successor bucket"
+        assert_eq!(page.items[0].name, later_object);
-        );
+        remove_seeded_free_version(&disk_paths, &later_bucket, later_object).await;
-        ecstore
-            .delete_object(&later_bucket, later_object, ObjectOptions::default())
-            .await
-            .expect("successor bucket object should be removed");
         for bucket in [&earlier_bucket, &later_bucket] {
             ecstore
                 .delete_bucket(bucket, &DeleteBucketOptions::default())

crates/ecstore/src/bucket/lifecycle/mod.rs

@@ -20,14 +20,13 @@ mod durable_namespace;
 pub mod evaluator;
 pub mod manual_transition_job;
 mod metadata_boundary;
-pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs};
+pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs, get_lifecycle_config};
 mod object_handlers_common;
 mod object_lock_boundary;
 pub use self::core as lifecycle;
 mod replication_sink;
 pub mod rule;
 mod runtime_boundary;
-mod tagging_boundary;
 pub mod tier_delete_journal;
 pub mod tier_free_version_recovery;
 pub mod tier_last_day_stats;

crates/ecstore/src/bucket/lifecycle/tagging_boundary.rs

@@ -1,37 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashMap;
-
-#[allow(
-    dead_code,
-    reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
-)]
-pub(crate) fn decode_tags_to_map(tags: &str) -> HashMap<String, String> {
-    crate::bucket::tagging::decode_tags_to_map(tags)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::decode_tags_to_map;
-
-    #[test]
-    fn decode_tags_to_map_preserves_bucket_tagging_parser_behavior() {
-        let tags = decode_tags_to_map("env=prod&encoded=a%2Fb&=ignored");
-
-        assert_eq!(tags.get("env").map(String::as_str), Some("prod"));
-        assert_eq!(tags.get("encoded").map(String::as_str), Some("a/b"));
-        assert!(!tags.contains_key(""));
-    }
-}

crates/ecstore/src/bucket/metadata_sys.rs

@@ -412,8 +412,14 @@ pub(crate) fn require_bucket_metadata_sys_in(
 }
 
 pub(crate) async fn object_store_in(ctx: &crate::runtime::instance::InstanceContext) -> Result<Arc<ECStore>> {
-    let sys = bucket_metadata_sys_of(ctx)?;
+    object_store_if_initialized_in(ctx)
-    Ok(sys.read().await.api.clone())
+        .await
+        .ok_or_else(|| Error::other("bucket metadata sys not initialized for this instance"))
+}
+
+pub(crate) async fn object_store_if_initialized_in(ctx: &crate::runtime::instance::InstanceContext) -> Option<Arc<ECStore>> {
+    let sys = ctx.bucket_metadata_sys().or_else(get_global_bucket_metadata_sys)?;
+    Some(sys.read().await.api.clone())
 }
 
 pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<Arc<BucketMetadata>> {
@@ -2512,11 +2518,169 @@ pub(crate) mod test_support {
 mod tests {
     use super::test_support::isolated_store_over_temp_disks;
     use super::*;
+    use crate::bucket::metadata::{
+        BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_NOTIFICATION_CONFIG,
+        BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG,
+        BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_VERSIONING_CONFIG, BUCKET_WEBSITE_CONFIG, OBJECT_LOCK_CONFIG,
+    };
     use crate::bucket::target::{BucketTarget, BucketTargetType, Credentials};
+    use crate::config::com::read_config;
     use crate::storage_api_contracts::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
+    use byteorder::{ByteOrder as _, LittleEndian};
     use serial_test::serial;
     use tokio::time::timeout;
 
+    const NEW_WRITER_REPLICATION_XML: &[u8] = br#"<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Role>arn:aws:iam::111122223333:role/replication-role</Role><Rule><ID>rollback</ID><Priority>1</Priority><Filter><Prefix>documents/</Prefix></Filter><Status>Enabled</Status><Destination><Bucket>arn:aws:s3:::replica-bucket</Bucket></Destination><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication></Rule></ReplicationConfiguration>"#;
+
+    const NEW_WRITER_CONFIGS: [(&str, &[u8]); 14] = [
+        (BUCKET_POLICY_CONFIG, br#"{"Version":"2012-10-17","Statement":[]}"#),
+        (BUCKET_NOTIFICATION_CONFIG, br#"<NotificationConfiguration/>"#),
+        (
+            BUCKET_LIFECYCLE_CONFIG,
+            br#"<LifecycleConfiguration><Rule><ID>expire</ID><Status>Enabled</Status><Filter><Prefix>logs/</Prefix></Filter><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>"#,
+        ),
+        (
+            OBJECT_LOCK_CONFIG,
+            br#"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>7</Days></DefaultRetention></Rule></ObjectLockConfiguration>"#,
+        ),
+        (
+            BUCKET_VERSIONING_CONFIG,
+            br#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#,
+        ),
+        (
+            BUCKET_SSECONFIG,
+            br#"<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>"#,
+        ),
+        (
+            BUCKET_TAGGING_CONFIG,
+            r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
+        ),
+        (BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
+        (
+            BUCKET_CORS_CONFIG,
+            br#"<CORSConfiguration><CORSRule><AllowedMethod>GET</AllowedMethod><AllowedOrigin>https://example.test</AllowedOrigin></CORSRule></CORSConfiguration>"#,
+        ),
+        (BUCKET_LOGGING_CONFIG, br#"<BucketLoggingStatus/>"#),
+        (
+            BUCKET_WEBSITE_CONFIG,
+            br#"<WebsiteConfiguration><IndexDocument><Suffix>index.html</Suffix></IndexDocument></WebsiteConfiguration>"#,
+        ),
+        (
+            BUCKET_ACCELERATE_CONFIG,
+            br#"<AccelerateConfiguration><Status>Enabled</Status></AccelerateConfiguration>"#,
+        ),
+        (
+            BUCKET_REQUEST_PAYMENT_CONFIG,
+            br#"<RequestPaymentConfiguration><Payer>Requester</Payer></RequestPaymentConfiguration>"#,
+        ),
+        (
+            BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG,
+            br#"<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls><IgnorePublicAcls>true</IgnorePublicAcls><BlockPublicPolicy>true</BlockPublicPolicy><RestrictPublicBuckets>false</RestrictPublicBuckets></PublicAccessBlockConfiguration>"#,
+        ),
+    ];
+
+    #[tokio::test]
+    async fn g_d3_003_new_writer_replication_loads_without_fail_closed_state() {
+        let (dirs, store) = isolated_store_over_temp_disks().await;
+        let bucket = "rollback-new-replication";
+        for dir in &dirs {
+            std::fs::create_dir_all(dir.path().join(bucket)).expect("rollback fixture bucket should be created");
+        }
+
+        let writer = BucketMetadataSys::new(store.clone());
+        let mut metadata = BucketMetadata::new(bucket);
+        metadata
+            .update_config(BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML.to_vec())
+            .expect("new-writer replication XML should be accepted before persistence");
+        writer
+            .persist_new_and_set(metadata)
+            .await
+            .expect("new-writer replication metadata should persist");
+
+        let old_reader = BucketMetadataSys::new(store);
+        let (loaded, _) = old_reader
+            .get_replication_config(bucket)
+            .await
+            .expect("old metadata_sys must not classify new-writer replication XML as invalid");
+        assert_eq!(loaded.role, "arn:aws:iam::111122223333:role/replication-role");
+        assert_eq!(loaded.rules.len(), 1);
+        assert_eq!(loaded.rules[0].id.as_deref(), Some("rollback"));
+    }
+
+    #[tokio::test]
+    async fn g_d3_004_new_writer_metadata_blob_keeps_legacy_header_and_configs() {
+        let (dirs, store) = isolated_store_over_temp_disks().await;
+        let bucket = "rollback-new-metadata";
+        for dir in &dirs {
+            std::fs::create_dir_all(dir.path().join(bucket)).expect("rollback fixture bucket should be created");
+        }
+
+        let writer = BucketMetadataSys::new(store.clone());
+        let mut metadata = BucketMetadata::new(bucket);
+        for (config_file, bytes) in NEW_WRITER_CONFIGS {
+            metadata
+                .update_config(config_file, bytes.to_vec())
+                .unwrap_or_else(|err| panic!("new-writer {config_file} fixture must be valid: {err}"));
+        }
+        writer
+            .persist_new_and_set(metadata)
+            .await
+            .expect("new-writer metadata should persist");
+
+        let path = BucketMetadata::new(bucket).save_file_path();
+        let blob = read_config(store.clone(), &path)
+            .await
+            .expect("persisted .metadata.bin should be readable");
+        assert_eq!(
+            LittleEndian::read_u16(&blob[0..2]),
+            1,
+            "bucket metadata format must stay rollback-readable"
+        );
+        assert_eq!(
+            LittleEndian::read_u16(&blob[2..4]),
+            1,
+            "bucket metadata version must stay rollback-readable"
+        );
+
+        let loaded = load_bucket_metadata(store, bucket)
+            .await
+            .expect("old read_bucket_metadata path must load the new-writer blob");
+        let loaded_configs: [(&str, &[u8]); 14] = [
+            (BUCKET_POLICY_CONFIG, &loaded.policy_config_json),
+            (BUCKET_NOTIFICATION_CONFIG, &loaded.notification_config_xml),
+            (BUCKET_LIFECYCLE_CONFIG, &loaded.lifecycle_config_xml),
+            (OBJECT_LOCK_CONFIG, &loaded.object_lock_config_xml),
+            (BUCKET_VERSIONING_CONFIG, &loaded.versioning_config_xml),
+            (BUCKET_SSECONFIG, &loaded.encryption_config_xml),
+            (BUCKET_TAGGING_CONFIG, &loaded.tagging_config_xml),
+            (BUCKET_REPLICATION_CONFIG, &loaded.replication_config_xml),
+            (BUCKET_CORS_CONFIG, &loaded.cors_config_xml),
+            (BUCKET_LOGGING_CONFIG, &loaded.logging_config_xml),
+            (BUCKET_WEBSITE_CONFIG, &loaded.website_config_xml),
+            (BUCKET_ACCELERATE_CONFIG, &loaded.accelerate_config_xml),
+            (BUCKET_REQUEST_PAYMENT_CONFIG, &loaded.request_payment_config_xml),
+            (BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, &loaded.public_access_block_config_xml),
+        ];
+        for ((expected_name, expected), (loaded_name, actual)) in NEW_WRITER_CONFIGS.into_iter().zip(loaded_configs) {
+            assert_eq!(loaded_name, expected_name);
+            assert_eq!(actual, expected, "old read_bucket_metadata changed {expected_name} bytes");
+        }
+        assert!(loaded.policy_config.is_some());
+        assert!(loaded.notification_config.is_some());
+        assert!(loaded.lifecycle_config.is_some());
+        assert!(loaded.object_lock_config.is_some());
+        assert!(loaded.versioning_config.is_some());
+        assert!(loaded.sse_config.is_some());
+        assert!(loaded.tagging_config.is_some());
+        assert!(loaded.replication_config.is_some());
+        assert!(loaded.cors_config.is_some());
+        assert!(loaded.logging_config.is_some());
+        assert!(loaded.website_config.is_some());
+        assert!(loaded.accelerate_config.is_some());
+        assert!(loaded.request_payment_config.is_some());
+        assert!(loaded.public_access_block_config.is_some());
+    }
+
     #[tokio::test]
     async fn malformed_delete_configs_are_not_treated_as_absent() {
         let (_dirs, ecstore) = isolated_store_over_temp_disks().await;

crates/ecstore/src/bucket/object_lock/objectlock_sys.rs

@@ -177,6 +177,28 @@ pub fn replication_write_may_pass_worm_gate(
     Ok(!(retention_locked && opts.replication_retention_timestamp.is_none()))
 }
 
+/// Whether an authorized replication delete (`ObjectOptions::replication_request`)
+/// addressed to an explicit version may bypass GOVERNANCE retention on the
+/// local replica, exactly as an `x-amz-bypass-governance-retention` caller
+/// with the bypass permission would.
+///
+/// The source is authoritative for a replicated version purge (issue #6850):
+/// the same WORM deletion gate already ran there, and GOVERNANCE retention
+/// with an authorized bypass is the only lock state it can purge through.
+/// Requiring the bypass header again here makes the purge permanently
+/// undeliverable — replication senders never carry it — and the sites diverge
+/// forever. COMPLIANCE retention and legal hold stay blocking: the source
+/// gate can never purge through them, so a replication purge that meets one
+/// here is divergence or forgery and fails closed.
+///
+/// The trust judgment is the same one the write-path exemption uses:
+/// `replication_request` is only set once the receiving handler has
+/// authorized the caller for the replication action
+/// (`ReplicateDeleteAction`), never straight from request headers.
+pub fn replication_delete_may_bypass_governance(opts: &ObjectOptions) -> bool {
+    opts.replication_request && opts.version_id.is_some()
+}
+
 /// Check if an object is locked based on its metadata.
 /// This is a common function used by both lifecycle evaluation and deletion checks.
 ///
@@ -680,6 +702,32 @@ mod tests {
         assert!(err.to_string().contains("modification time"));
     }
 
+    /// The replicated-purge GOVERNANCE bypass (#6850) applies only to an
+    /// authorized replication delete addressed to an explicit version: a
+    /// local delete never gets it, and a replicated delete without a version
+    /// id creates a delete marker rather than purging anything.
+    #[test]
+    fn replication_delete_bypasses_governance_only_for_authorized_version_purges() {
+        let version_purge = ObjectOptions {
+            replication_request: true,
+            version_id: Some("6b6ffbc0-b0d3-4a86-8f6c-fe19163b8dcd".to_string()),
+            ..Default::default()
+        };
+        assert!(replication_delete_may_bypass_governance(&version_purge));
+
+        let local_version_delete = ObjectOptions {
+            replication_request: false,
+            ..version_purge.clone()
+        };
+        assert!(!replication_delete_may_bypass_governance(&local_version_delete));
+
+        let replicated_marker_creation = ObjectOptions {
+            version_id: None,
+            ..version_purge
+        };
+        assert!(!replication_delete_may_bypass_governance(&replicated_marker_creation));
+    }
+
     /// A local PutObjectRetention / PutObjectLegalHold "clear" persists the
     /// lock keys as empty strings (the MinIO on-disk shape, see
     /// `parse_object_lock_retention`); that is "no lock", not corruption, and

crates/ecstore/src/bucket/replication/mod.rs

@@ -44,14 +44,14 @@ mod replication_versioning_boundary;
 mod runtime_boundary;
 
 pub use replication_config_boundary::{
-    ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
+    ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_READ_ONLY_HISTORICAL_FIELDS,
-    REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
+    REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION,
-    REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
+    REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError,
-    assign_site_replication_rule_priorities, invalid_replication_config_status_field, is_site_replication_role,
+    ReplicationConfigurationExt, ReplicationTargetValidationError, assign_site_replication_rule_priorities,
-    is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
+    invalid_replication_config_status_field, is_site_replication_role, is_site_replication_rule,
-    replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
+    merge_incoming_replication_config, merge_user_replication_config, replication_target_arn_deployment_id,
-    site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
+    replication_target_arns, should_remove_replication_target, site_replication_rule_deployment_id,
-    validate_replication_config_target_arns,
+    unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
 };
 pub(crate) use replication_filemeta_boundary::version_purge_statuses_map;
 pub use replication_filemeta_boundary::{

crates/ecstore/src/bucket/replication/replication_config_boundary.rs

@@ -13,12 +13,12 @@
 // limitations under the License.
 
 pub use rustfs_replication::{
-    ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
+    ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_READ_ONLY_HISTORICAL_FIELDS,
-    REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
+    REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION,
-    REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt,
+    REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError,
-    ReplicationTargetValidationError, assign_site_replication_rule_priorities, invalid_replication_config_status_field,
+    ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError, assign_site_replication_rule_priorities,
-    is_site_replication_role, is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
+    invalid_replication_config_status_field, is_site_replication_role, is_site_replication_rule,
-    replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
+    merge_incoming_replication_config, merge_user_replication_config, replication_target_arn_deployment_id,
-    site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
+    replication_target_arns, should_remove_replication_target, site_replication_rule_deployment_id,
-    validate_replication_config_target_arns,
+    unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
 };

crates/ecstore/src/bucket/replication/replication_object_config.rs

@@ -436,16 +436,21 @@ pub(crate) async fn check_replicate_delete_strict(
     }
 
     for target in decision.targets_map.values_mut() {
-        if let Some(client) = ReplicationTargetStore::remote_target_client(bucket, &target.arn).await {
+        let replicate_sync = ReplicationTargetStore::remote_target_client(bucket, &target.arn)
-            target.synchronous = client.replicate_sync;
+            .await
-        } else {
+            .map(|client| client.replicate_sync);
-            target.replicate = false;
+        apply_target_delivery_mode(target, replicate_sync);
-            target.synchronous = false;
-        }
     }
     Ok(decision)
 }
 
+fn apply_target_delivery_mode(target: &mut ReplicateTargetDecision, replicate_sync: Option<bool>) {
+    // A missing runtime client is a delivery failure, not a rule mismatch.
+    // Preserve admission and fall back to the asynchronous worker, which can
+    // persist FAILED state for the heal/retry path.
+    target.synchronous = replicate_sync.unwrap_or(false);
+}
+
 pub(crate) fn check_replicate_delete_with_snapshot(
     dobj: &ObjectToDelete,
     oi: &ObjectInfo,
@@ -629,6 +634,23 @@ mod tests {
         }));
     }
 
+    #[test]
+    fn missing_target_client_preserves_delete_admission_as_async() {
+        let mut target = ReplicateTargetDecision::new("arn:target".to_string(), true, true);
+
+        apply_target_delivery_mode(&mut target, None);
+
+        assert!(target.replicate, "a runtime client miss must not erase the replication rule decision");
+        assert!(
+            !target.synchronous,
+            "unavailable synchronous targets must fall back to the async retry path"
+        );
+
+        apply_target_delivery_mode(&mut target, Some(true));
+        assert!(target.replicate);
+        assert!(target.synchronous);
+    }
+
     #[test]
     fn must_replicate_options_preserve_request_flag() {
         let user_defined = HashMap::new();

crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs

@@ -19,9 +19,9 @@ pub use rustfs_replication::{
 };
 pub(crate) use rustfs_replication::{
     ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject, delete_marker_purge_mrf_entry,
-    delete_marker_purge_version_id, delete_replication_missing_source_decision, delete_replication_object_opts,
+    delete_marker_purge_version_id, delete_replication_creates_marker, delete_replication_missing_source_decision,
-    heal_uses_delete_replication_path, is_retryable_delete_replication_head_error, is_version_delete_replication,
+    delete_replication_object_opts, heal_uses_delete_replication_path, is_object_lock_denied_delete,
-    replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
+    is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match,
-    replication_multipart_part_plan, resync_existing_delete_replication_info, resync_target_for_object,
+    replication_multipart_complete_actual_size, replication_multipart_part_plan, resync_existing_delete_replication_info,
-    should_retry_delete_marker_purge, target_delete_version_id,
+    resync_target_for_object, should_retry_delete_marker_purge, single_part_replica_etag_mismatch, target_delete_version_id,
 };

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -1048,7 +1048,6 @@ pub fn resync_start_conflict_id(error: &EcstoreError) -> Option<&str> {
 }
 
 /// Main replication pool structure
-#[derive(Debug)]
 pub struct ReplicationPool<S: ReplicationStorage> {
     // Atomic counters for active workers
     active_workers: Arc<AtomicI32>,
@@ -1094,6 +1093,16 @@ pub struct ReplicationPool<S: ReplicationStorage> {
     resyncer: Arc<ReplicationResyncer>,
 }
 
+impl<S: ReplicationStorage> std::fmt::Debug for ReplicationPool<S> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ReplicationPool")
+            .field("active_workers", &self.active_workers.load(Ordering::Relaxed))
+            .field("active_lrg_workers", &self.active_lrg_workers.load(Ordering::Relaxed))
+            .field("active_mrf_workers", &self.active_mrf_workers.load(Ordering::Relaxed))
+            .finish_non_exhaustive()
+    }
+}
+
 impl<S: ReplicationStorage> ReplicationPool<S> {
     /// Creates a new replication pool with specified options
     pub async fn new(opts: ReplicationPoolOpts, stats: Arc<ReplicationStats>, storage: Arc<S>) -> Arc<Self> {
@@ -2132,7 +2141,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
     }
 
     /// Load bucket replication resync statuses into memory
-    #[instrument(skip(_cancellation_token))]
+    #[instrument(skip(self, buckets, _cancellation_token), fields(bucket_count = buckets.len()))]
     async fn load_resync(
         self: Arc<Self>,
         buckets: &[String],
@@ -3168,6 +3177,19 @@ pub(crate) async fn queue_replication_heal_internal(
             }
         }
         ReplicationHealQueueAction::QueueDelete(dv) => {
+            // A purge the peer denied under object lock cannot succeed until
+            // the lock lapses (#6850); requeuing it every heal cycle only
+            // burns bandwidth and failure counters. The backoff expires on
+            // its own, so the purge is probed again — and converges — once
+            // the retention window has a chance of being over.
+            if super::replication_object_decision_boundary::is_version_delete_replication(&dv.delete_object)
+                && super::replication_resyncer::object_lock_denied_purge_backoff_active(&dv)
+            {
+                return ReplicationHealQueueResult {
+                    object_info: roi,
+                    admission: ReplicationQueueAdmission::Skipped,
+                };
+            }
             let admission = if let Some(pool) = runtime_sources::replication_pool() {
                 pool.queue_replica_delete_task(dv).await
             } else {

crates/ecstore/src/bucket/replication/replication_target_boundary.rs

@@ -36,8 +36,8 @@ use time::OffsetDateTime;
 use time::format_description::well_known::Rfc3339;
 
 pub(crate) use crate::bucket::bucket_target_sys::{
-    AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
+    AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemotePutObjectResponse, RemoveObjectOptions,
-    resolve_read_api_version_id,
+    S3ClientError, TargetClient, resolve_read_api_version_id,
 };
 #[cfg(test)]
 pub(crate) use crate::bucket::target::BucketTarget;

crates/ecstore/src/bucket/target/bucket_target.rs

@@ -25,6 +25,8 @@ use time::OffsetDateTime;
 use url::Url;
 
 const REDACTED_CREDENTIAL: &str = "<redacted>";
+const GO_YEAR_ONE_START_UNIX_SECONDS: i64 = -62_135_596_800;
+const GO_YEAR_TWO_START_UNIX_SECONDS: i64 = -62_104_060_800;
 
 #[derive(Deserialize, Serialize, Default, Clone)]
 pub struct Credentials {
@@ -41,6 +43,26 @@ pub struct Credentials {
 }
 
 impl Credentials {
+    /// Returns the session token used for request signing.
+    ///
+    /// MinIO-compatible payloads may carry an empty token. Treat whitespace-only
+    /// values as absent without rewriting a real token, whose bytes are opaque.
+    pub fn effective_session_token(&self) -> Option<&str> {
+        self.session_token.as_deref().filter(|token| !token.trim().is_empty())
+    }
+
+    /// Returns the credential expiry after normalizing Go's zero `time.Time`.
+    ///
+    /// Go JSON encoders emit year 1 for an unset `time.Time`; persisted MinIO
+    /// target metadata can therefore contain that sentinel even for static
+    /// credentials.
+    pub fn effective_expiration(&self) -> Option<Timestamp> {
+        self.expiration.filter(|expiration| {
+            let unix_seconds = expiration.as_second();
+            !(GO_YEAR_ONE_START_UNIX_SECONDS..GO_YEAR_TWO_START_UNIX_SECONDS).contains(&unix_seconds)
+        })
+    }
+
     pub fn redacted(&self) -> Self {
         Self {
             access_key: self.access_key.clone(),
@@ -355,6 +377,24 @@ mod tests {
     use std::time::Duration;
     use time::OffsetDateTime;
 
+    #[test]
+    fn credential_effective_values_normalize_only_compatibility_sentinels() {
+        let mut credentials = Credentials {
+            access_key: "access".to_string(),
+            secret_key: "secret".to_string(),
+            session_token: Some("  ".to_string()),
+            expiration: Some("0001-01-01T08:00:00+08:00".parse().expect("Go zero time should parse")),
+        };
+
+        assert!(credentials.effective_session_token().is_none());
+        assert!(credentials.effective_expiration().is_none());
+
+        credentials.session_token = Some(" opaque token ".to_string());
+        credentials.expiration = Some("2099-01-01T00:00:00Z".parse().expect("future timestamp should parse"));
+        assert_eq!(credentials.effective_session_token(), Some(" opaque token "));
+        assert_eq!(credentials.effective_expiration(), credentials.expiration);
+    }
+
     #[test]
     fn test_bucket_target_json_deserialize() {
         let json = r#"

crates/ecstore/src/bucket/utils.rs

@@ -73,6 +73,7 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> {
     check_bucket_name_common(bucket_name, true)
 }
 
+// RUSTFS_COMPAT_TODO(s3gate-metadata-xml): the s3s codec reads persisted XML during migration. Remove after every supported writer uses the gateway codec and every retained metadata object and backup archive is verified or rewritten.
 pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
 where
     T: for<'xml> xml::Deserialize<'xml>,

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -1307,6 +1307,32 @@ pub fn verify_tonic_mutation_body_digest<T>(request: &tonic::Request<T>, canonic
     verify_tonic_mutation_body_digest_with_strictness(request, canonical_body, internode_rpc_body_digest_strict())
 }
 
+/// Verify a non-disk mutation without accepting a newly-generated unsigned v2 body.
+///
+/// The disk mutation lane has a rolling-upgrade exception for `UNSIGNED-PAYLOAD`
+/// while peer replay-cache capability is being discovered. Historical v2 peers
+/// used the fixed `unsigned` nonce before body-digest rollout; preserve that
+/// exact marker for mixed-version compatibility, but reject unsigned v2
+/// requests that omit it or present a different nonce.
+pub fn verify_tonic_mutation_body_digest_reject_unsigned<T>(
+    request: &tonic::Request<T>,
+    canonical_body: &[u8],
+) -> std::io::Result<()> {
+    let version = request
+        .metadata()
+        .get(RPC_AUTH_VERSION_HEADER)
+        .and_then(|value| value.to_str().ok());
+    let digest = request
+        .metadata()
+        .get(RPC_CONTENT_SHA256_HEADER)
+        .and_then(|value| value.to_str().ok());
+    let nonce = request.metadata().get(RPC_NONCE_HEADER).and_then(|value| value.to_str().ok());
+    if version == Some(RPC_AUTH_VERSION_V2) && digest == Some(UNSIGNED_PAYLOAD) && nonce != Some("unsigned") {
+        return Err(std::io::Error::other("RPC mutation requires a body-bound v2 signature"));
+    }
+    verify_tonic_mutation_body_digest(request, canonical_body)
+}
+
 /// [`verify_tonic_mutation_body_digest`] with the strict gate injected as a parameter, so both
 /// rollout postures are unit-testable without racing on process-global environment variables.
 fn verify_tonic_mutation_body_digest_with_strictness<T>(

crates/ecstore/src/cluster/rpc/internode_data_transport.rs

@@ -36,6 +36,7 @@ use rustfs_rio::{ChunkReaderBox, HttpChunkReader, HttpReader, HttpWriter};
 use sha2::{Digest, Sha256};
 use std::collections::HashMap;
 use std::future::Future;
+use std::io;
 use std::pin::Pin;
 use std::sync::{Arc, LazyLock, OnceLock};
 use std::task::{Context, Poll};
@@ -105,9 +106,13 @@ struct PutFileCapabilityCacheState {
     cached: Option<PutFileCapabilityState>,
     generation: u64,
     in_flight: Option<PutFileCapabilityFlight>,
+    rejected_server_epoch: Option<Uuid>,
 }
 
-type PutFileCapabilityCacheEntry = Arc<tokio::sync::RwLock<PutFileCapabilityCacheState>>;
+// The registry lock is released before taking an entry lock. Entry guards cover
+// only cache transitions, never a probe or await; poll-based writers must be
+// able to reject an epoch atomically with those transitions.
+type PutFileCapabilityCacheEntry = Arc<parking_lot::RwLock<PutFileCapabilityCacheState>>;
 
 static PUT_FILE_CAPABILITY_CACHE: LazyLock<parking_lot::RwLock<HashMap<String, PutFileCapabilityCacheEntry>>> =
     LazyLock::new(|| parking_lot::RwLock::new(HashMap::new()));
@@ -119,7 +124,7 @@ fn put_file_capability_cache_entry(endpoint: &str) -> PutFileCapabilityCacheEntr
     PUT_FILE_CAPABILITY_CACHE
         .write()
         .entry(endpoint.to_owned())
-        .or_insert_with(|| Arc::new(tokio::sync::RwLock::new(PutFileCapabilityCacheState::default())))
+        .or_insert_with(|| Arc::new(parking_lot::RwLock::new(PutFileCapabilityCacheState::default())))
         .clone()
 }
 
@@ -134,6 +139,23 @@ fn fresh_put_file_capability(state: Option<PutFileCapabilityState>, now: Instant
     }
 }
 
+fn reject_put_file_server_epoch(endpoint: &str, server_epoch: Uuid) {
+    let entry = PUT_FILE_CAPABILITY_CACHE.read().get(endpoint).cloned();
+    if let Some(entry) = entry {
+        let mut state = entry.write();
+        if matches!(state.cached, Some(PutFileCapabilityState::V1 { server_epoch: cached, .. }) if cached == server_epoch) {
+            state.rejected_server_epoch = Some(server_epoch);
+        }
+    }
+}
+
+fn usable_put_file_capability(state: &PutFileCapabilityCacheState, now: Instant) -> Option<Option<Uuid>> {
+    match fresh_put_file_capability(state.cached, now)? {
+        Some(server_epoch) if state.rejected_server_epoch == Some(server_epoch) => None,
+        capability => Some(capability),
+    }
+}
+
 fn put_file_capability_status_is_legacy(status: u16) -> bool {
     status == 404
 }
@@ -322,13 +344,14 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
 
     async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
         let server_epoch = self.put_file_auth_capability(&request.endpoint).await?;
-        let nonce = server_epoch.map(|_| Uuid::new_v4());
+        let auth_scope = server_epoch.map(|server_epoch| (Uuid::new_v4(), server_epoch));
-        let url = build_put_file_stream_url(&request, nonce.zip(server_epoch));
+        let url = build_put_file_stream_url(&request, auth_scope);
+        let endpoint = request.endpoint;
         let mut headers = json_headers();
         build_auth_headers(&url, &Method::PUT, &mut headers)?;
         let writer = HttpWriter::new(url.clone(), Method::PUT, headers).await?;
-        match nonce {
+        match auth_scope {
-            Some(nonce) => Ok(Box::new(PutFileAuthWriter::new(writer, url, nonce))),
+            Some((nonce, server_epoch)) => Ok(Box::new(PutFileAuthWriter::new(writer, url, nonce, endpoint, server_epoch))),
             None => Ok(Box::new(writer)),
         }
     }
@@ -498,15 +521,15 @@ where
 {
     let entry = put_file_capability_cache_entry(endpoint);
     {
-        let state = entry.read().await;
+        let state = entry.read();
-        if let Some(cached) = fresh_put_file_capability(state.cached, Instant::now()) {
+        if let Some(cached) = usable_put_file_capability(&state, Instant::now()) {
             return Ok(cached);
         }
     }
 
     let flight = {
-        let mut state = entry.write().await;
+        let mut state = entry.write();
-        if let Some(cached) = fresh_put_file_capability(state.cached, Instant::now()) {
+        if let Some(cached) = usable_put_file_capability(&state, Instant::now()) {
             return Ok(cached);
         }
         if let Some(flight) = state.in_flight.clone() {
@@ -532,7 +555,7 @@ where
         .await;
 
     {
-        let mut state = entry.write().await;
+        let mut state = entry.write();
         let is_current_flight = state
             .in_flight
             .as_ref()
@@ -540,6 +563,9 @@ where
         if is_current_flight {
             match outcome {
                 Ok(Some(server_epoch)) => {
+                    if state.rejected_server_epoch != Some(*server_epoch) {
+                        state.rejected_server_epoch = None;
+                    }
                     state.cached = Some(PutFileCapabilityState::V1 {
                         server_epoch: *server_epoch,
                         revalidate_after: Instant::now() + PUT_FILE_V1_CAPABILITY_TTL,
@@ -630,17 +656,23 @@ struct PutFileAuthWriter<W> {
     inner: W,
     url: String,
     nonce: Uuid,
+    endpoint: String,
+    server_epoch: Uuid,
+    server_epoch_rejected: bool,
     hasher: Sha256,
     trailer: Option<Vec<u8>>,
     trailer_offset: usize,
 }
 
 impl<W> PutFileAuthWriter<W> {
-    fn new(inner: W, url: String, nonce: Uuid) -> Self {
+    fn new(inner: W, url: String, nonce: Uuid, endpoint: String, server_epoch: Uuid) -> Self {
         Self {
             inner,
             url,
             nonce,
+            endpoint,
+            server_epoch,
+            server_epoch_rejected: false,
             hasher: Sha256::new(),
             trailer: None,
             trailer_offset: 0,
@@ -656,6 +688,14 @@ impl<W> PutFileAuthWriter<W> {
         Ok(())
     }
 
+    fn reject_server_epoch_on_conflict(&mut self, error: &io::Error) {
+        if self.server_epoch_rejected || !io_error_has_put_file_epoch_conflict(error) {
+            return;
+        }
+        reject_put_file_server_epoch(&self.endpoint, self.server_epoch);
+        self.server_epoch_rejected = true;
+    }
+
     fn poll_write_trailer(&mut self, cx: &mut Context<'_>) -> Poll<std::io::Result<()>>
     where
         W: AsyncWrite + Unpin,
@@ -673,7 +713,10 @@ impl<W> PutFileAuthWriter<W> {
                     )));
                 }
                 Poll::Ready(Ok(written)) => written,
-                Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
+                Poll::Ready(Err(err)) => {
+                    self.reject_server_epoch_on_conflict(&err);
+                    return Poll::Ready(Err(err));
+                }
                 Poll::Pending => return Poll::Pending,
             };
             self.trailer_offset += written;
@@ -682,6 +725,15 @@ impl<W> PutFileAuthWriter<W> {
     }
 }
 
+fn io_error_has_put_file_epoch_conflict(error: &io::Error) -> bool {
+    error
+        .get_ref()
+        .and_then(|source| source.downcast_ref::<rustfs_rio::InternodeHttpError>())
+        .is_some_and(
+            |error| matches!(error.kind(), rustfs_rio::InternodeHttpErrorKind::HttpStatus(status) if status.as_u16() == 409),
+        )
+}
+
 impl<W> AsyncWrite for PutFileAuthWriter<W>
 where
     W: AsyncWrite + Unpin,
@@ -698,12 +750,22 @@ where
                 self.hasher.update(&buf[..written]);
                 Poll::Ready(Ok(written))
             }
-            other => other,
+            Poll::Ready(Err(err)) => {
+                self.reject_server_epoch_on_conflict(&err);
+                Poll::Ready(Err(err))
+            }
+            Poll::Pending => Poll::Pending,
         }
     }
 
     fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
-        Pin::new(&mut self.inner).poll_flush(cx)
+        match Pin::new(&mut self.inner).poll_flush(cx) {
+            Poll::Ready(Err(err)) => {
+                self.reject_server_epoch_on_conflict(&err);
+                Poll::Ready(Err(err))
+            }
+            other => other,
+        }
     }
 
     fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
@@ -712,7 +774,13 @@ where
             Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
             Poll::Pending => return Poll::Pending,
         }
-        Pin::new(&mut self.inner).poll_shutdown(cx)
+        match Pin::new(&mut self.inner).poll_shutdown(cx) {
+            Poll::Ready(Err(err)) => {
+                self.reject_server_epoch_on_conflict(&err);
+                Poll::Ready(Err(err))
+            }
+            other => other,
+        }
     }
 }
 
@@ -840,7 +908,6 @@ mod tests {
             loop {
                 let strong_count = entry
                     .read()
-                    .await
                     .in_flight
                     .as_ref()
                     .map(|flight| Arc::strong_count(&flight.outcome))
@@ -858,6 +925,50 @@ mod tests {
     #[derive(Debug)]
     struct LegacyTestTransport;
 
+    #[derive(Clone, Copy, Debug)]
+    enum PutFileFailurePhase {
+        Write,
+        Flush,
+        Shutdown,
+    }
+
+    struct PutFileFailureWriter {
+        phase: PutFileFailurePhase,
+        status: reqwest::StatusCode,
+    }
+
+    impl PutFileFailureWriter {
+        fn error(&self) -> io::Error {
+            rustfs_rio::new_test_internode_http_io_error(rustfs_rio::InternodeHttpErrorKind::HttpStatus(self.status))
+        }
+    }
+
+    impl tokio::io::AsyncWrite for PutFileFailureWriter {
+        fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
+            Poll::Ready(if matches!(self.phase, PutFileFailurePhase::Write) {
+                Err(self.error())
+            } else {
+                Ok(buf.len())
+            })
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            Poll::Ready(if matches!(self.phase, PutFileFailurePhase::Flush) {
+                Err(self.error())
+            } else {
+                Ok(())
+            })
+        }
+
+        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            Poll::Ready(if matches!(self.phase, PutFileFailurePhase::Shutdown) {
+                Err(self.error())
+            } else {
+                Ok(())
+            })
+        }
+    }
+
     #[async_trait::async_trait]
     impl InternodeDataTransport for LegacyTestTransport {
         async fn open_read(&self, _request: ReadStreamRequest) -> Result<FileReader> {
@@ -1048,7 +1159,7 @@ mod tests {
         let v1_endpoint = format!("http://v1-{}.invalid", Uuid::new_v4());
         let v1_entry = put_file_capability_cache_entry(&v1_endpoint);
         let server_epoch = Uuid::new_v4();
-        v1_entry.write().await.cached = Some(PutFileCapabilityState::V1 {
+        v1_entry.write().cached = Some(PutFileCapabilityState::V1 {
             server_epoch,
             revalidate_after: Instant::now() + PUT_FILE_V1_CAPABILITY_TTL,
         });
@@ -1067,7 +1178,7 @@ mod tests {
             Some(server_epoch)
         );
         assert!(!cache_probe_called.load(Ordering::SeqCst));
-        v1_entry.write().await.cached = Some(PutFileCapabilityState::V1 {
+        v1_entry.write().cached = Some(PutFileCapabilityState::V1 {
             server_epoch,
             revalidate_after: Instant::now(),
         });
@@ -1086,8 +1197,7 @@ mod tests {
 
         let legacy_endpoint = format!("http://legacy-{}.invalid", Uuid::new_v4());
         let legacy_entry = put_file_capability_cache_entry(&legacy_endpoint);
-        legacy_entry.write().await.cached =
+        legacy_entry.write().cached = Some(PutFileCapabilityState::LegacyUntil(Instant::now() + PUT_FILE_LEGACY_CAPABILITY_TTL));
-            Some(PutFileCapabilityState::LegacyUntil(Instant::now() + PUT_FILE_LEGACY_CAPABILITY_TTL));
         assert!(
             transport
                 .put_file_auth_capability(&legacy_endpoint)
@@ -1098,7 +1208,7 @@ mod tests {
 
         let expired_endpoint = format!("http://expired-legacy-{}.invalid", Uuid::new_v4());
         let expired_entry = put_file_capability_cache_entry(&expired_endpoint);
-        expired_entry.write().await.cached = Some(PutFileCapabilityState::LegacyUntil(Instant::now()));
+        expired_entry.write().cached = Some(PutFileCapabilityState::LegacyUntil(Instant::now()));
         let reprobed = std::sync::atomic::AtomicBool::new(false);
         assert_eq!(
             resolve_put_file_auth_capability(&expired_endpoint, || async {
@@ -1349,7 +1459,7 @@ mod tests {
         };
         probe_started.notified().await;
         {
-            let mut state = entry.write().await;
+            let mut state = entry.write();
             state.generation = state.generation.checked_add(1).expect("test generation should advance");
             state.cached = Some(PutFileCapabilityState::V1 {
                 server_epoch: newer_epoch,
@@ -1362,10 +1472,7 @@ mod tests {
             task.await.expect("stale task should finish").expect("stale probe result"),
             Some(stale_epoch)
         );
-        assert_eq!(
+        assert_eq!(fresh_put_file_capability(entry.read().cached, Instant::now()), Some(Some(newer_epoch)));
-            fresh_put_file_capability(entry.read().await.cached, Instant::now()),
-            Some(Some(newer_epoch))
-        );
     }
 
     #[test]
@@ -1398,6 +1505,8 @@ mod tests {
 
         let _ = rustfs_credentials::set_global_rpc_secret("put-file-auth-writer-test-secret".to_string());
         let nonce = Uuid::parse_str("11111111-2222-4333-8444-555555555555").expect("nonce");
+        let server_epoch = Uuid::parse_str("aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee").expect("server epoch");
+        let endpoint = "http://node1:9000".to_string();
         let url = concat!(
             "http://node1:9000/rustfs/rpc/put_file_stream?disk=disk-a&volume=bucket&path=object%2Fpart.1",
             "&append=false&size=11&put_file_auth=digest-trailer-v1&put_file_nonce=11111111-2222-4333-8444-555555555555"
@@ -1406,7 +1515,7 @@ mod tests {
         let mut sink = Vec::new();
 
         {
-            let mut writer = PutFileAuthWriter::new(&mut sink, url.clone(), nonce);
+            let mut writer = PutFileAuthWriter::new(&mut sink, url.clone(), nonce, endpoint, server_epoch);
             writer.write_all(b"hello world").await.expect("body write should succeed");
             writer.shutdown().await.expect("shutdown should append auth trailer");
             let err = writer
@@ -1424,6 +1533,143 @@ mod tests {
         assert_eq!(verified, expected_digest);
     }
 
+    #[tokio::test]
+    async fn put_file_auth_writer_reprobes_after_server_epoch_conflict() {
+        use tokio::io::AsyncWriteExt;
+
+        let _ = rustfs_credentials::set_global_rpc_secret("put-file-epoch-conflict-test-secret".to_string());
+        for status in [reqwest::StatusCode::CONFLICT, reqwest::StatusCode::BAD_REQUEST] {
+            for (phase, trailer_write) in [
+                (PutFileFailurePhase::Write, false),
+                (PutFileFailurePhase::Write, true),
+                (PutFileFailurePhase::Flush, false),
+                (PutFileFailurePhase::Shutdown, false),
+            ] {
+                let endpoint = format!("http://epoch-conflict-{}.invalid", Uuid::new_v4());
+                let stale_epoch = Uuid::new_v4();
+                let replacement_epoch = Uuid::new_v4();
+                resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(stale_epoch)) })
+                    .await
+                    .expect("initial capability should resolve");
+                let mut writer = PutFileAuthWriter::new(
+                    PutFileFailureWriter { phase, status },
+                    format!("{endpoint}{PUT_FILE_AUTH_STREAM_PATH}"),
+                    Uuid::new_v4(),
+                    endpoint.clone(),
+                    stale_epoch,
+                );
+                let error = match (phase, trailer_write) {
+                    (PutFileFailurePhase::Write, false) => writer.write_all(b"body").await,
+                    (PutFileFailurePhase::Flush, _) => writer.flush().await,
+                    _ => writer.shutdown().await,
+                }
+                .expect_err("injected writer error must reach the caller");
+                let conflict = status == reqwest::StatusCode::CONFLICT;
+                assert_eq!(io_error_has_put_file_epoch_conflict(&error), conflict);
+
+                let probe_called = AtomicBool::new(false);
+                let resolved = resolve_put_file_auth_capability(&endpoint, || async {
+                    probe_called.store(true, Ordering::SeqCst);
+                    Ok(Some(replacement_epoch))
+                })
+                .await
+                .expect("capability should remain usable or be reprobed");
+                assert_eq!(probe_called.load(Ordering::SeqCst), conflict, "phase={phase:?}, trailer={trailer_write}");
+                assert_eq!(resolved, Some(if conflict { replacement_epoch } else { stale_epoch }));
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn late_put_file_epoch_rejection_preserves_current_rejection() {
+        let endpoint = format!("http://late-epoch-conflict-{}.invalid", Uuid::new_v4());
+        let old_epoch = Uuid::new_v4();
+        let current_epoch = Uuid::new_v4();
+        let replacement_epoch = Uuid::new_v4();
+        assert_eq!(
+            resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(old_epoch)) })
+                .await
+                .expect("initial epoch should be cached"),
+            Some(old_epoch)
+        );
+        reject_put_file_server_epoch(&endpoint, old_epoch);
+        assert_eq!(
+            resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(current_epoch)) })
+                .await
+                .expect("first restart should install a new epoch"),
+            Some(current_epoch)
+        );
+
+        reject_put_file_server_epoch(&endpoint, current_epoch);
+        // A writer opened before the first restart can report its 409 after
+        // a newer writer has already rejected the second server incarnation.
+        reject_put_file_server_epoch(&endpoint, old_epoch);
+        let probe_called = AtomicBool::new(false);
+        let resolved = resolve_put_file_auth_capability(&endpoint, || async {
+            probe_called.store(true, Ordering::SeqCst);
+            Ok(Some(replacement_epoch))
+        })
+        .await
+        .expect("late old-epoch rejection must preserve the current rejection");
+
+        assert!(probe_called.load(Ordering::SeqCst), "known-rejected current epoch must be reprobed");
+        assert_eq!(resolved, Some(replacement_epoch));
+    }
+
+    #[tokio::test]
+    async fn put_file_epoch_rejection_is_endpoint_and_epoch_scoped() {
+        let endpoint = format!("http://scoped-epoch-{}.invalid", Uuid::new_v4());
+        let other_endpoint = format!("http://other-epoch-{}.invalid", Uuid::new_v4());
+        let current_epoch = Uuid::new_v4();
+        for endpoint in [&endpoint, &other_endpoint] {
+            resolve_put_file_auth_capability(endpoint, || async { Ok(Some(current_epoch)) })
+                .await
+                .expect("initial epoch should resolve");
+        }
+        reject_put_file_server_epoch(&endpoint, Uuid::new_v4());
+        assert_eq!(
+            resolve_put_file_auth_capability(&endpoint, || async { panic!("old writer must not invalidate a new epoch") })
+                .await
+                .expect("new epoch must remain cached"),
+            Some(current_epoch)
+        );
+        reject_put_file_server_epoch(&endpoint, current_epoch);
+        assert_eq!(
+            resolve_put_file_auth_capability(&other_endpoint, || async { panic!("another endpoint must stay cached") })
+                .await
+                .expect("other endpoint must remain cached"),
+            Some(current_epoch)
+        );
+    }
+
+    #[tokio::test]
+    async fn put_file_rejected_epoch_survives_failed_stale_and_downgrade_probes() {
+        let endpoint = format!("http://rejected-probe-{}.invalid", Uuid::new_v4());
+        let rejected_epoch = Uuid::new_v4();
+        let replacement_epoch = Uuid::new_v4();
+        resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(rejected_epoch)) })
+            .await
+            .expect("initial epoch should resolve");
+        reject_put_file_server_epoch(&endpoint, rejected_epoch);
+        let failure = resolve_put_file_auth_capability(&endpoint, || async { Err(Error::other("injected probe failure")) })
+            .await
+            .expect_err("probe failure must be returned");
+        assert!(failure.to_string().contains("injected probe failure"));
+        let downgrade = resolve_put_file_auth_capability(&endpoint, || async { Ok(None) })
+            .await
+            .expect_err("rejection must not unpin authenticated v1");
+        assert!(downgrade.to_string().contains("downgrade rejected"));
+        resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(rejected_epoch)) })
+            .await
+            .expect("a probe racing a restart can still return the old epoch");
+        assert_eq!(
+            resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(replacement_epoch)) })
+                .await
+                .expect("same-epoch probe must not clear known rejection"),
+            Some(replacement_epoch)
+        );
+    }
+
     #[test]
     fn walk_dir_url_encodes_disk_ref() {
         let url = build_walk_dir_url(&WalkDirStreamRequest {

crates/ecstore/src/cluster/rpc/mod.rs

@@ -39,8 +39,8 @@ pub use http_auth::{
     sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
     verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
     verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
-    verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
+    verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
-    verify_tonic_rpc_signature_with_bootstrap,
+    verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
 };
 #[cfg(test)]
 pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -122,6 +122,14 @@ fn control_plane_failure(op: &str, bucket: Option<&str>, error_code: Option<i32>
     if error_code == Some(rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32) {
         return Error::RemoteNotInitialized;
     }
+    if error_code == Some(rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode::ControlPlaneErrorInvalidArgument as i32)
+    {
+        return Error::InvalidArgument(
+            "control-plane".to_string(),
+            op.to_string(),
+            error_info.unwrap_or_else(|| format!("{op}: peer rejected invalid argument without details")),
+        );
+    }
     match error_info {
         Some(msg) => Error::other(msg),
         None => peer_failure_without_details(op, bucket),
@@ -725,7 +733,7 @@ impl PeerRestClient {
     /// never take it offline no matter what its message says. The substring
     /// fallback only covers failures that exist purely as text, such as the
     /// dial errors `get_client` wraps.
-    fn is_network_like_error(err: &Error) -> bool {
+    pub(crate) fn is_network_like_error(err: &Error) -> bool {
         if let Error::Io(io_err) = err
             && let Some(status) = embedded_tonic_status(io_err)
         {
@@ -2335,6 +2343,29 @@ mod tests {
         use rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode;
         assert_eq!(ControlPlaneErrorCode::ControlPlaneErrorUnspecified as i32, 0);
         assert_eq!(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32, 1);
+        assert_eq!(ControlPlaneErrorCode::ControlPlaneErrorInvalidArgument as i32, 2);
+    }
+
+    #[test]
+    fn control_plane_failure_preserves_typed_invalid_argument_reason() {
+        use rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode;
+
+        let reason = "durable unresolved-entry recovery requires pool metadata V2 or V3";
+        let err = control_plane_failure(
+            "start_decommission",
+            None,
+            Some(ControlPlaneErrorCode::ControlPlaneErrorInvalidArgument as i32),
+            Some(reason.to_string()),
+        );
+
+        assert!(
+            matches!(
+                err,
+                Error::InvalidArgument(ref scope, ref operation, ref actual_reason)
+                    if scope == "control-plane" && operation == "start_decommission" && actual_reason == reason
+            ),
+            "forwarded validation failures must remain typed and actionable"
+        );
     }
 
     #[test]

crates/ecstore/src/config/audit.rs

@@ -12,35 +12,28 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use super::target_defaults::{amqp_kvs, kafka_kvs, mysql_kvs, nats_kvs, postgres_kvs, pulsar_kvs, redis_kvs};
 use rustfs_config::audit::AUDIT_REDIS_DEFAULT_CHANNEL;
 use rustfs_config::server_config::{KV, KVS};
 use rustfs_config::{
-    AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_QUEUE_LIMIT, AMQP_ROUTING_KEY,
+    COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
-    AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY,
+    MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
-    EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_SASL_ENABLE,
+    MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, WEBHOOK_AUTH_TOKEN,
-    KAFKA_SASL_MECHANISM, KAFKA_SASL_PASSWORD, KAFKA_SASL_USERNAME, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY,
+    WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT,
-    KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
+    WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
-    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
-    MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT,
-    MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT,
-    MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS,
-    NATS_JETSTREAM_ACK_TIMEOUT_SECS, NATS_JETSTREAM_ENABLE, NATS_JETSTREAM_STREAM_NAME, NATS_PASSWORD, NATS_QUEUE_DIR,
-    NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
-    NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
-    POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
-    PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
-    PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
-    REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD,
-    REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT,
-    REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL,
-    REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
-    WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL,
-    WEBHOOK_SKIP_TLS_VERIFY,
 };
 use std::sync::LazyLock;
 
 #[allow(clippy::declare_interior_mutable_const)]
 /// Default KVS for audit webhook settings.
+///
+/// `WEBHOOK_BATCH_SIZE`/`WEBHOOK_MAX_RETRY`/`WEBHOOK_RETRY_INTERVAL`/`WEBHOOK_HTTP_TIMEOUT`
+/// exist here but not in [`crate::config::notify::DEFAULT_NOTIFY_WEBHOOK_KVS`]. This mirrors
+/// MinIO upstream: `internal/logger/config.go`'s `DefaultAuditWebhookKVS` carries the same
+/// four keys with the same defaults (`"1"`/`"0"`/`"3s"`/`"5s"`), while
+/// `internal/config/notify/parse.go`'s `DefaultWebhookKVS` (bucket event notifications) does
+/// not — the notify webhook delivery path never supported them. Not a copy/paste gap
+/// (backlog#2054).
 pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
     KVS(vec![
         KV {
@@ -56,7 +49,7 @@ pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
         KV {
             key: WEBHOOK_AUTH_TOKEN.to_owned(),
             value: "".to_owned(),
-            hidden_if_empty: false,
+            hidden_if_empty: true, // Sensitive field; matches notify's webhook auth_token (backlog#2054)
         },
         KV {
             key: WEBHOOK_CLIENT_CERT.to_owned(),
@@ -118,6 +111,15 @@ pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
 
 #[allow(clippy::declare_interior_mutable_const)]
 /// Default KVS for audit MQTT settings.
+///
+/// `MQTT_QOS`/`MQTT_KEEP_ALIVE_INTERVAL`/`MQTT_RECONNECT_INTERVAL` default to a stronger
+/// delivery posture here (`"1"`/`"60s"`/`"5s"`) than
+/// [`crate::config::notify::DEFAULT_NOTIFY_MQTT_KVS`] (`"0"`/`"0s"`/`"0s"`, which matches
+/// MinIO's own `DefaultMQTTKVS` in `internal/config/notify/parse.go` byte-for-byte). MinIO has
+/// no MQTT audit target to compare against — audit-over-MQTT is a RustFS-original addition —
+/// so this divergence cannot be checked against upstream; it is intentional (audit favors
+/// at-least-once delivery and faster reconnect over notify's opt-in defaults), not a
+/// copy/paste gap (backlog#2054).
 pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
     KVS(vec![
         KV {
@@ -208,542 +210,18 @@ pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
     ])
 });
 
-pub static DEFAULT_AUDIT_AMQP_KVS: LazyLock<KVS> = LazyLock::new(|| {
+// The remaining targets declare the same defaults as notify, so both sides build them from
-    KVS(vec![
+// `target_defaults`. Redis and mysql pass in the single default that audit and notify disagree on.
-        KV {
+pub static DEFAULT_AUDIT_AMQP_KVS: LazyLock<KVS> = LazyLock::new(amqp_kvs);
-            key: ENABLE_KEY.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: AMQP_URL.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: AMQP_EXCHANGE.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: AMQP_ROUTING_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: AMQP_MANDATORY.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: AMQP_PERSISTENT.to_owned(),
-            value: EnableState::On.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: AMQP_USERNAME.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: AMQP_PASSWORD.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: AMQP_TLS_CA.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: AMQP_TLS_CLIENT_CERT.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: AMQP_TLS_CLIENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: AMQP_QUEUE_DIR.to_owned(),
-            value: EVENT_DEFAULT_DIR.to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: AMQP_QUEUE_LIMIT.to_owned(),
-            value: DEFAULT_LIMIT.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: COMMENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-    ])
-});
 
-pub static DEFAULT_AUDIT_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
+pub static DEFAULT_AUDIT_NATS_KVS: LazyLock<KVS> = LazyLock::new(nats_kvs);
-    KVS(vec![
-        KV {
-            key: ENABLE_KEY.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_ADDRESS.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_SUBJECT.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_USERNAME.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_PASSWORD.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: NATS_TOKEN.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: NATS_CREDENTIALS_FILE.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: NATS_TLS_CA.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: NATS_TLS_CLIENT_CERT.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: NATS_TLS_CLIENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: NATS_TLS_REQUIRED.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_QUEUE_DIR.to_owned(),
-            value: EVENT_DEFAULT_DIR.to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_QUEUE_LIMIT.to_owned(),
-            value: DEFAULT_LIMIT.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_JETSTREAM_ENABLE.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_JETSTREAM_STREAM_NAME.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_JETSTREAM_ACK_TIMEOUT_SECS.to_owned(),
-            value: NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: COMMENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-    ])
-});
 
-pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
+pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(pulsar_kvs);
-    KVS(vec![
-        KV {
-            key: ENABLE_KEY.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: PULSAR_BROKER.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: PULSAR_TOPIC.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: PULSAR_AUTH_TOKEN.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: PULSAR_USERNAME.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: PULSAR_PASSWORD.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: PULSAR_TLS_CA.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: PULSAR_TLS_ALLOW_INSECURE.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: PULSAR_TLS_HOSTNAME_VERIFICATION.to_owned(),
-            value: EnableState::On.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: PULSAR_QUEUE_DIR.to_owned(),
-            value: EVENT_DEFAULT_DIR.to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: PULSAR_QUEUE_LIMIT.to_owned(),
-            value: DEFAULT_LIMIT.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: COMMENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-    ])
-});
 
-pub static DEFAULT_AUDIT_REDIS_KVS: LazyLock<KVS> = LazyLock::new(|| {
+pub static DEFAULT_AUDIT_REDIS_KVS: LazyLock<KVS> = LazyLock::new(|| redis_kvs(AUDIT_REDIS_DEFAULT_CHANNEL));
-    KVS(vec![
-        KV {
-            key: ENABLE_KEY.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_URL.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_CHANNEL.to_owned(),
-            value: AUDIT_REDIS_DEFAULT_CHANNEL.to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_USERNAME.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_PASSWORD.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: REDIS_KEEP_ALIVE_INTERVAL.to_owned(),
-            value: "15".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_QUEUE_DIR.to_owned(),
-            value: EVENT_DEFAULT_DIR.to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_QUEUE_LIMIT.to_owned(),
-            value: DEFAULT_LIMIT.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_MAX_RETRY_ATTEMPTS.to_owned(),
-            value: "3".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_RECONNECT_RETRY_ATTEMPTS.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_MIN_RETRY_DELAY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_MAX_RETRY_DELAY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_CONNECTION_TIMEOUT.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_RESPONSE_TIMEOUT.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_PIPELINE_BUFFER_SIZE.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_TLS_POLICY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: REDIS_TLS_CA.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: REDIS_TLS_CLIENT_CERT.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: REDIS_TLS_CLIENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: REDIS_TLS_ALLOW_INSECURE.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: COMMENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-    ])
-});
 
-pub static DEFAULT_AUDIT_POSTGRES_KVS: LazyLock<KVS> = LazyLock::new(|| {
+pub static DEFAULT_AUDIT_POSTGRES_KVS: LazyLock<KVS> = LazyLock::new(postgres_kvs);
-    KVS(vec![
-        KV {
-            key: ENABLE_KEY.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: POSTGRES_DSN_STRING.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: POSTGRES_TABLE.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: POSTGRES_FORMAT.to_owned(),
-            value: "namespace".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: POSTGRES_TLS_REQUIRED.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: POSTGRES_TLS_CA.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: POSTGRES_TLS_CLIENT_CERT.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: POSTGRES_TLS_CLIENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: POSTGRES_QUEUE_DIR.to_owned(),
-            value: EVENT_DEFAULT_DIR.to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: POSTGRES_QUEUE_LIMIT.to_owned(),
-            value: DEFAULT_LIMIT.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: COMMENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-    ])
-});
 
-pub static DEFAULT_AUDIT_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
+pub static DEFAULT_AUDIT_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(kafka_kvs);
-    KVS(vec![
-        KV {
-            key: ENABLE_KEY.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: KAFKA_BROKERS.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: KAFKA_TOPIC.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: KAFKA_ACKS.to_owned(),
-            value: "1".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: KAFKA_TLS_ENABLE.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: KAFKA_TLS_CA.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: KAFKA_TLS_CLIENT_CERT.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: KAFKA_TLS_CLIENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: KAFKA_SASL_ENABLE.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: KAFKA_SASL_MECHANISM.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: KAFKA_SASL_USERNAME.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: KAFKA_SASL_PASSWORD.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: KAFKA_QUEUE_DIR.to_owned(),
-            value: EVENT_DEFAULT_DIR.to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: KAFKA_QUEUE_LIMIT.to_owned(),
-            value: DEFAULT_LIMIT.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: COMMENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-    ])
-});
 
-pub static DEFAULT_AUDIT_MYSQL_KVS: LazyLock<KVS> = LazyLock::new(|| {
+pub static DEFAULT_AUDIT_MYSQL_KVS: LazyLock<KVS> = LazyLock::new(|| mysql_kvs("rustfs_audit_logs"));
-    KVS(vec![
-        KV {
-            key: ENABLE_KEY.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: MYSQL_DSN_STRING.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: MYSQL_TABLE.to_owned(),
-            value: "rustfs_audit_logs".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: MYSQL_FORMAT.to_owned(),
-            value: "access".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: MYSQL_TLS_CA.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: MYSQL_TLS_CLIENT_CERT.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: MYSQL_TLS_CLIENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: MYSQL_QUEUE_DIR.to_owned(),
-            value: EVENT_DEFAULT_DIR.to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: MYSQL_QUEUE_LIMIT.to_owned(),
-            value: DEFAULT_LIMIT.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: MYSQL_MAX_OPEN_CONNECTIONS.to_owned(),
-            value: "2".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: COMMENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-    ])
-});

crates/ecstore/src/config/com.rs

@@ -800,11 +800,22 @@ where
             if log_error {
                 error!("save_config_with_opts: err: {:?}, file: {}", err, file);
             }
-            Err(err)
+            Err(map_system_metadata_write_error(err, file))
         }
     }
 }
 
+/// A system metadata volume outage must remain retryable instead of being
+/// exposed as the user-facing bucket-not-found response.
+pub(crate) fn map_system_metadata_write_error(err: Error, file: &str) -> Error {
+    match err {
+        Error::BucketNotFound(_) | Error::VolumeNotFound => {
+            Error::InsufficientWriteQuorum(RUSTFS_META_BUCKET.to_string(), file.to_string())
+        }
+        other => other,
+    }
+}
+
 fn new_server_config() -> Config {
     Config::new()
 }
@@ -2361,6 +2372,7 @@ where
         scan_mode: HealScanMode::Deep,
         update_parity: false,
         no_lock: false,
+        read_repair: false,
         pool: None,
         set: None,
     };
@@ -2795,14 +2807,14 @@ mod tests {
     use super::{
         SERVER_CONFIG_LOCK, ServerConfigSnapshot, apply_dynamic_config_for_sub_sys_with, build_scalar_config_object,
         config_task_join_error, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob,
-        heal_config_descriptor, is_standard_object_server_config, lookup_configs, new_and_save_server_config, read_config,
+        heal_config_descriptor, is_standard_object_server_config, lookup_configs, map_system_metadata_write_error,
-        read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty, read_config_with_metadata,
+        new_and_save_server_config, read_config, read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty,
-        read_config_without_migrate, read_server_config_snapshot, save_server_config, save_server_config_snapshot,
+        read_config_with_metadata, read_config_without_migrate, read_server_config_snapshot, save_config_with_opts_inner,
-        save_server_config_snapshot_with_generation, server_config_transaction_lock_path, should_warn_ignored_scalar_section,
+        save_server_config, save_server_config_snapshot, save_server_config_snapshot_with_generation,
-        storage_class_kvs_mut,
+        server_config_transaction_lock_path, should_warn_ignored_scalar_section, storage_class_kvs_mut,
     };
     use crate::config::{audit, heal, notify, oidc, scanner};
-    use crate::disk::endpoint::Endpoint;
+    use crate::disk::{RUSTFS_META_BUCKET, endpoint::Endpoint};
     use crate::error::{Error, Result};
     use crate::layout::endpoints::SetupType;
     use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
@@ -2834,6 +2846,72 @@ mod tests {
         assert!(rendered.contains("panicked"));
         assert!(!rendered.contains("do-not-expose-payload"));
     }
+
+    #[test]
+    fn system_metadata_volume_failures_map_to_retryable_write_errors() {
+        for error in [Error::VolumeNotFound, Error::BucketNotFound(RUSTFS_META_BUCKET.to_string())] {
+            assert_eq!(
+                map_system_metadata_write_error(error, "buckets/example/.metadata.bin"),
+                Error::InsufficientWriteQuorum(RUSTFS_META_BUCKET.to_string(), "buckets/example/.metadata.bin".to_string())
+            );
+        }
+
+        let other = Error::other("metadata encoding failed");
+        assert_eq!(map_system_metadata_write_error(other.clone(), "buckets/example/.metadata.bin"), other);
+    }
+
+    #[derive(Debug, Default)]
+    struct MetadataWriteStore {
+        error: Option<Error>,
+    }
+
+    #[async_trait::async_trait]
+    impl crate::storage_api_contracts::object::ObjectIO for MetadataWriteStore {
+        type Error = Error;
+        type RangeSpec = HTTPRangeSpec;
+        type HeaderMap = HeaderMap;
+        type ObjectOptions = ObjectOptions;
+        type ObjectInfo = ObjectInfo;
+        type GetObjectReader = GetObjectReader;
+        type PutObjectReader = PutObjReader;
+
+        async fn get_object_reader(
+            &self,
+            _bucket: &str,
+            _object: &str,
+            _range: Option<Self::RangeSpec>,
+            _headers: Self::HeaderMap,
+            _opts: &Self::ObjectOptions,
+        ) -> core::result::Result<Self::GetObjectReader, Self::Error> {
+            Err(Error::FileNotFound)
+        }
+
+        async fn put_object(
+            &self,
+            _bucket: &str,
+            _object: &str,
+            _data: &mut Self::PutObjectReader,
+            _opts: &Self::ObjectOptions,
+        ) -> core::result::Result<Self::ObjectInfo, Self::Error> {
+            Err(self.error.clone().expect("test store error should be configured"))
+        }
+    }
+
+    #[tokio::test]
+    async fn save_config_preserves_retryable_system_volume_errors() {
+        let store = Arc::new(MetadataWriteStore {
+            error: Some(Error::BucketNotFound(RUSTFS_META_BUCKET.to_string())),
+        });
+        let error =
+            save_config_with_opts_inner(store, "buckets/example/.metadata.bin", Vec::new(), &ObjectOptions::default(), false)
+                .await
+                .expect_err("missing metadata volume must fail");
+
+        assert_eq!(
+            error,
+            Error::InsufficientWriteQuorum(RUSTFS_META_BUCKET.to_string(), "buckets/example/.metadata.bin".to_string())
+        );
+    }
     use rustfs_lock::client::LockClient;
     use rustfs_lock::client::local::LocalClient;
     use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats};

crates/ecstore/src/config/mod.rs

@@ -21,6 +21,7 @@ mod notify;
 mod oidc;
 mod scanner;
 pub mod storageclass;
+mod target_defaults;
 
 use crate::error::Result;
 use crate::store::ECStore;

crates/ecstore/src/config/notify.rs

@@ -12,34 +12,26 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use super::target_defaults::{amqp_kvs, kafka_kvs, mysql_kvs, nats_kvs, postgres_kvs, pulsar_kvs, redis_kvs};
 use rustfs_config::notify::NOTIFY_REDIS_DEFAULT_CHANNEL;
 use rustfs_config::server_config::{KV, KVS};
 use rustfs_config::{
-    AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_QUEUE_LIMIT, AMQP_ROUTING_KEY,
+    COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
-    AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY,
+    MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
-    EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_SASL_ENABLE,
+    MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, WEBHOOK_AUTH_TOKEN,
-    KAFKA_SASL_MECHANISM, KAFKA_SASL_PASSWORD, KAFKA_SASL_USERNAME, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY,
+    WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
-    KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
+    WEBHOOK_SKIP_TLS_VERIFY,
-    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
-    MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT,
-    MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT,
-    MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS,
-    NATS_JETSTREAM_ACK_TIMEOUT_SECS, NATS_JETSTREAM_ENABLE, NATS_JETSTREAM_STREAM_NAME, NATS_PASSWORD, NATS_QUEUE_DIR,
-    NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
-    NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
-    POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
-    PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
-    PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
-    REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD,
-    REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT,
-    REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL,
-    REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT,
-    WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use std::sync::LazyLock;
 
 /// The default configuration collection of webhooks，
 /// Initialized only once during the program life cycle, enabling high-performance lazy loading.
+///
+/// This table has no `batch_size`/`max_retry`/`retry_interval`/`http_timeout` keys, unlike
+/// [`crate::config::audit::DEFAULT_AUDIT_WEBHOOK_KVS`] — matching MinIO upstream, whose
+/// `internal/config/notify/parse.go` `DefaultWebhookKVS` (bucket event notifications) also
+/// omits them while `internal/logger/config.go`'s `DefaultAuditWebhookKVS` carries them.
+/// Intentional, not a copy/paste gap (backlog#2054).
 pub static DEFAULT_NOTIFY_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
     KVS(vec![
         KV {
@@ -97,6 +89,12 @@ pub static DEFAULT_NOTIFY_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
 });
 
 /// MQTT's default configuration collection
+///
+/// `MQTT_QOS`/`MQTT_KEEP_ALIVE_INTERVAL`/`MQTT_RECONNECT_INTERVAL` default to `"0"`/`"0s"`/`"0s"`
+/// here, matching MinIO's `DefaultMQTTKVS` in `internal/config/notify/parse.go`
+/// byte-for-byte — this table is a faithful port. [`crate::config::audit::DEFAULT_AUDIT_MQTT_KVS`]
+/// uses stronger, RustFS-original defaults instead (MinIO has no MQTT audit target to compare
+/// against); that divergence is intentional, not a copy/paste gap (backlog#2054).
 pub static DEFAULT_NOTIFY_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
     KVS(vec![
         KV {
@@ -188,543 +186,17 @@ pub static DEFAULT_NOTIFY_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
     ])
 });
 
-pub static DEFAULT_NOTIFY_AMQP_KVS: LazyLock<KVS> = LazyLock::new(|| {
+pub static DEFAULT_NOTIFY_AMQP_KVS: LazyLock<KVS> = LazyLock::new(amqp_kvs);
-    KVS(vec![
-        KV {
-            key: ENABLE_KEY.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: AMQP_URL.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: AMQP_EXCHANGE.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: AMQP_ROUTING_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: AMQP_MANDATORY.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: AMQP_PERSISTENT.to_owned(),
-            value: EnableState::On.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: AMQP_USERNAME.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: AMQP_PASSWORD.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: AMQP_TLS_CA.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: AMQP_TLS_CLIENT_CERT.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: AMQP_TLS_CLIENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: AMQP_QUEUE_DIR.to_owned(),
-            value: EVENT_DEFAULT_DIR.to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: AMQP_QUEUE_LIMIT.to_owned(),
-            value: DEFAULT_LIMIT.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: COMMENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-    ])
-});
 
-pub static DEFAULT_NOTIFY_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
+pub static DEFAULT_NOTIFY_NATS_KVS: LazyLock<KVS> = LazyLock::new(nats_kvs);
-    KVS(vec![
-        KV {
-            key: ENABLE_KEY.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_ADDRESS.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_SUBJECT.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_USERNAME.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_PASSWORD.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: NATS_TOKEN.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: NATS_CREDENTIALS_FILE.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: NATS_TLS_CA.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: NATS_TLS_CLIENT_CERT.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: NATS_TLS_CLIENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: NATS_TLS_REQUIRED.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_QUEUE_DIR.to_owned(),
-            value: EVENT_DEFAULT_DIR.to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_QUEUE_LIMIT.to_owned(),
-            value: DEFAULT_LIMIT.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_JETSTREAM_ENABLE.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_JETSTREAM_STREAM_NAME.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: NATS_JETSTREAM_ACK_TIMEOUT_SECS.to_owned(),
-            value: NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: COMMENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-    ])
-});
 
-pub static DEFAULT_NOTIFY_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
+pub static DEFAULT_NOTIFY_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(pulsar_kvs);
-    KVS(vec![
-        KV {
-            key: ENABLE_KEY.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: PULSAR_BROKER.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: PULSAR_TOPIC.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: PULSAR_AUTH_TOKEN.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: PULSAR_USERNAME.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: PULSAR_PASSWORD.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: PULSAR_TLS_CA.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: PULSAR_TLS_ALLOW_INSECURE.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: PULSAR_TLS_HOSTNAME_VERIFICATION.to_owned(),
-            value: EnableState::On.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: PULSAR_QUEUE_DIR.to_owned(),
-            value: EVENT_DEFAULT_DIR.to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: PULSAR_QUEUE_LIMIT.to_owned(),
-            value: DEFAULT_LIMIT.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: COMMENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-    ])
-});
 
-pub static DEFAULT_NOTIFY_REDIS_KVS: LazyLock<KVS> = LazyLock::new(|| {
+pub static DEFAULT_NOTIFY_REDIS_KVS: LazyLock<KVS> = LazyLock::new(|| redis_kvs(NOTIFY_REDIS_DEFAULT_CHANNEL));
-    KVS(vec![
-        KV {
-            key: ENABLE_KEY.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_URL.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_CHANNEL.to_owned(),
-            value: NOTIFY_REDIS_DEFAULT_CHANNEL.to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_USERNAME.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_PASSWORD.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: REDIS_KEEP_ALIVE_INTERVAL.to_owned(),
-            value: "15".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_QUEUE_DIR.to_owned(),
-            value: EVENT_DEFAULT_DIR.to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_QUEUE_LIMIT.to_owned(),
-            value: DEFAULT_LIMIT.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_MAX_RETRY_ATTEMPTS.to_owned(),
-            value: "3".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_RECONNECT_RETRY_ATTEMPTS.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_MIN_RETRY_DELAY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_MAX_RETRY_DELAY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_CONNECTION_TIMEOUT.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_RESPONSE_TIMEOUT.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_PIPELINE_BUFFER_SIZE.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: REDIS_TLS_POLICY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: REDIS_TLS_CA.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: REDIS_TLS_CLIENT_CERT.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: REDIS_TLS_CLIENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: REDIS_TLS_ALLOW_INSECURE.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: COMMENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-    ])
-});
 
-pub static DEFAULT_NOTIFY_POSTGRES_KVS: LazyLock<KVS> = LazyLock::new(|| {
+pub static DEFAULT_NOTIFY_POSTGRES_KVS: LazyLock<KVS> = LazyLock::new(postgres_kvs);
-    KVS(vec![
-        KV {
-            key: ENABLE_KEY.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: POSTGRES_DSN_STRING.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: POSTGRES_TABLE.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: POSTGRES_FORMAT.to_owned(),
-            value: "namespace".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: POSTGRES_TLS_REQUIRED.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: POSTGRES_TLS_CA.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: POSTGRES_TLS_CLIENT_CERT.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: POSTGRES_TLS_CLIENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: POSTGRES_QUEUE_DIR.to_owned(),
-            value: EVENT_DEFAULT_DIR.to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: POSTGRES_QUEUE_LIMIT.to_owned(),
-            value: DEFAULT_LIMIT.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: COMMENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-    ])
-});
 
-pub static DEFAULT_NOTIFY_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
+pub static DEFAULT_NOTIFY_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(kafka_kvs);
-    KVS(vec![
-        KV {
-            key: ENABLE_KEY.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: KAFKA_BROKERS.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: KAFKA_TOPIC.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: KAFKA_ACKS.to_owned(),
-            value: "1".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: KAFKA_TLS_ENABLE.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: KAFKA_TLS_CA.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: KAFKA_TLS_CLIENT_CERT.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: KAFKA_TLS_CLIENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: KAFKA_SASL_ENABLE.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: KAFKA_SASL_MECHANISM.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: KAFKA_SASL_USERNAME.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: KAFKA_SASL_PASSWORD.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: KAFKA_QUEUE_DIR.to_owned(),
-            value: EVENT_DEFAULT_DIR.to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: KAFKA_QUEUE_LIMIT.to_owned(),
-            value: DEFAULT_LIMIT.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: COMMENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-    ])
-});
 
 /// MySQL notification target default configuration
-pub static DEFAULT_NOTIFY_MYSQL_KVS: LazyLock<KVS> = LazyLock::new(|| {
+pub static DEFAULT_NOTIFY_MYSQL_KVS: LazyLock<KVS> = LazyLock::new(|| mysql_kvs("rustfs_events"));
-    KVS(vec![
-        KV {
-            key: ENABLE_KEY.to_owned(),
-            value: EnableState::Off.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: MYSQL_DSN_STRING.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: MYSQL_TABLE.to_owned(),
-            value: "rustfs_events".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: MYSQL_FORMAT.to_owned(),
-            value: "access".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: MYSQL_TLS_CA.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: MYSQL_TLS_CLIENT_CERT.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: MYSQL_TLS_CLIENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: true,
-        },
-        KV {
-            key: MYSQL_QUEUE_DIR.to_owned(),
-            value: EVENT_DEFAULT_DIR.to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: MYSQL_QUEUE_LIMIT.to_owned(),
-            value: DEFAULT_LIMIT.to_string(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: MYSQL_MAX_OPEN_CONNECTIONS.to_owned(),
-            value: "2".to_owned(),
-            hidden_if_empty: false,
-        },
-        KV {
-            key: COMMENT_KEY.to_owned(),
-            value: "".to_owned(),
-            hidden_if_empty: false,
-        },
-    ])
-});

crates/ecstore/src/config/storageclass.rs

@@ -246,16 +246,7 @@ impl Config {
         }
 
         let shard_size = shard_size as usize;
-        // Keep the historical two-data-shard object budget while preventing
+        let inline_block = self.effective_inline_block(data_shards);
-        // wider EC layouts from multiplying the maximum inline object size.
-        // Use div_ceil to match the shard_file_size calculation (which also uses
-        // div_ceil), avoiding a 1-byte rounding discrepancy that prevents inline
-        // for objects right at the threshold.
-        let inline_block = if self.initialized && self.inline_block_explicit {
-            self.inline_block
-        } else {
-            DEFAULT_INLINE_OBJECT_BUDGET.div_ceil(data_shards).min(DEFAULT_INLINE_BLOCK)
-        };
 
         if versioned {
             shard_size <= inline_block / 8
@@ -264,6 +255,27 @@ impl Config {
         }
     }
 
+    /// Returns the per-shard inline budget used by both write admission and
+    /// legacy read fallback.
+    ///
+    /// The default budget is scaled by the number of data shards so a wider EC
+    /// layout does not silently increase the maximum inline object size. An
+    /// explicitly configured `inline_block` remains a fixed per-shard limit for
+    /// compatibility with deployments that opted into the historical policy.
+    pub(crate) fn effective_inline_block(&self, data_shards: usize) -> usize {
+        if data_shards == 0 {
+            return 0;
+        }
+
+        if self.initialized && self.inline_block_explicit {
+            self.inline_block
+        } else {
+            // Keep the historical two-data-shard object budget while preventing
+            // wider EC layouts from multiplying the maximum inline object size.
+            DEFAULT_INLINE_OBJECT_BUDGET.div_ceil(data_shards).min(DEFAULT_INLINE_BLOCK)
+        }
+    }
+
     pub fn inline_block(&self) -> usize {
         if !self.initialized {
             DEFAULT_INLINE_BLOCK
@@ -602,6 +614,51 @@ mod tests {
         }
     }
 
+    #[test]
+    fn should_inline_keeps_ec8_and_ec12_object_boundaries_consistent() {
+        let config = Config::default();
+        let object_sizes = [128 * 1024, 256 * 1024, 512 * 1024, 1024 * 1024, 4 * 1024 * 1024];
+
+        for (data_shards, parity_shards) in [(8, 4), (12, 4)] {
+            let erasure = crate::erasure::coding::Erasure::new(data_shards, parity_shards, 1024 * 1024);
+            let mut previous = true;
+            for object_size in object_sizes {
+                let shard_size = erasure.shard_file_size(object_size);
+                let inline = config.should_inline(shard_size, data_shards, false);
+
+                // The effective policy is monotonic across object sizes. This
+                // table covers the boundaries that previously exposed the
+                // fixed-shard read-ahead mismatch, including the 1 MiB case.
+                assert!(!inline || previous, "inline decision must not re-enable at {object_size} bytes");
+                previous = inline;
+            }
+
+            assert!(
+                !config.should_inline(erasure.shard_file_size(1024 * 1024), data_shards, false),
+                "1 MiB must use the non-inline path for EC{data_shards}+{parity_shards}"
+            );
+        }
+    }
+
+    #[test]
+    fn effective_inline_block_scales_default_budget_and_preserves_explicit_limit() {
+        let config = Config::default();
+        assert_eq!(config.effective_inline_block(8), 32 * 1024);
+        assert_eq!(config.effective_inline_block(12), 21_846);
+        assert_eq!(config.effective_inline_block(0), 0);
+
+        let explicit = lookup_config_for_pools_with_env(
+            &KVS::new(),
+            &[12],
+            StorageClassEnvOverrides {
+                inline_block: Some("128KiB".to_string()),
+                ..Default::default()
+            },
+        )
+        .expect("explicit inline block should resolve");
+        assert_eq!(explicit.effective_inline_block(12), 128 * 1024);
+    }
+
     #[test]
     fn explicit_inline_block_preserves_fixed_per_shard_rollback() {
         let overrides = StorageClassEnvOverrides {

crates/ecstore/src/config/target_defaults.rs

@@ -0,0 +1,414 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Shared default KVS tables for delivery targets that audit and notify declare identically.
+//!
+//! The audit and notify subsystems register one default KVS per delivery target. For amqp, nats,
+//! pulsar, postgres and kafka both sides declare byte-identical tables; for redis and mysql they
+//! differ only in a single default literal, which the caller passes in.
+//!
+//! Webhook and mqtt are deliberately absent: audit's webhook table carries extra batching/retry
+//! keys and both tables disagree on key order and on several defaults (mqtt qos, keep-alive and
+//! reconnect intervals), so they are real behavioral forks, not duplication.
+//!
+//! Key order is part of the contract: it drives the order admin config output lists the keys in,
+//! so every constructor reproduces the existing order exactly.
+
+use rustfs_config::server_config::{KV, KVS};
+use rustfs_config::{
+    AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_QUEUE_LIMIT, AMQP_ROUTING_KEY,
+    AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY,
+    EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_SASL_ENABLE,
+    KAFKA_SASL_MECHANISM, KAFKA_SASL_PASSWORD, KAFKA_SASL_USERNAME, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY,
+    KAFKA_TLS_ENABLE, KAFKA_TOPIC, MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR,
+    MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS,
+    NATS_CREDENTIALS_FILE, NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS, NATS_JETSTREAM_ACK_TIMEOUT_SECS, NATS_JETSTREAM_ENABLE,
+    NATS_JETSTREAM_STREAM_NAME, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
+    NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR,
+    POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY,
+    POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT,
+    PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL,
+    REDIS_CONNECTION_TIMEOUT, REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY,
+    REDIS_PASSWORD, REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS,
+    REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY,
+    REDIS_TLS_POLICY, REDIS_URL, REDIS_USERNAME,
+};
+
+/// Builds one default entry. `hidden_if_empty` marks values the admin API elides when unset.
+fn kv(key: &str, value: impl Into<String>, hidden_if_empty: bool) -> KV {
+    KV {
+        key: key.to_owned(),
+        value: value.into(),
+        hidden_if_empty,
+    }
+}
+
+/// Default KVS for the amqp delivery target.
+// Unused until the audit/notify tables are migrated onto these constructors.
+#[allow(dead_code)]
+pub fn amqp_kvs() -> KVS {
+    KVS(vec![
+        kv(ENABLE_KEY, EnableState::Off.to_string(), false),
+        kv(AMQP_URL, "", false),
+        kv(AMQP_EXCHANGE, "", false),
+        kv(AMQP_ROUTING_KEY, "", false),
+        kv(AMQP_MANDATORY, EnableState::Off.to_string(), false),
+        kv(AMQP_PERSISTENT, EnableState::On.to_string(), false),
+        kv(AMQP_USERNAME, "", false),
+        kv(AMQP_PASSWORD, "", true),
+        kv(AMQP_TLS_CA, "", true),
+        kv(AMQP_TLS_CLIENT_CERT, "", true),
+        kv(AMQP_TLS_CLIENT_KEY, "", true),
+        kv(AMQP_QUEUE_DIR, EVENT_DEFAULT_DIR, false),
+        kv(AMQP_QUEUE_LIMIT, DEFAULT_LIMIT.to_string(), false),
+        kv(COMMENT_KEY, "", false),
+    ])
+}
+
+/// Default KVS for the nats delivery target.
+// Unused until the audit/notify tables are migrated onto these constructors.
+#[allow(dead_code)]
+pub fn nats_kvs() -> KVS {
+    KVS(vec![
+        kv(ENABLE_KEY, EnableState::Off.to_string(), false),
+        kv(NATS_ADDRESS, "", false),
+        kv(NATS_SUBJECT, "", false),
+        kv(NATS_USERNAME, "", false),
+        kv(NATS_PASSWORD, "", true),
+        kv(NATS_TOKEN, "", true),
+        kv(NATS_CREDENTIALS_FILE, "", true),
+        kv(NATS_TLS_CA, "", true),
+        kv(NATS_TLS_CLIENT_CERT, "", true),
+        kv(NATS_TLS_CLIENT_KEY, "", true),
+        kv(NATS_TLS_REQUIRED, EnableState::Off.to_string(), false),
+        kv(NATS_QUEUE_DIR, EVENT_DEFAULT_DIR, false),
+        kv(NATS_QUEUE_LIMIT, DEFAULT_LIMIT.to_string(), false),
+        kv(NATS_JETSTREAM_ENABLE, EnableState::Off.to_string(), false),
+        kv(NATS_JETSTREAM_STREAM_NAME, "", false),
+        kv(
+            NATS_JETSTREAM_ACK_TIMEOUT_SECS,
+            NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS.to_string(),
+            false,
+        ),
+        kv(COMMENT_KEY, "", false),
+    ])
+}
+
+/// Default KVS for the pulsar delivery target.
+// Unused until the audit/notify tables are migrated onto these constructors.
+#[allow(dead_code)]
+pub fn pulsar_kvs() -> KVS {
+    KVS(vec![
+        kv(ENABLE_KEY, EnableState::Off.to_string(), false),
+        kv(PULSAR_BROKER, "", false),
+        kv(PULSAR_TOPIC, "", false),
+        kv(PULSAR_AUTH_TOKEN, "", true),
+        kv(PULSAR_USERNAME, "", false),
+        kv(PULSAR_PASSWORD, "", true),
+        kv(PULSAR_TLS_CA, "", true),
+        kv(PULSAR_TLS_ALLOW_INSECURE, EnableState::Off.to_string(), false),
+        kv(PULSAR_TLS_HOSTNAME_VERIFICATION, EnableState::On.to_string(), false),
+        kv(PULSAR_QUEUE_DIR, EVENT_DEFAULT_DIR, false),
+        kv(PULSAR_QUEUE_LIMIT, DEFAULT_LIMIT.to_string(), false),
+        kv(COMMENT_KEY, "", false),
+    ])
+}
+
+/// Default KVS for the postgres delivery target.
+// Unused until the audit/notify tables are migrated onto these constructors.
+#[allow(dead_code)]
+pub fn postgres_kvs() -> KVS {
+    KVS(vec![
+        kv(ENABLE_KEY, EnableState::Off.to_string(), false),
+        kv(POSTGRES_DSN_STRING, "", true),
+        kv(POSTGRES_TABLE, "", false),
+        kv(POSTGRES_FORMAT, "namespace", false),
+        kv(POSTGRES_TLS_REQUIRED, EnableState::Off.to_string(), false),
+        kv(POSTGRES_TLS_CA, "", true),
+        kv(POSTGRES_TLS_CLIENT_CERT, "", true),
+        kv(POSTGRES_TLS_CLIENT_KEY, "", true),
+        kv(POSTGRES_QUEUE_DIR, EVENT_DEFAULT_DIR, false),
+        kv(POSTGRES_QUEUE_LIMIT, DEFAULT_LIMIT.to_string(), false),
+        kv(COMMENT_KEY, "", false),
+    ])
+}
+
+/// Default KVS for the kafka delivery target.
+// Unused until the audit/notify tables are migrated onto these constructors.
+#[allow(dead_code)]
+pub fn kafka_kvs() -> KVS {
+    KVS(vec![
+        kv(ENABLE_KEY, EnableState::Off.to_string(), false),
+        kv(KAFKA_BROKERS, "", false),
+        kv(KAFKA_TOPIC, "", false),
+        kv(KAFKA_ACKS, "1", false),
+        kv(KAFKA_TLS_ENABLE, EnableState::Off.to_string(), false),
+        kv(KAFKA_TLS_CA, "", true),
+        kv(KAFKA_TLS_CLIENT_CERT, "", true),
+        kv(KAFKA_TLS_CLIENT_KEY, "", true),
+        kv(KAFKA_SASL_ENABLE, EnableState::Off.to_string(), false),
+        kv(KAFKA_SASL_MECHANISM, "", false),
+        kv(KAFKA_SASL_USERNAME, "", false),
+        kv(KAFKA_SASL_PASSWORD, "", true),
+        kv(KAFKA_QUEUE_DIR, EVENT_DEFAULT_DIR, false),
+        kv(KAFKA_QUEUE_LIMIT, DEFAULT_LIMIT.to_string(), false),
+        kv(COMMENT_KEY, "", false),
+    ])
+}
+
+/// Default KVS for the redis delivery target. `channel` is the subsystem's default pub/sub channel,
+/// which is the only value audit and notify disagree on.
+// Unused until the audit/notify tables are migrated onto these constructors.
+#[allow(dead_code)]
+pub fn redis_kvs(channel: &str) -> KVS {
+    KVS(vec![
+        kv(ENABLE_KEY, EnableState::Off.to_string(), false),
+        kv(REDIS_URL, "", false),
+        kv(REDIS_CHANNEL, channel, false),
+        kv(REDIS_USERNAME, "", false),
+        kv(REDIS_PASSWORD, "", true),
+        kv(REDIS_KEEP_ALIVE_INTERVAL, "15", false),
+        kv(REDIS_QUEUE_DIR, EVENT_DEFAULT_DIR, false),
+        kv(REDIS_QUEUE_LIMIT, DEFAULT_LIMIT.to_string(), false),
+        kv(REDIS_MAX_RETRY_ATTEMPTS, "3", false),
+        kv(REDIS_RECONNECT_RETRY_ATTEMPTS, "", false),
+        kv(REDIS_MIN_RETRY_DELAY, "", false),
+        kv(REDIS_MAX_RETRY_DELAY, "", false),
+        kv(REDIS_CONNECTION_TIMEOUT, "", false),
+        kv(REDIS_RESPONSE_TIMEOUT, "", false),
+        kv(REDIS_PIPELINE_BUFFER_SIZE, "", false),
+        kv(REDIS_TLS_POLICY, "", true),
+        kv(REDIS_TLS_CA, "", true),
+        kv(REDIS_TLS_CLIENT_CERT, "", true),
+        kv(REDIS_TLS_CLIENT_KEY, "", true),
+        kv(REDIS_TLS_ALLOW_INSECURE, EnableState::Off.to_string(), false),
+        kv(COMMENT_KEY, "", false),
+    ])
+}
+
+/// Default KVS for the mysql delivery target. `table` is the subsystem's default destination table,
+/// which is the only value audit and notify disagree on.
+// Unused until the audit/notify tables are migrated onto these constructors.
+#[allow(dead_code)]
+pub fn mysql_kvs(table: &str) -> KVS {
+    KVS(vec![
+        kv(ENABLE_KEY, EnableState::Off.to_string(), false),
+        kv(MYSQL_DSN_STRING, "", true),
+        kv(MYSQL_TABLE, table, false),
+        kv(MYSQL_FORMAT, "access", false),
+        kv(MYSQL_TLS_CA, "", true),
+        kv(MYSQL_TLS_CLIENT_CERT, "", true),
+        kv(MYSQL_TLS_CLIENT_KEY, "", true),
+        kv(MYSQL_QUEUE_DIR, EVENT_DEFAULT_DIR, false),
+        kv(MYSQL_QUEUE_LIMIT, DEFAULT_LIMIT.to_string(), false),
+        kv(MYSQL_MAX_OPEN_CONNECTIONS, "2", false),
+        kv(COMMENT_KEY, "", false),
+    ])
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Expected values are spelled out as literals on purpose: they mirror the tables currently
+    /// declared in `audit.rs` and `notify.rs`, so a drift in key order or in any default breaks
+    /// the test instead of silently changing admin config output.
+    fn assert_table(actual: &KVS, expected: &[(&str, &str, bool)]) {
+        let actual: Vec<(&str, &str, bool)> = actual
+            .0
+            .iter()
+            .map(|kv| (kv.key.as_str(), kv.value.as_str(), kv.hidden_if_empty))
+            .collect();
+        assert_eq!(actual, expected);
+    }
+
+    const QUEUE_DIR: &str = "/opt/rustfs/events";
+    const QUEUE_LIMIT: &str = "100000";
+
+    #[test]
+    fn amqp_table_matches_audit_and_notify() {
+        assert_table(
+            &amqp_kvs(),
+            &[
+                ("enable", "off", false),
+                ("url", "", false),
+                ("exchange", "", false),
+                ("routing_key", "", false),
+                ("mandatory", "off", false),
+                ("persistent", "on", false),
+                ("username", "", false),
+                ("password", "", true),
+                ("tls_ca", "", true),
+                ("tls_client_cert", "", true),
+                ("tls_client_key", "", true),
+                ("queue_dir", QUEUE_DIR, false),
+                ("queue_limit", QUEUE_LIMIT, false),
+                ("comment", "", false),
+            ],
+        );
+    }
+
+    #[test]
+    fn nats_table_matches_audit_and_notify() {
+        assert_table(
+            &nats_kvs(),
+            &[
+                ("enable", "off", false),
+                ("address", "", false),
+                ("subject", "", false),
+                ("username", "", false),
+                ("password", "", true),
+                ("token", "", true),
+                ("credentials_file", "", true),
+                ("tls_ca", "", true),
+                ("tls_client_cert", "", true),
+                ("tls_client_key", "", true),
+                ("tls_required", "off", false),
+                ("queue_dir", QUEUE_DIR, false),
+                ("queue_limit", QUEUE_LIMIT, false),
+                ("jetstream_enable", "off", false),
+                ("jetstream_stream_name", "", false),
+                ("jetstream_ack_timeout_secs", "30", false),
+                ("comment", "", false),
+            ],
+        );
+    }
+
+    #[test]
+    fn pulsar_table_matches_audit_and_notify() {
+        assert_table(
+            &pulsar_kvs(),
+            &[
+                ("enable", "off", false),
+                ("broker", "", false),
+                ("topic", "", false),
+                ("auth_token", "", true),
+                ("username", "", false),
+                ("password", "", true),
+                ("tls_ca", "", true),
+                ("tls_allow_insecure", "off", false),
+                ("tls_hostname_verification", "on", false),
+                ("queue_dir", QUEUE_DIR, false),
+                ("queue_limit", QUEUE_LIMIT, false),
+                ("comment", "", false),
+            ],
+        );
+    }
+
+    #[test]
+    fn postgres_table_matches_audit_and_notify() {
+        assert_table(
+            &postgres_kvs(),
+            &[
+                ("enable", "off", false),
+                ("dsn_string", "", true),
+                ("table", "", false),
+                ("format", "namespace", false),
+                ("tls_required", "off", false),
+                ("tls_ca", "", true),
+                ("tls_client_cert", "", true),
+                ("tls_client_key", "", true),
+                ("queue_dir", QUEUE_DIR, false),
+                ("queue_limit", QUEUE_LIMIT, false),
+                ("comment", "", false),
+            ],
+        );
+    }
+
+    #[test]
+    fn kafka_table_matches_audit_and_notify() {
+        assert_table(
+            &kafka_kvs(),
+            &[
+                ("enable", "off", false),
+                ("brokers", "", false),
+                ("topic", "", false),
+                ("acks", "1", false),
+                ("tls_enable", "off", false),
+                ("tls_ca", "", true),
+                ("tls_client_cert", "", true),
+                ("tls_client_key", "", true),
+                ("sasl_enable", "off", false),
+                ("sasl_mechanism", "", false),
+                ("sasl_username", "", false),
+                ("sasl_password", "", true),
+                ("queue_dir", QUEUE_DIR, false),
+                ("queue_limit", QUEUE_LIMIT, false),
+                ("comment", "", false),
+            ],
+        );
+    }
+
+    fn expected_redis(channel: &str) -> Vec<(&str, &str, bool)> {
+        vec![
+            ("enable", "off", false),
+            ("url", "", false),
+            ("channel", channel, false),
+            ("username", "", false),
+            ("password", "", true),
+            ("keep_alive_interval", "15", false),
+            ("queue_dir", QUEUE_DIR, false),
+            ("queue_limit", QUEUE_LIMIT, false),
+            ("max_retry_attempts", "3", false),
+            ("reconnect_retry_attempts", "", false),
+            ("min_retry_delay", "", false),
+            ("max_retry_delay", "", false),
+            ("connection_timeout", "", false),
+            ("response_timeout", "", false),
+            ("pipeline_buffer_size", "", false),
+            ("tls_policy", "", true),
+            ("tls_ca", "", true),
+            ("tls_client_cert", "", true),
+            ("tls_client_key", "", true),
+            ("tls_allow_insecure", "off", false),
+            ("comment", "", false),
+        ]
+    }
+
+    #[test]
+    fn redis_table_matches_audit() {
+        assert_table(&redis_kvs("rustfs_audit_channel"), &expected_redis("rustfs_audit_channel"));
+    }
+
+    #[test]
+    fn redis_table_matches_notify() {
+        assert_table(&redis_kvs("rustfs_notify_channel"), &expected_redis("rustfs_notify_channel"));
+    }
+
+    fn expected_mysql(table: &str) -> Vec<(&str, &str, bool)> {
+        vec![
+            ("enable", "off", false),
+            ("dsn_string", "", true),
+            ("table", table, false),
+            ("format", "access", false),
+            ("tls_ca", "", true),
+            ("tls_client_cert", "", true),
+            ("tls_client_key", "", true),
+            ("queue_dir", QUEUE_DIR, false),
+            ("queue_limit", QUEUE_LIMIT, false),
+            ("max_open_connections", "2", false),
+            ("comment", "", false),
+        ]
+    }
+
+    #[test]
+    fn mysql_table_matches_audit() {
+        assert_table(&mysql_kvs("rustfs_audit_logs"), &expected_mysql("rustfs_audit_logs"));
+    }
+
+    #[test]
+    fn mysql_table_matches_notify() {
+        assert_table(&mysql_kvs("rustfs_events"), &expected_mysql("rustfs_events"));
+    }
+}

crates/ecstore/src/core/sets.rs

@@ -781,7 +781,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
             .await
     }
 
-    #[tracing::instrument(skip(self))]
+    #[tracing::instrument(skip(self, opts))]
     async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
         if opts.delete_prefix && !opts.delete_prefix_object {
             self.delete_prefix(bucket, object, &opts).await?;
@@ -1351,11 +1351,20 @@ pub(crate) async fn make_local_two_set_sets_with_ctx(ctx: Arc<InstanceContext>)
 pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
     ctx: Arc<InstanceContext>,
     pool_idx: usize,
+) -> (Vec<tempfile::TempDir>, Arc<Sets>) {
+    make_local_two_set_sets_for_pool_with_drive_count_and_ctx(ctx, pool_idx, 2).await
+}
+
+#[cfg(any(test, feature = "test-util"))]
+pub(crate) async fn make_local_two_set_sets_for_pool_with_drive_count_and_ctx(
+    ctx: Arc<InstanceContext>,
+    pool_idx: usize,
+    set_drive_count: usize,
 ) -> (Vec<tempfile::TempDir>, Arc<Sets>) {
     use crate::layout::endpoint::Endpoint;
     use rustfs_lock::client::local::LocalClient;
 
-    let format = FormatV3::new(2, 2);
+    let format = FormatV3::new(2, set_drive_count);
     let mut temp_dirs = Vec::new();
     let mut all_endpoints = Vec::new();
     let mut disk_sets = Vec::new();
@@ -1363,7 +1372,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
     for set_index in 0..2 {
         let mut endpoints = Vec::new();
         let mut disks = Vec::new();
-        for disk_index in 0..2 {
+        for disk_index in 0..set_drive_count {
             let temp_dir = tempfile::tempdir().expect("tempdir should be created");
             let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("tempdir path should be utf8"))
                 .expect("endpoint should parse");
@@ -1389,7 +1398,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
             endpoints.push(endpoint);
             disks.push(Some(disk));
         }
-        let lockers = (0..2)
+        let lockers = (0..set_drive_count)
             .map(|_| {
                 Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
                     rustfs_lock::FastObjectLockManager::new(),
@@ -1400,7 +1409,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
             SetDisks::new_with_instance_ctx(
                 "test-owner".to_string(),
                 Arc::new(RwLock::new(disks)),
-                2,
+                set_drive_count,
                 1,
                 set_index,
                 pool_idx,
@@ -1420,7 +1429,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
         endpoints: PoolEndpoints {
             legacy: false,
             set_count: 2,
-            drives_per_set: 2,
+            drives_per_set: set_drive_count,
             endpoints: Endpoints::from(all_endpoints),
             cmd_line: String::new(),
             platform: String::new(),
@@ -1428,7 +1437,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
         format,
         parity_count: 1,
         set_count: 2,
-        set_drive_count: 2,
+        set_drive_count,
         default_parity_count: 1,
         distribution_algo: DistributionAlgoVersion::V1,
         exit_signal: None,

crates/ecstore/src/data_movement/backpressure.rs

@@ -15,7 +15,8 @@
 use crate::error::{Error, Result};
 use crate::runtime::sources::{self as runtime_sources, WorkloadSnapshotProviderRef};
 use metrics::{counter, histogram};
-use rustfs_concurrency::{AdmissionState, WorkloadAdmissionSnapshotProvider, WorkloadClass};
+use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
+use rustfs_concurrency::workload::ForegroundPressure;
 use std::time::{Duration, Instant};
 use tokio::time::sleep;
 use tokio_util::sync::CancellationToken;
@@ -137,23 +138,6 @@ async fn wait_for_data_movement_admission_with_provider(
     }
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-struct ForegroundPressure {
-    class: WorkloadClass,
-    usage_pct: usize,
-    threshold_pct: usize,
-}
-
-impl ForegroundPressure {
-    const fn reason(self) -> &'static str {
-        match self.class {
-            WorkloadClass::ForegroundRead => "foreground_read_pressure",
-            WorkloadClass::ForegroundWrite => "foreground_write_pressure",
-            _ => "foreground_pressure",
-        }
-    }
-}
-
 fn foreground_pressure(
     config: &DataMovementBackpressureConfig,
     provider: Option<&(dyn WorkloadAdmissionSnapshotProvider + Send + Sync)>,
@@ -163,39 +147,11 @@ fn foreground_pressure(
     }
 
     let snapshot = provider?.workload_admission_snapshot();
-    [
+    rustfs_concurrency::workload::foreground_pressure(
-        (WorkloadClass::ForegroundRead, config.foreground_read_high_percent),
+        &snapshot,
-        (WorkloadClass::ForegroundWrite, config.foreground_write_high_percent),
+        config.foreground_read_high_percent,
-    ]
+        config.foreground_write_high_percent,
-    .into_iter()
+    )
-    .filter_map(|(class, threshold_pct)| {
-        if threshold_pct == 0 {
-            return None;
-        }
-
-        let entry = snapshot.get(class)?;
-        let usage_pct = if matches!(entry.state, AdmissionState::Saturated) {
-            100
-        } else {
-            let limit = entry.limit?;
-            if limit == 0 {
-                return None;
-            }
-            entry
-                .active
-                .unwrap_or(0)
-                .saturating_mul(100)
-                .checked_div(limit)
-                .unwrap_or(100)
-        };
-
-        (usage_pct >= threshold_pct).then_some(ForegroundPressure {
-            class,
-            usage_pct,
-            threshold_pct,
-        })
-    })
-    .max_by_key(|pressure| pressure.usage_pct)
 }
 
 fn record_delay_start(
@@ -276,7 +232,7 @@ fn record_delay_completion(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use rustfs_concurrency::{WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot};
+    use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadClass};
     use std::sync::Arc;
 
     #[derive(Debug)]

crates/ecstore/src/data_movement/mod.rs

@@ -16,13 +16,14 @@
 
 pub(crate) mod backpressure;
 
+use crate::core::pools::{DecommissionCapacityOwner, decommission_capacity_mutation_id};
 use crate::error::{
     Error, Result, is_err_data_movement_overwrite, is_err_invalid_upload_id, is_err_object_not_found, is_err_version_not_found,
 };
 use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
 use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
 use crate::storage_api_contracts::{
-    multipart::{CompletePart, MultipartOperations as _},
+    multipart::CompletePart,
     namespace::NamespaceLocking as _,
     object::{HTTPPreconditions, ObjectOperations as _},
 };
@@ -160,6 +161,99 @@ pub fn mark_multipart_upload_completed(flag: &Arc<AtomicBool>) {
     flag.store(false, Ordering::Relaxed);
 }
 
+#[cfg(test)]
+struct DataMovementMultipartAbortBarrierState {
+    bucket: String,
+    object: String,
+    arrived: tokio::sync::Notify,
+    release: tokio::sync::Notify,
+}
+
+#[cfg(test)]
+pub(crate) struct DataMovementMultipartAbortBarrier {
+    state: Arc<DataMovementMultipartAbortBarrierState>,
+}
+
+#[cfg(test)]
+static DATA_MOVEMENT_MULTIPART_ABORT_BARRIER: std::sync::OnceLock<
+    std::sync::Mutex<Option<Arc<DataMovementMultipartAbortBarrierState>>>,
+> = std::sync::OnceLock::new();
+
+#[cfg(test)]
+impl DataMovementMultipartAbortBarrier {
+    pub(crate) fn install(bucket: &str, object: &str) -> Self {
+        let state = Arc::new(DataMovementMultipartAbortBarrierState {
+            bucket: bucket.to_string(),
+            object: object.to_string(),
+            arrived: tokio::sync::Notify::new(),
+            release: tokio::sync::Notify::new(),
+        });
+        let mut slot = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("data movement multipart abort barrier mutex should not poison");
+        assert!(slot.is_none(), "data movement multipart abort barrier must be unique");
+        *slot = Some(Arc::clone(&state));
+        Self { state }
+    }
+
+    pub(crate) async fn wait_until_paused(&self) {
+        tokio::time::timeout(StdDuration::from_secs(30), self.state.arrived.notified())
+            .await
+            .expect("data movement multipart failure should reach abort cleanup");
+    }
+}
+
+#[cfg(test)]
+impl Drop for DataMovementMultipartAbortBarrier {
+    fn drop(&mut self) {
+        self.state.release.notify_one();
+        let mut slot = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("data movement multipart abort barrier mutex should not poison");
+        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *slot = None;
+        }
+    }
+}
+
+#[cfg(test)]
+async fn pause_data_movement_multipart_before_abort(bucket: &str, object: &str) {
+    let barrier = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("data movement multipart abort barrier mutex should not poison")
+        .as_ref()
+        .filter(|barrier| barrier.bucket == bucket && barrier.object == object)
+        .cloned();
+    if let Some(barrier) = barrier {
+        barrier.arrived.notify_one();
+        barrier.release.notified().await;
+    }
+}
+
+fn data_movement_abort_opts(
+    src_pool_idx: usize,
+    expected_bucket_incarnation_id: Option<uuid::Uuid>,
+    lock_lost_signal: Option<&Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
+    capacity_owner: Option<DecommissionCapacityOwner>,
+) -> ObjectOptions {
+    let mut opts = ObjectOptions {
+        data_movement: true,
+        src_pool_idx,
+        expected_bucket_incarnation_id,
+        ..Default::default()
+    };
+    if let Some(capacity_owner) = capacity_owner {
+        capacity_owner.apply_to(&mut opts);
+    }
+    if let Some(signal) = lock_lost_signal {
+        opts.add_namespace_lock_lost_signal(Arc::clone(signal));
+    }
+    opts
+}
+
 fn insert_data_movement_checksum(user_defined: &mut HashMap<String, String>, object_info: &ObjectInfo) {
     rustfs_utils::http::remove_header_map(user_defined, rustfs_utils::http::SUFFIX_REPLICATION_SSEC_CRC);
     if let Some(checksum) = object_info.checksum.as_ref().filter(|checksum| !checksum.is_empty()) {
@@ -192,7 +286,7 @@ fn data_movement_new_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usiz
         preserve_etag: object_info.etag.clone(),
         src_pool_idx,
         data_movement: true,
-        ..Default::default()
+        ..ObjectOptions::with_capacity_expected_data_bytes(usize::try_from(object_info.size).ok())
     }
 }
 
@@ -363,7 +457,7 @@ fn data_movement_complete_multipart_opts(
         preserve_etag: object_info.etag.clone(),
         user_defined,
         src_pool_idx,
-        ..Default::default()
+        ..ObjectOptions::with_capacity_expected_data_bytes(usize::try_from(object_info.size).ok())
     })
 }
 
@@ -533,6 +627,7 @@ fn schedule_data_movement_multipart_abort_cleanup(
     bucket: String,
     object: String,
     upload_id: String,
+    opts: ObjectOptions,
     op_label: &str,
 ) {
     let op_label = op_label.to_string();
@@ -540,23 +635,32 @@ fn schedule_data_movement_multipart_abort_cleanup(
         for attempt in 1..=DATA_MOVEMENT_MULTIPART_ABORT_RETRY_ATTEMPTS {
             tokio::time::sleep(StdDuration::from_secs(DATA_MOVEMENT_MULTIPART_ABORT_RETRY_DELAY_SECS)).await;
 
-            let Some(pool) = store.pools.get(target_pool_idx).cloned() else {
+            if store.pools.get(target_pool_idx).is_none() {
                 error!(
                     "{op_label}: background abort_multipart_upload cleanup skipped for {bucket}/{object} upload {upload_id}: target pool {target_pool_idx} is out of range"
                 );
                 return;
+            }
+
+            let mut cleanup_opts = opts.clone();
+            let _multipart_mutation_fence = match DecommissionCapacityOwner::from_options(&cleanup_opts) {
+                Some(owner) => match store.acquire_decommission_multipart_mutation_fence(owner).await {
+                    Ok(fence) => {
+                        fence.add_namespace_lock_fence(&mut cleanup_opts);
+                        Some(fence)
+                    }
+                    Err(err) => {
+                        error!(
+                            "{op_label}: background abort_multipart_upload cleanup could not fence {bucket}/{object} upload {upload_id} on attempt {attempt}: {err:?}"
+                        );
+                        continue;
+                    }
+                },
+                None => None,
             };
 
-            match pool
+            match store
-                .abort_multipart_upload(
+                .abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, &object, &upload_id, &cleanup_opts)
-                    &bucket,
-                    &object,
-                    &upload_id,
-                    &ObjectOptions {
-                        data_movement: true,
-                        ..Default::default()
-                    },
-                )
                 .await
             {
                 Ok(()) => {
@@ -1334,27 +1438,43 @@ fn resolve_data_movement_overwrite_resume_result_for(
     Ok(matches!(err, Error::PreconditionFailed) && is_superseding_unversioned_data_movement_object(source, &target))
 }
 
+#[derive(Clone, Copy)]
+struct DataMovementOverwriteCapacity {
+    owner: Option<DecommissionCapacityOwner>,
+    expected_data_bytes: Option<usize>,
+}
+
 async fn should_treat_data_movement_overwrite_as_complete(
     store: &ECStore,
-    src_pool_idx: usize,
+    pool_indices: (usize, usize),
-    target_pool_idx: usize,
     bucket: &str,
     object_info: &ObjectInfo,
     err: &Error,
     compare_part_checksums: bool,
+    capacity: DataMovementOverwriteCapacity,
 ) -> Result<bool> {
     if !should_check_data_movement_overwrite_resume(err) {
         return Ok(false);
     }
+    let (src_pool_idx, target_pool_idx) = pool_indices;
 
-    resolve_data_movement_overwrite_resume_result_for(
+    let equivalent = resolve_data_movement_overwrite_resume_result_for(
         err,
         find_data_movement_target_info(store, target_pool_idx, bucket, object_info).await,
         object_info,
         src_pool_idx,
         target_pool_idx,
         compare_part_checksums,
-    )
+    )?;
+    if equivalent && let Some(owner) = capacity.owner {
+        let expected_data_bytes = capacity
+            .expected_data_bytes
+            .ok_or_else(|| Error::other("equivalent data-movement target cannot reconcile unknown committed data size"))?;
+        store
+            .reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, expected_data_bytes)
+            .await?;
+    }
+    Ok(equivalent)
 }
 
 fn data_movement_part_stage_error(
@@ -1395,6 +1515,7 @@ pub(crate) async fn migrate_decommission_object(
     rd: GetObjectReader,
     source_bucket_incarnation_id: Option<uuid::Uuid>,
     op_label: &str,
+    capacity_owner: Option<DecommissionCapacityOwner>,
 ) -> Result<()> {
     let source = rd.object_info.clone();
     let _mutation_fence = store
@@ -1415,6 +1536,7 @@ pub(crate) async fn migrate_decommission_object(
         source_bucket_incarnation_id,
         op_label,
         None,
+        capacity_owner,
         Some(&_mutation_fence),
     )
     .await
@@ -1451,6 +1573,7 @@ pub(crate) async fn migrate_object_with_lock_lost_signal(
         op_label,
         lock_lost_signal,
         None,
+        None,
     )
     .await
 }
@@ -1464,22 +1587,102 @@ async fn migrate_object_inner(
     source_bucket_incarnation_id: Option<uuid::Uuid>,
     op_label: &str,
     lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
+    capacity_owner: Option<DecommissionCapacityOwner>,
     mutation_fence: Option<&ObjectLockDiagGuard>,
 ) -> Result<()> {
     let object_info = rd.object_info.clone();
+    let capacity_owner = capacity_owner.map(|owner| {
+        let version_id = object_info.version_id.map(|version_id| version_id.to_string());
+        let mutation_id = owner.mutation_id.unwrap_or_else(|| {
+            decommission_capacity_mutation_id(
+                owner,
+                &bucket,
+                &object_info.name,
+                version_id.as_deref(),
+                object_info.delete_marker,
+                object_info.mod_time,
+            )
+        });
+        owner.with_mutation_id(mutation_id)
+    });
     let has_part_checksums = object_info
         .parts
         .iter()
         .any(|part| part.checksums.as_ref().is_some_and(|checksums| !checksums.is_empty()));
 
     let preserve_part_checksums = data_movement_part_checksum_writer_enabled();
+    let capacity_expected_data_bytes = usize::try_from(object_info.size).ok();
 
     if should_use_multipart_data_movement(&object_info, has_part_checksums) {
+        // The decommission object fence already covers the source/target
+        // namespace for this migration. Acquiring the synthetic multipart
+        // fence while holding that read lock deadlocks local lock domains;
+        // retain the extra fence only for callers without the outer fence.
+        let multipart_mutation_fence = match (capacity_owner, mutation_fence.is_some()) {
+            (Some(owner), false) => Some(store.acquire_decommission_multipart_mutation_fence(owner).await?),
+            _ => None,
+        };
         let mut new_multipart_opts = data_movement_new_multipart_opts(&object_info, pool_idx);
+        if let Some(capacity_owner) = capacity_owner {
+            capacity_owner.apply_to(&mut new_multipart_opts);
+        }
         new_multipart_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
         if let Some(signal) = lock_lost_signal.as_ref() {
             new_multipart_opts.add_namespace_lock_lost_signal(Arc::clone(signal));
         }
+        if let Some(fence) = multipart_mutation_fence.as_ref() {
+            fence.add_namespace_lock_fence(&mut new_multipart_opts);
+        }
+        if let Some(owner) = capacity_owner {
+            let existing_target_pool_idx = store
+                .select_data_movement_pool_idx(&bucket, &object_info.name, -1, &new_multipart_opts, false)
+                .await?;
+            if existing_target_pool_idx != pool_idx
+                && let Some(target) =
+                    find_data_movement_target_info(store.as_ref(), existing_target_pool_idx, &bucket, &object_info).await?
+                && is_equivalent_data_movement_object_identity(&object_info, &target, true, preserve_part_checksums)
+            {
+                let expected_data_bytes = capacity_expected_data_bytes
+                    .ok_or_else(|| Error::other("equivalent multipart target cannot reconcile unknown committed data size"))?;
+                store
+                    .reconcile_decommission_capacity_after_equivalent_target(owner, existing_target_pool_idx, expected_data_bytes)
+                    .await?;
+                info!(
+                    "{op_label}: multipart upload restart reconciled equivalent target for {}/{}",
+                    bucket.as_str(),
+                    object_info.name.as_str()
+                );
+                return Ok(());
+            }
+            let mut cleanup_opts =
+                data_movement_abort_opts(pool_idx, source_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
+            if let Some(fence) = mutation_fence {
+                fence.add_namespace_lock_fence(&mut cleanup_opts);
+            }
+            if let Some(fence) = multipart_mutation_fence.as_ref() {
+                fence.add_namespace_lock_fence(&mut cleanup_opts);
+            }
+            for target_pool_idx in store.decommission_capacity_cleanup_target_indices(owner).await? {
+                store
+                    .reconcile_multipart_uploads_for_data_movement(
+                        target_pool_idx,
+                        &bucket,
+                        &object_info.name,
+                        &data_movement_upload_identity(&object_info),
+                        &cleanup_opts,
+                    )
+                    .await
+                    .map_err(|err| {
+                        data_movement_stage_error(
+                            op_label,
+                            "reconcile_multipart_upload",
+                            bucket.as_str(),
+                            object_info.name.as_str(),
+                            err,
+                        )
+                    })?;
+            }
+        }
         let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
             .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
             .await
@@ -1532,9 +1735,15 @@ async fn migrate_object_inner(
                     expected_bucket_incarnation_id,
                     ..Default::default()
                 };
+                if let Some(capacity_owner) = capacity_owner {
+                    capacity_owner.apply_to(&mut part_opts);
+                }
                 if let Some(signal) = lock_lost_signal.as_ref() {
                     part_opts.add_namespace_lock_lost_signal(Arc::clone(signal));
                 }
+                if let Some(fence) = multipart_mutation_fence.as_ref() {
+                    fence.add_namespace_lock_fence(&mut part_opts);
+                }
                 let pi = match store
                     .put_object_part_for_data_movement(
                         target_pool_idx,
@@ -1578,10 +1787,16 @@ async fn migrate_object_inner(
                         err,
                     )
                 })?;
+            if let Some(capacity_owner) = capacity_owner {
+                capacity_owner.apply_to(&mut complete_multipart_opts);
+            }
             complete_multipart_opts.expected_bucket_incarnation_id = expected_bucket_incarnation_id;
             if let Some(signal) = lock_lost_signal.as_ref() {
                 complete_multipart_opts.add_namespace_lock_lost_signal(Arc::clone(signal));
             }
+            if let Some(fence) = multipart_mutation_fence.as_ref() {
+                fence.add_namespace_lock_fence(&mut complete_multipart_opts);
+            }
             if let Err(err) = store
                 .clone()
                 .complete_multipart_upload_for_data_movement(
@@ -1596,12 +1811,15 @@ async fn migrate_object_inner(
             {
                 if should_treat_data_movement_overwrite_as_complete(
                     store.as_ref(),
-                    pool_idx,
+                    (pool_idx, target_pool_idx),
-                    target_pool_idx,
                     bucket.as_str(),
                     &object_info,
                     &err,
                     preserve_part_checksums,
+                    DataMovementOverwriteCapacity {
+                        owner: capacity_owner,
+                        expected_data_bytes: capacity_expected_data_bytes,
+                    },
                 )
                 .await?
                 {
@@ -1629,31 +1847,37 @@ async fn migrate_object_inner(
         .await;
 
         if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) {
+            let mut abort_opts =
+                data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
+            if let Some(fence) = mutation_fence {
+                fence.add_namespace_lock_fence(&mut abort_opts);
+            }
+            if let Some(fence) = multipart_mutation_fence.as_ref() {
+                fence.add_namespace_lock_fence(&mut abort_opts);
+            }
             let abort_result = store
-                .abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, &object_info.name, &res.upload_id, &{
+                .abort_multipart_upload_for_data_movement(
-                    let mut opts = ObjectOptions {
+                    target_pool_idx,
-                        data_movement: true,
+                    &bucket,
-                        src_pool_idx: pool_idx,
+                    &object_info.name,
-                        expected_bucket_incarnation_id,
+                    &res.upload_id,
-                        ..Default::default()
+                    &abort_opts,
-                    };
+                )
-                    if let Some(signal) = lock_lost_signal.as_ref() {
-                        opts.add_namespace_lock_lost_signal(Arc::clone(signal));
-                    }
-                    opts
-                })
                 .await;
             match abort_result {
                 Ok(()) => return Ok(()),
                 Err(abort_err) if is_err_invalid_upload_id(&abort_err) => {
                     if should_treat_data_movement_overwrite_as_complete(
                         store.as_ref(),
-                        pool_idx,
+                        (pool_idx, target_pool_idx),
-                        target_pool_idx,
                         bucket.as_str(),
                         &object_info,
                         &abort_err,
                         preserve_part_checksums,
+                        DataMovementOverwriteCapacity {
+                            owner: capacity_owner,
+                            expected_data_bytes: capacity_expected_data_bytes,
+                        },
                     )
                     .await?
                     {
@@ -1683,6 +1907,7 @@ async fn migrate_object_inner(
                         bucket.clone(),
                         object_info.name.clone(),
                         res.upload_id.clone(),
+                        abort_opts,
                         op_label,
                     );
                     return Err(data_movement_stage_error(
@@ -1698,19 +1923,24 @@ async fn migrate_object_inner(
 
         if let Err(primary_err) = multipart_result {
             if should_abort_multipart_upload(&abort_multipart_flag) {
+                #[cfg(test)]
+                pause_data_movement_multipart_before_abort(&bucket, &object_info.name).await;
+                let mut abort_opts =
+                    data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
+                if let Some(fence) = mutation_fence {
+                    fence.add_namespace_lock_fence(&mut abort_opts);
+                }
+                if let Some(fence) = multipart_mutation_fence.as_ref() {
+                    fence.add_namespace_lock_fence(&mut abort_opts);
+                }
                 return match store
-                    .abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, &object_info.name, &res.upload_id, &{
+                    .abort_multipart_upload_for_data_movement(
-                        let mut opts = ObjectOptions {
+                        target_pool_idx,
-                            data_movement: true,
+                        &bucket,
-                            src_pool_idx: pool_idx,
+                        &object_info.name,
-                            expected_bucket_incarnation_id,
+                        &res.upload_id,
-                            ..Default::default()
+                        &abort_opts,
-                        };
+                    )
-                        if let Some(signal) = lock_lost_signal.as_ref() {
-                            opts.add_namespace_lock_lost_signal(Arc::clone(signal));
-                        }
-                        opts
-                    })
                     .await
                 {
                     Ok(()) => Err(primary_err),
@@ -1722,6 +1952,7 @@ async fn migrate_object_inner(
                             bucket.clone(),
                             object_info.name.clone(),
                             res.upload_id.clone(),
+                            abort_opts,
                             op_label,
                         );
                         Err(resolve_data_movement_abort_result(
@@ -1744,6 +1975,9 @@ async fn migrate_object_inner(
     let mut data = data_movement_put_object_reader(bucket.as_str(), &object_info, rd, op_label)?;
 
     let mut put_opts = data_movement_put_object_opts(&object_info, pool_idx);
+    if let Some(capacity_owner) = capacity_owner {
+        capacity_owner.apply_to(&mut put_opts);
+    }
     put_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
     if let Some(signal) = lock_lost_signal {
         put_opts.add_namespace_lock_lost_signal(signal);
@@ -1755,12 +1989,15 @@ async fn migrate_object_inner(
     if let Err(err) = put_result {
         if should_treat_data_movement_overwrite_as_complete(
             store.as_ref(),
-            pool_idx,
+            (pool_idx, target_pool_idx),
-            target_pool_idx,
             bucket.as_str(),
             &object_info,
             &err,
             preserve_part_checksums,
+            DataMovementOverwriteCapacity {
+                owner: capacity_owner,
+                expected_data_bytes: capacity_expected_data_bytes,
+            },
         )
         .await?
         {

crates/ecstore/src/data_usage/mod.rs

@@ -109,7 +109,10 @@ static USAGE_MEMORY_GENERATION: AtomicU64 = AtomicU64::new(0);
 /// strictly tighter than beta.11 (usage treated as 0) and strictly more
 /// available than a blanket 503. The fallback applies to any window without
 /// authoritative usage, not only pre-v2 upgrades; the values always come from
-/// the last persisted scanner output. Loads go through the TTL-bounded
+/// the last persisted scanner output — pre-discard sizes of the
+/// authoritative snapshot first, backfilled per bucket from the observed
+/// (nonconverged) snapshot for buckets no authoritative cycle has covered
+/// yet (issue #6852). Loads go through the TTL-bounded
 /// snapshot cache, so the quota path adds at most one backend read per
 /// [`DATA_USAGE_CACHE_TTL_SECS`] window. Returns `None` for buckets absent
 /// from every persisted snapshot — those still fail closed.
@@ -168,7 +171,7 @@ fn fresh_cached_data_usage_snapshot(
 
 fn cache_data_usage_snapshot_result(
     cache: &mut Option<CachedDataUsageSnapshot>,
-    result: Result<(DataUsageInfo, HashMap<String, u64>), Error>,
+    result: Result<LoadedUsageBaseline, Error>,
     loaded_at: tokio::time::Instant,
     refresh_generation: u64,
     current_generation: u64,
@@ -178,7 +181,19 @@ fn cache_data_usage_snapshot_result(
     }
 
     Some(match result {
-        Ok((info, degraded_baseline)) => {
+        Ok(LoadedUsageBaseline {
+            info,
+            mut degraded_baseline,
+            observed_unavailable,
+        }) => {
+            // A flaky observed read must not shrink quota coverage for a TTL
+            // window: carry the previous refresh's baseline entries forward,
+            // letting the fresh (authoritative) values win where they exist.
+            if observed_unavailable && let Some(previous) = cache.as_ref() {
+                for (bucket, size) in &previous.degraded_baseline {
+                    degraded_baseline.entry(bucket.clone()).or_insert(*size);
+                }
+            }
             *cache = Some(CachedDataUsageSnapshot {
                 info: Some(info.clone()),
                 loaded_at,
@@ -422,9 +437,7 @@ async fn save_data_usage_in_backend(
     if publication_epoch != expected_publication_epoch {
         return Err(Error::other("data usage publication epoch changed before save"));
     }
-    crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data)
+    crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data).await?;
-        .await
-        .map_err(Error::other)?;
     drop(publication_guard);
 
     cleanup_observed_data_usage_after_authoritative_save_with_publication(store.as_ref(), &data_usage_info, Some(store.as_ref()))
@@ -578,7 +591,7 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change(
     guard: Option<&rustfs_lock::NamespaceLockGuard>,
 ) -> Result<(), Error> {
     ensure_bucket_namespace_guard(guard, bucket, "data usage cache cleanup")?;
-    let _ = USAGE_MEMORY_GENERATION.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(1)));
+    let _ = USAGE_MEMORY_GENERATION.try_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(1)));
     live_bucket_usage_cache().invalidate(bucket).await;
     clear_bucket_usage_memory(bucket, guard).await?;
 
@@ -641,7 +654,7 @@ where
     {
         Ok(reader) => reader,
         Err(Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::ConfigNotFound) => return Ok(None),
-        Err(err) => return Err(err),
+        Err(err) => return Err(map_data_usage_metadata_read_error(err, object)),
     };
     let revision = reader
         .object_info
@@ -656,6 +669,18 @@ where
     Ok(Some((data_usage_info, revision)))
 }
 
+/// A missing usage object is harmless during bucket creation, but a missing
+/// system metadata volume is a storage outage. Keep the latter retryable and
+/// distinguishable from the user bucket not existing.
+fn map_data_usage_metadata_read_error(err: Error, object: &str) -> Error {
+    match err {
+        Error::BucketNotFound(_) | Error::VolumeNotFound => {
+            Error::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), object.to_string())
+        }
+        other => other,
+    }
+}
+
 fn data_usage_contains_bucket(data_usage_info: &DataUsageInfo, bucket: &str) -> bool {
     data_usage_info.buckets_usage.contains_key(bucket) || data_usage_info.bucket_sizes.contains_key(bucket)
 }
@@ -912,7 +937,7 @@ where
             )
             .await;
         drop(publication_guard);
-        match save_result {
+        match save_result.map_err(|err| crate::config::com::map_system_metadata_write_error(err, object)) {
             Ok(_) => return Ok(()),
             Err(err) => {
                 if let Some((observed, observed_revision)) = load_data_usage_for_bucket_removal(store, object).await? {
@@ -1103,24 +1128,78 @@ async fn load_data_usage_snapshot(store: Arc<ECStore>) -> Result<(DataUsageInfo,
 /// Load data usage info from backend storage
 #[instrument(skip(store))]
 pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
-    Ok(load_data_usage_from_backend_with_baseline(store).await?.0)
+    Ok(load_data_usage_from_backend_with_baseline(store).await?.info)
+}
+
+/// One refresh of the persisted usage snapshot plus the quota-admission
+/// baseline derived from it.
+struct LoadedUsageBaseline {
+    info: DataUsageInfo,
+    degraded_baseline: HashMap<String, u64>,
+    /// True when the observed snapshot could not be read (a transport error,
+    /// not absence): the cached loader then carries the previous refresh's
+    /// baseline entries forward instead of shrinking quota coverage for a
+    /// whole TTL window over one flaky read.
+    observed_unavailable: bool,
 }
 
 /// Like [`load_data_usage_from_backend`], but also returns the pre-discard
 /// per-bucket sizes so the cached loader can retain them as the degraded
 /// quota-admission baseline (issue #5716).
-async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Result<(DataUsageInfo, HashMap<String, u64>), Error> {
+async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Result<LoadedUsageBaseline, Error> {
-    let (data_usage_info, source) = load_data_usage_snapshot(store).await?;
+    let (loaded_snapshot, source) = load_data_usage_snapshot(store.clone()).await?;
-    Ok(normalize_loaded_data_usage(data_usage_info, source.is_authoritative()).await)
+    // The observed-newness gate below compares against the snapshot as
+    // persisted, before normalization demotes or discards anything.
+    let authoritative_as_persisted = loaded_snapshot.clone();
+    let (info, mut degraded_baseline) = normalize_loaded_data_usage(loaded_snapshot, source.is_authoritative()).await;
+
+    // A bucket without a converged scanner cycle behind it — a freshly joined
+    // replica whose every cycle is superseded by the sustained replication
+    // write stream, or a bucket created after the last converged cycle on a
+    // busy site (#6852) — has no authoritative size, and quota admission
+    // fails its writes closed indefinitely. The observed (nonconverged)
+    // snapshot those superseded cycles still publish is the only grounded
+    // usage in that window, so it backfills buckets the loaded baseline does
+    // not cover; a value already in the baseline always wins. The newness
+    // gate ties the observation to this exact authoritative snapshot, so a
+    // stale observed object left behind by an earlier incarnation (e.g. a
+    // deleted and recreated bucket) cannot inject ghost usage. Loads sit
+    // behind the same TTL cache as the snapshot itself, so this adds at most
+    // one backend read per TTL window.
+    let mut observed_unavailable = false;
+    match load_observed_data_usage_snapshot(store).await {
+        Ok(Some(observed)) if observed_data_usage_is_newer(&observed, &authoritative_as_persisted) => {
+            backfill_degraded_baseline_from_observed(&mut degraded_baseline, &observed);
+        }
+        Ok(_) => {}
+        Err(_) => observed_unavailable = true,
+    }
+
+    Ok(LoadedUsageBaseline {
+        info,
+        degraded_baseline,
+        observed_unavailable,
+    })
 }
 
-async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUsageInfo> {
+/// Fill quota-baseline gaps from an observed (nonconverged) snapshot without
+/// overriding any bucket the authoritative baseline already covers.
+fn backfill_degraded_baseline_from_observed(degraded_baseline: &mut HashMap<String, u64>, observed: &DataUsageInfo) {
+    for (bucket, usage) in &observed.buckets_usage {
+        degraded_baseline.entry(bucket.clone()).or_insert(usage.size);
+    }
+}
+
+/// `Ok(None)` means the observed snapshot is absent or invalid (a settled
+/// answer); `Err` means it could not be read at all, so the caller may keep
+/// using what it learned from a previous read.
+async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Result<Option<DataUsageInfo>, Error> {
     let data = match read_config_preserve_empty(store, &DATA_USAGE_OBSERVED_OBJ_NAME_PATH).await {
         Ok(data) => data,
-        Err(Error::ConfigNotFound) => return None,
+        Err(Error::ConfigNotFound) => return Ok(None),
         Err(err) => {
             record_usage_snapshot_failure("read_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
-            return None;
+            return Err(err);
         }
     };
 
@@ -1129,7 +1208,7 @@ async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUs
             if info.usage_snapshot_converged == Some(false)
                 && (info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot()) =>
         {
-            Some(info)
+            Ok(Some(info))
         }
         Ok(_) => {
             error!(
@@ -1140,11 +1219,11 @@ async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUs
                 object = %DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
                 "observed data usage snapshot was not a structurally complete nonconverged view"
             );
-            None
+            Ok(None)
         }
         Err(err) => {
             record_usage_snapshot_decode_failure("parse_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
-            None
+            Ok(None)
         }
     }
 }
@@ -1161,14 +1240,50 @@ fn select_admin_data_usage_snapshot(
         authoritative.usage_snapshot_converged = Some(true);
     }
     match observed {
+        Some(observed)
+            if observed.usage_snapshot_partial
+                && authoritative.is_complete_bucket_usage_snapshot()
+                && observed_data_usage_is_newer(&observed, &authoritative) =>
+        {
+            (merge_partial_observation_for_admin(authoritative, observed), true)
+        }
         Some(observed) if observed_data_usage_is_newer(&observed, &authoritative) => (observed, true),
         _ => (authoritative, authoritative_format),
     }
 }
 
+fn merge_partial_observation_for_admin(mut authoritative: DataUsageInfo, observed: DataUsageInfo) -> DataUsageInfo {
+    for (bucket, usage) in observed.buckets_usage {
+        authoritative.buckets_usage.insert(bucket, usage);
+    }
+
+    authoritative.last_update = observed.last_update;
+    authoritative.scanner_cycle = observed.scanner_cycle;
+    authoritative.scanner_epoch = observed.scanner_epoch;
+    authoritative.usage_snapshot_complete = false;
+    authoritative.usage_snapshot_partial = true;
+    authoritative.usage_snapshot_converged = Some(false);
+    authoritative.usage_snapshot_authoritative_baseline = observed.usage_snapshot_authoritative_baseline;
+    authoritative.usage_snapshot_set_states = observed.usage_snapshot_set_states;
+    authoritative.usage_snapshot_bootstrap_pending = false;
+    authoritative.buckets_count = authoritative.buckets_usage.len() as u64;
+    authoritative.bucket_sizes = authoritative
+        .buckets_usage
+        .iter()
+        .map(|(bucket, usage)| (bucket.clone(), usage.size))
+        .collect();
+    authoritative.replication_info.clear();
+    authoritative.tier_stats = None;
+    authoritative.unknown_tier_stats = None;
+    authoritative.calculate_totals();
+    authoritative
+}
+
 async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
     let (authoritative, source) = load_data_usage_snapshot(store.clone()).await?;
-    let observed = load_observed_data_usage_snapshot(store).await;
+    // For the one-shot admin view a failed observed read degrades to "no
+    // observation", same as before the read was fallible.
+    let observed = load_observed_data_usage_snapshot(store).await.ok().flatten();
     let (selected, selected_is_current_format) =
         select_admin_data_usage_snapshot(authoritative, source.is_authoritative(), observed);
     Ok(normalize_loaded_data_usage(selected, selected_is_current_format).await.0)
@@ -1331,7 +1446,11 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
         let refresh_generation = admin_data_usage_snapshot_generation();
         let result = load_admin_data_usage_from_backend(store.clone())
             .await
-            .map(|info| (info, HashMap::new()));
+            .map(|info| LoadedUsageBaseline {
+                info,
+                degraded_baseline: HashMap::new(),
+                observed_unavailable: false,
+            });
         let loaded_at = tokio::time::Instant::now();
         let mut cache = admin_data_usage_snapshot_cache().write().await;
         if let Some(result) = cache_data_usage_snapshot_result(
@@ -2482,6 +2601,37 @@ mod tests {
     use std::sync::Arc;
     use tokio::{io::AsyncReadExt, sync::Mutex};
 
+    #[test]
+    fn observed_snapshot_only_backfills_baseline_gaps() {
+        let mut baseline = HashMap::from([("covered".to_string(), 111_u64)]);
+        let observed = DataUsageInfo {
+            buckets_usage: HashMap::from([
+                (
+                    "covered".to_string(),
+                    BucketUsageInfo {
+                        size: 999,
+                        ..Default::default()
+                    },
+                ),
+                (
+                    "replica-only".to_string(),
+                    BucketUsageInfo {
+                        size: 42,
+                        ..Default::default()
+                    },
+                ),
+            ]),
+            ..Default::default()
+        };
+
+        backfill_degraded_baseline_from_observed(&mut baseline, &observed);
+
+        // The authoritative value must win; only the uncovered bucket (#6852:
+        // a replica that never landed a converged cycle) is filled in.
+        assert_eq!(baseline.get("covered"), Some(&111));
+        assert_eq!(baseline.get("replica-only"), Some(&42));
+    }
+
     #[derive(Debug, Default)]
     struct UsageCasState {
         object: Option<(Vec<u8>, u64)>,
@@ -2725,6 +2875,7 @@ mod tests {
     struct UsageCacheReadStore {
         transient_failures: Mutex<usize>,
         reads: Mutex<Vec<String>>,
+        terminal_error: Mutex<Option<Error>>,
     }
 
     impl UsageCacheReadStore {
@@ -2732,6 +2883,15 @@ mod tests {
             Self {
                 transient_failures: Mutex::new(n),
                 reads: Mutex::new(Vec::new()),
+                terminal_error: Mutex::new(None),
+            }
+        }
+
+        fn with_terminal_error(error: Error) -> Self {
+            Self {
+                transient_failures: Mutex::new(0),
+                reads: Mutex::new(Vec::new()),
+                terminal_error: Mutex::new(Some(error)),
             }
         }
 
@@ -2759,6 +2919,9 @@ mod tests {
             _opts: &Self::ObjectOptions,
         ) -> Result<Self::GetObjectReader, Self::Error> {
             self.reads.lock().await.push(object.to_string());
+            if let Some(error) = self.terminal_error.lock().await.clone() {
+                return Err(error);
+            }
             let mut remaining = self.transient_failures.lock().await;
             if *remaining > 0 {
                 *remaining -= 1;
@@ -2801,6 +2964,7 @@ mod tests {
             decommission_cancelers: RwLock::new(Vec::new()),
             start_gate: TokioMutex::new(()),
             pool_meta_save_gate: TokioMutex::default(),
+            decommission_capacity_entry_gate: TokioMutex::default(),
             ctx,
             bucket_fence_registry: Arc::default(),
         })
@@ -2823,6 +2987,22 @@ mod tests {
         assert!(!is_data_usage_cache_absent(&Error::DiskNotFound));
     }
 
+    #[test]
+    fn data_usage_removal_maps_missing_system_volume_to_read_quorum() {
+        for error in [Error::VolumeNotFound, Error::BucketNotFound(RUSTFS_META_BUCKET.to_string())] {
+            assert_eq!(
+                map_data_usage_metadata_read_error(error, "bucket-metadata/.usage.json"),
+                Error::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "bucket-metadata/.usage.json".to_string())
+            );
+        }
+
+        let missing_object = Error::ObjectNotFound(RUSTFS_META_BUCKET.to_string(), "bucket-metadata/.usage.json".to_string());
+        assert_eq!(
+            map_data_usage_metadata_read_error(missing_object.clone(), "bucket-metadata/.usage.json"),
+            missing_object
+        );
+    }
+
     #[tokio::test]
     async fn load_data_usage_cache_treats_absence_as_an_empty_cache_without_retrying() {
         let name = "usage-cache";
@@ -2838,6 +3018,22 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn data_usage_removal_surfaces_missing_system_volume_as_read_quorum() {
+        for cause in [Error::BucketNotFound(RUSTFS_META_BUCKET.to_string()), Error::VolumeNotFound] {
+            let store = UsageCacheReadStore::with_terminal_error(cause);
+
+            let error = load_data_usage_for_bucket_removal(&store, "bucket-metadata/.usage.json")
+                .await
+                .expect_err("missing system metadata volume must not be treated as an absent usage object");
+
+            assert_eq!(
+                error,
+                Error::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "bucket-metadata/.usage.json".to_string())
+            );
+        }
+    }
+
     #[tokio::test]
     async fn load_data_usage_cache_retries_a_transient_failure() {
         let name = "usage-cache";
@@ -3217,6 +3413,108 @@ mod tests {
         assert_eq!(selected.buckets_usage.get("bucket").map(|usage| usage.size), Some(100));
     }
 
+    #[test]
+    fn partial_admin_observation_preserves_authoritative_cold_buckets() {
+        let baseline_time = SystemTime::UNIX_EPOCH + Duration::from_secs(10);
+        let mut authoritative = data_usage_info_for_test("cold", 152_318, 80 * 1024 * 1024 * 1024, baseline_time);
+        authoritative.scanner_epoch = Some(4);
+        authoritative.scanner_cycle = Some(10);
+        authoritative.buckets_usage.insert(
+            "hot".to_string(),
+            BucketUsageInfo {
+                objects_count: 3_000,
+                versions_count: 3_000,
+                size: 400 * 1024 * 1024,
+                ..Default::default()
+            },
+        );
+        authoritative.buckets_count = 2;
+        authoritative.bucket_sizes = authoritative
+            .buckets_usage
+            .iter()
+            .map(|(bucket, usage)| (bucket.clone(), usage.size))
+            .collect();
+        authoritative.calculate_totals();
+        authoritative.replication_info.insert(
+            "stale-target".to_string(),
+            BucketTargetUsageInfo {
+                replicated_size: 400 * 1024 * 1024,
+                replicated_count: 3_000,
+                ..Default::default()
+            },
+        );
+        authoritative.tier_stats = Some(rustfs_data_usage::AllTierStats {
+            tiers: HashMap::from([(
+                "WARM".to_string(),
+                rustfs_data_usage::TierStats {
+                    total_size: 80 * 1024 * 1024 * 1024,
+                    num_versions: 152_318,
+                    num_objects: 152_318,
+                },
+            )]),
+        });
+
+        let mut observed = DataUsageInfo {
+            last_update: Some(baseline_time + Duration::from_secs(1)),
+            scanner_epoch: Some(4),
+            scanner_cycle: Some(11),
+            usage_snapshot_complete: false,
+            usage_snapshot_partial: true,
+            usage_snapshot_converged: Some(false),
+            usage_snapshot_authoritative_baseline: Some(authoritative.snapshot_identity()),
+            usage_snapshot_set_states: vec![rustfs_data_usage::DataUsageSnapshotSetState {
+                pool_index: 0,
+                set_index: 0,
+                scanner_cycle: Some(11),
+                scanner_epoch: Some(4),
+                scan_plan_digest: Some([1; 32]),
+                complete: true,
+                tombstone: false,
+            }],
+            ..Default::default()
+        };
+        observed.buckets_usage.insert(
+            "hot".to_string(),
+            BucketUsageInfo {
+                objects_count: 34,
+                versions_count: 34,
+                size: 8 * 1024 * 1024,
+                ..Default::default()
+            },
+        );
+        observed.buckets_count = 1;
+        observed.bucket_sizes.insert("hot".to_string(), 8 * 1024 * 1024);
+        observed.calculate_totals();
+
+        let (selected, current_format) = select_admin_data_usage_snapshot(authoritative, true, Some(observed));
+
+        assert!(current_format);
+        assert!(!selected.usage_snapshot_complete);
+        assert!(selected.usage_snapshot_partial);
+        assert!(selected.is_valid_partial_snapshot());
+        assert_eq!(selected.usage_snapshot_converged, Some(false));
+        assert_eq!(selected.buckets_count, 2);
+        assert_eq!(
+            selected
+                .buckets_usage
+                .get("cold")
+                .map(|usage| (usage.objects_count, usage.size)),
+            Some((152_318, 80 * 1024 * 1024 * 1024))
+        );
+        assert_eq!(
+            selected
+                .buckets_usage
+                .get("hot")
+                .map(|usage| (usage.objects_count, usage.size)),
+            Some((34, 8 * 1024 * 1024))
+        );
+        assert_eq!(selected.objects_total_count, 152_352);
+        assert_eq!(selected.objects_total_size, 80 * 1024 * 1024 * 1024 + 8 * 1024 * 1024);
+        assert!(selected.replication_info.is_empty());
+        assert!(selected.tier_stats.is_none());
+        assert!(selected.unknown_tier_stats.is_none());
+    }
+
     #[tokio::test]
     async fn authoritative_save_cleanup_removes_observed_snapshot_best_effort() {
         let store = UsageCasStore::default();
@@ -3288,7 +3586,11 @@ mod tests {
 
         let first = cache_data_usage_snapshot_result(
             &mut cache,
-            Ok((expected, HashMap::new())),
+            Ok(LoadedUsageBaseline {
+                info: expected,
+                degraded_baseline: HashMap::new(),
+                observed_unavailable: false,
+            }),
             loaded_at,
             refresh_generation,
             data_usage_snapshot_generation(),
@@ -3303,6 +3605,38 @@ mod tests {
         assert_snapshot_bucket(&cached, "bucket");
     }
 
+    #[test]
+    #[serial]
+    fn unavailable_observed_read_keeps_previous_baseline_coverage() {
+        let loaded_at = tokio::time::Instant::now();
+        let refresh_generation = data_usage_snapshot_generation();
+        let mut cache = Some(CachedDataUsageSnapshot {
+            info: Some(data_usage_info_for_test("bucket", 1, 42, SystemTime::UNIX_EPOCH)),
+            loaded_at,
+            degraded_baseline: HashMap::from([("observed-only".to_string(), 7_u64), ("covered".to_string(), 1)]),
+        });
+
+        cache_data_usage_snapshot_result(
+            &mut cache,
+            Ok(LoadedUsageBaseline {
+                info: data_usage_info_for_test("bucket", 1, 42, SystemTime::UNIX_EPOCH),
+                degraded_baseline: HashMap::from([("covered".to_string(), 2_u64)]),
+                observed_unavailable: true,
+            }),
+            loaded_at,
+            refresh_generation,
+            data_usage_snapshot_generation(),
+        )
+        .expect("an uninterrupted refresh should populate the cache")
+        .expect("successful load must be returned");
+
+        let baseline = &cache.as_ref().expect("cache must be populated").degraded_baseline;
+        // The bucket only the (now unreadable) observed snapshot covered must
+        // survive the refresh; the freshly loaded value wins where it exists.
+        assert_eq!(baseline.get("observed-only"), Some(&7));
+        assert_eq!(baseline.get("covered"), Some(&2));
+    }
+
     #[test]
     #[serial]
     fn cache_invalidation_during_refresh_prevents_stale_snapshot_resurrection() {
@@ -3317,7 +3651,11 @@ mod tests {
 
         let stale_result = cache_data_usage_snapshot_result(
             &mut cache,
-            Ok((data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH), HashMap::new())),
+            Ok(LoadedUsageBaseline {
+                info: data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH),
+                degraded_baseline: HashMap::new(),
+                observed_unavailable: false,
+            }),
             loaded_at,
             refresh_generation,
             data_usage_snapshot_generation(),

crates/ecstore/src/disk/disk_store.rs

@@ -250,6 +250,40 @@ pub(crate) trait DiskStoreRenameDataExt {
         dst_volume: &str,
         dst_path: &str,
     ) -> Result<RenameDataResp>;
+
+    async fn rename_data_borrowed_with_guard(
+        &self,
+        src_volume: &str,
+        src_path: &str,
+        fi: &FileInfo,
+        dst_volume: &str,
+        dst_path: &str,
+        external_guard: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<RenameDataResp> {
+        let _ = external_guard;
+        self.rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
+            .await
+    }
+}
+
+/// Run a mutation in an owned task when a caller supplied publication guard.
+/// RPC cancellation drops only the waiter; the mutation owner keeps the guard
+/// until its operation has returned, including any detached blocking syscall.
+async fn run_owned_mutation<T, F, Fut>(external_guard: Option<Arc<dyn Send + Sync>>, operation: F) -> Result<T>
+where
+    T: Send + 'static,
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = Result<T>> + Send + 'static,
+{
+    if external_guard.is_none() {
+        return operation().await;
+    }
+    tokio::spawn(async move {
+        let _external_guard = external_guard;
+        operation().await
+    })
+    .await
+    .map_err(|_| Error::other("owned mutation task failed"))?
 }
 
 impl DiskStoreRenameDataExt for LocalDiskWrapper {
@@ -273,6 +307,49 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
         )
         .await
     }
+
+    async fn rename_data_borrowed_with_guard(
+        &self,
+        src_volume: &str,
+        src_path: &str,
+        fi: &FileInfo,
+        dst_volume: &str,
+        dst_path: &str,
+        external_guard: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<RenameDataResp> {
+        let operation = self.clone();
+        let src_volume = src_volume.to_owned();
+        let src_path = src_path.to_owned();
+        let fi = fi.clone();
+        let dst_volume = dst_volume.to_owned();
+        let dst_path = dst_path.to_owned();
+        let timeout_duration = if external_guard.is_some() {
+            // A fenced mutation owns the publication guard until the storage
+            // operation returns. Timing out this waiter would cancel the
+            // LocalDisk future while a spawn_blocking namespace syscall could
+            // still be committing, reopening the movement window. The caller
+            // may drop its waiter; the owned task drains the mutation.
+            Duration::ZERO
+        } else {
+            get_max_timeout_duration()
+        };
+        run_owned_mutation(external_guard, move || async move {
+            operation
+                .track_disk_health_mutation(
+                    "rename_data",
+                    DiskMetricMutation::Write,
+                    || async {
+                        operation
+                            .disk
+                            .rename_data_borrowed(&src_volume, &src_path, &fi, &dst_volume, &dst_path)
+                            .await
+                    },
+                    timeout_duration,
+                )
+                .await
+        })
+        .await
+    }
 }
 
 pub fn get_drive_walkdir_timeout() -> Duration {
@@ -678,17 +755,20 @@ impl DiskOperationMetrics {
         let elapsed_nanos = u64::try_from(elapsed.as_nanos()).unwrap_or(u64::MAX);
         let slot = &self.last_minute[(now_sec % 60) as usize];
         loop {
-            let version = slot.version.load(Ordering::Acquire);
+            // The successful CAS below is AcqRel, so it is the publication
+            // fence for the writer that owns this slot. The initial parity
+            // check does not need to acquire the slot payload.
+            let version = slot.version.load(Ordering::Relaxed);
             if !version.is_multiple_of(2) {
                 std::hint::spin_loop();
                 continue;
             }
             if slot
                 .version
-                .compare_exchange(version, version.wrapping_add(1), Ordering::AcqRel, Ordering::Acquire)
+                .compare_exchange(version, version.wrapping_add(1), Ordering::AcqRel, Ordering::Relaxed)
                 .is_ok()
             {
-                if slot.unix_sec.load(Ordering::Acquire) != now_sec {
+                if slot.unix_sec.load(Ordering::Relaxed) != now_sec {
                     slot.count.store(0, Ordering::Relaxed);
                     slot.acc_time.store(0, Ordering::Relaxed);
                     slot.unix_sec.store(now_sec, Ordering::Release);
@@ -704,14 +784,10 @@ impl DiskOperationMetrics {
     fn last_minute_snapshot(&self, now_sec: u64) -> TimedAction {
         let mut snapshot = TimedAction::default();
         for slot in &self.last_minute {
-            let version = slot.version.load(Ordering::Acquire);
+            let Some((slot_sec, count, acc_time)) = slot.snapshot() else {
-            if !version.is_multiple_of(2) {
                 continue;
-            }
+            };
-            let slot_sec = slot.unix_sec.load(Ordering::Acquire);
+            if slot_sec <= now_sec && now_sec.saturating_sub(slot_sec) < 60 {
-            let count = slot.count.load(Ordering::Acquire);
-            let acc_time = slot.acc_time.load(Ordering::Acquire);
-            if slot.version.load(Ordering::Acquire) == version && slot_sec <= now_sec && now_sec.saturating_sub(slot_sec) < 60 {
                 snapshot.count = snapshot.count.saturating_add(count);
                 snapshot.acc_time = snapshot.acc_time.saturating_add(acc_time);
             }
@@ -720,6 +796,23 @@ impl DiskOperationMetrics {
     }
 }
 
+impl TimedActionSlot {
+    fn snapshot(&self) -> Option<(u64, u64, u64)> {
+        let version = self.version.load(Ordering::Acquire);
+        if !version.is_multiple_of(2) {
+            return None;
+        }
+
+        // The first Acquire load publishes the payload written before the
+        // matching Release store. Relaxed payload loads are sufficient while
+        // the final Acquire version load validates that no writer intervened.
+        let slot_sec = self.unix_sec.load(Ordering::Relaxed);
+        let count = self.count.load(Ordering::Relaxed);
+        let acc_time = self.acc_time.load(Ordering::Relaxed);
+        (self.version.load(Ordering::Acquire) == version).then_some((slot_sec, count, acc_time))
+    }
+}
+
 pub(crate) struct DiskHealthWaitingGuard<'a> {
     health: &'a DiskHealthTracker,
 }
@@ -1097,6 +1190,37 @@ impl LocalDiskWrapper {
         )
     }
 
+    /// Run a delete under an owned coordinator task when a publication guard
+    /// is present. This keeps the guard alive if the RPC waiter is cancelled
+    /// while the local namespace mutation is still in progress.
+    pub(crate) async fn delete_with_publication_guard(
+        &self,
+        volume: &str,
+        path: &str,
+        options: DeleteOptions,
+        external_guard: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<()> {
+        let operation = self.clone();
+        let volume = volume.to_owned();
+        let path = path.to_owned();
+        let timeout_duration = if external_guard.is_some() {
+            Duration::ZERO
+        } else {
+            get_max_timeout_duration()
+        };
+        run_owned_mutation(external_guard, move || async move {
+            operation
+                .track_disk_health_mutation(
+                    "delete",
+                    DiskMetricMutation::Delete,
+                    || async { operation.disk.delete(&volume, &path, options).await },
+                    timeout_duration,
+                )
+                .await
+        })
+        .await
+    }
+
     pub(crate) fn new_with_reconnect_state(
         disk: Arc<LocalDisk>,
         health_check: bool,
@@ -2247,6 +2371,44 @@ mod tests {
     };
     use tokio::io::AsyncWrite;
 
+    struct DropProbe(Arc<std::sync::atomic::AtomicUsize>);
+
+    impl Drop for DropProbe {
+        fn drop(&mut self) {
+            self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+        }
+    }
+
+    #[tokio::test]
+    async fn owned_mutation_keeps_publication_guard_after_waiter_cancellation() {
+        let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let guard: Arc<dyn Send + Sync> = Arc::new(DropProbe(Arc::clone(&drops)));
+        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
+        let (finished_tx, finished_rx) = tokio::sync::oneshot::channel();
+
+        let waiter = tokio::spawn(run_owned_mutation(Some(guard), move || async move {
+            started_tx.send(()).expect("mutation should signal start");
+            release_rx.await.expect("mutation should be released");
+            finished_tx.send(()).expect("mutation should signal completion");
+            Ok::<_, Error>(())
+        }));
+
+        started_rx.await.expect("mutation owner should start");
+        waiter.abort();
+        assert_eq!(drops.load(std::sync::atomic::Ordering::SeqCst), 0);
+
+        release_tx.send(()).expect("mutation owner should still be alive");
+        finished_rx.await.expect("mutation owner should finish");
+        tokio::time::timeout(Duration::from_secs(1), async {
+            while drops.load(std::sync::atomic::Ordering::SeqCst) == 0 {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("publication guard should be released after mutation completion");
+    }
+
     struct PendingWriter;
 
     #[test]
@@ -2273,6 +2435,25 @@ mod tests {
         assert_eq!(window.acc_time, 18_000);
     }
 
+    #[test]
+    fn timed_action_slot_snapshot_skips_writer_owned_slot() {
+        let slot = TimedActionSlot::default();
+        slot.unix_sec.store(70, Ordering::Relaxed);
+        slot.count.store(2, Ordering::Relaxed);
+        slot.acc_time.store(18_000, Ordering::Relaxed);
+        slot.version.store(2, Ordering::Release);
+        assert_eq!(slot.snapshot(), Some((70, 2, 18_000)));
+
+        assert_eq!(slot.version.compare_exchange(2, 3, Ordering::AcqRel, Ordering::Relaxed), Ok(2));
+        slot.unix_sec.store(71, Ordering::Relaxed);
+        slot.count.store(1, Ordering::Relaxed);
+        slot.acc_time.store(11_000, Ordering::Relaxed);
+        assert_eq!(slot.snapshot(), None);
+
+        slot.version.store(4, Ordering::Release);
+        assert_eq!(slot.snapshot(), Some((71, 1, 11_000)));
+    }
+
     #[test]
     fn disk_health_metrics_snapshot_exports_waiting_errors_and_operation_windows() {
         let metrics = DiskHealthMetricEpoch::default();

crates/ecstore/src/disk/error.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_PUT_FILE_STREAM;
 use rustfs_rio::{InternodeHttpError, InternodeHttpErrorKind};
 use std::error::Error as StdError;
 use std::hash::{Hash, Hasher};
@@ -22,6 +23,7 @@ pub type Error = DiskError;
 pub type Result<T> = core::result::Result<T, Error>;
 
 const METACACHE_OUTPUT_STREAM_CLOSED: &str = "metacache output stream closed";
+pub(crate) const HEAL_DANGLING_DELETE_GRACE_MESSAGE: &str = "dangling object deletion deferred by heal grace window";
 
 /// Marker carried by a shard-read `io::Error` when the underlying reader can
 /// no longer be realigned after a fresh remote open failed.  The marker is
@@ -33,6 +35,12 @@ pub(crate) struct TerminalReadError {
     source: DiskError,
 }
 
+#[derive(Debug)]
+struct DanglingDeleteGraceError {
+    retry_after_secs: i64,
+    grace_secs: i64,
+}
+
 // DiskError == StorageErr
 #[derive(Debug, thiserror::Error)]
 pub enum DiskError {
@@ -200,6 +208,18 @@ impl StdError for TerminalReadError {
     }
 }
 
+impl std::fmt::Display for DanglingDeleteGraceError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{HEAL_DANGLING_DELETE_GRACE_MESSAGE}; retry_after_secs={}; grace_secs={}",
+            self.retry_after_secs, self.grace_secs
+        )
+    }
+}
+
+impl StdError for DanglingDeleteGraceError {}
+
 fn classify_internode_missing_error(error: &InternodeHttpError) -> Option<DiskError> {
     if error.is_remote_file_not_found() {
         return Some(DiskError::FileNotFound);
@@ -210,6 +230,19 @@ fn classify_internode_missing_error(error: &InternodeHttpError) -> Option<DiskEr
     None
 }
 
+fn internode_write_error_is_retryable(error: &InternodeHttpError) -> bool {
+    error.kind().is_retryable()
+        || (matches!(error.kind(), InternodeHttpErrorKind::HttpStatus(status) if status.as_u16() == 409)
+            && error.context().operation() == Some(INTERNODE_OPERATION_PUT_FILE_STREAM))
+}
+
+fn io_error_contains_retryable_internode_write(error: &io::Error) -> bool {
+    error
+        .get_ref()
+        .and_then(|source| source.downcast_ref::<InternodeHttpError>())
+        .is_some_and(internode_write_error_is_retryable)
+}
+
 /// Wrap a terminal shard-read failure without changing its typed
 /// classification.  Timeout-like disk errors retain `TimedOut`; other errors
 /// retain their inner I/O kind or use `Other` when no more specific kind exists.
@@ -253,6 +286,24 @@ impl DiskError {
         DiskError::Io(std::io::Error::other(error))
     }
 
+    pub(crate) fn dangling_delete_grace(retry_after_secs: i64, grace_secs: i64) -> Self {
+        DiskError::other(DanglingDeleteGraceError {
+            retry_after_secs,
+            grace_secs,
+        })
+    }
+
+    pub fn is_dangling_delete_grace(&self) -> bool {
+        matches!(self, DiskError::Io(io_error) if Self::io_error_is_dangling_delete_grace(io_error))
+    }
+
+    pub fn io_error_is_dangling_delete_grace(io_error: &io::Error) -> bool {
+        io_error
+            .get_ref()
+            .is_some_and(|source| source.downcast_ref::<DanglingDeleteGraceError>().is_some())
+            || io_error.to_string().contains(HEAL_DANGLING_DELETE_GRACE_MESSAGE)
+    }
+
     pub(crate) fn metacache_output_stream_closed() -> Self {
         DiskError::Io(std::io::Error::new(std::io::ErrorKind::BrokenPipe, METACACHE_OUTPUT_STREAM_CLOSED))
     }
@@ -299,10 +350,7 @@ impl DiskError {
 
     pub fn is_retryable_internode_write_failure(&self) -> bool {
         match self {
-            DiskError::Io(io_error) => io_error
+            DiskError::Io(io_error) => io_error_contains_retryable_internode_write(io_error),
-                .get_ref()
-                .and_then(|source| source.downcast_ref::<InternodeHttpError>())
-                .is_some_and(|err| err.kind().is_retryable()),
             _ => false,
         }
     }
@@ -1203,6 +1251,68 @@ mod tests {
         assert!(!DiskError::FileNotFound.is_internode_http_status(429));
     }
 
+    #[test]
+    fn test_put_file_server_epoch_conflict_is_retryable_write_failure() {
+        let conflict = DiskError::from(rustfs_rio::new_test_internode_http_io_error(
+            rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::CONFLICT),
+        ));
+        let bad_request = DiskError::from(rustfs_rio::new_test_internode_http_io_error(
+            rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::BAD_REQUEST),
+        ));
+
+        assert!(conflict.is_retryable_internode_write_failure());
+        assert!(!bad_request.is_retryable_internode_write_failure());
+    }
+
+    #[tokio::test]
+    async fn read_stream_conflict_is_not_a_retryable_put_file_failure() {
+        use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
+        tokio::time::timeout(std::time::Duration::from_secs(5), async {
+            let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
+                .await
+                .expect("bind isolated HTTP fixture");
+            let address = listener.local_addr().expect("fixture address");
+            let server = tokio::spawn(async move {
+                let (mut stream, _) = listener.accept().await.expect("accept read request");
+                let mut request = [0_u8; 4096];
+                let mut read = 0;
+                loop {
+                    let count = stream.read(&mut request[read..]).await.expect("read HTTP request");
+                    assert!(count > 0, "request ended before its complete headers");
+                    read += count;
+                    if request[..read].windows(4).any(|bytes| bytes == b"\r\n\r\n") {
+                        break;
+                    }
+                    assert!(read < request.len(), "fixture request headers exceed their budget");
+                }
+                stream
+                    .write_all(b"HTTP/1.1 409 Conflict\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
+                    .await
+                    .expect("send typed conflict response");
+            });
+            let error = match rustfs_rio::HttpReader::new(
+                format!("http://{address}/rustfs/rpc/read_file_stream"),
+                http::Method::GET,
+                http::HeaderMap::new(),
+                None,
+            )
+            .await
+            {
+                Ok(_) => panic!("HTTP 409 must fail the read"),
+                Err(error) => DiskError::from(error),
+            };
+            server.await.expect("fixture task should complete");
+            assert!(error.is_internode_http_status(409));
+            assert!(
+                !error.is_retryable_internode_write_failure(),
+                "read-operation 409 must not trigger put-file retry"
+            );
+        })
+        .await
+        .expect("isolated read-conflict test must finish within its budget");
+    }
+
     #[test]
     fn test_internode_missing_errors_preserve_disk_error_types() {
         let file_missing = DiskError::from(rustfs_rio::new_test_remote_file_not_found_http_io_error());

crates/ecstore/src/disk/local.rs

@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::config::storageclass::DEFAULT_INLINE_BLOCK;
 use crate::crash_inject::{self, CrashPoint};
 use crate::data_usage::local_snapshot::ensure_data_usage_layout;
 use crate::diagnostics::get::{
@@ -3149,9 +3148,10 @@ impl LocalIoBackend for StdBackend {
                 direct_read_copy_fault_delta: MmapPageFaultDelta,
                 blocking_task_duration: StdDuration,
                 used_direct_io: bool,
-                /// The descriptor opened by THIS call (None on a cache hit), handed
+                /// The descriptor and size snapshot opened by THIS call (None on a
-                /// back so the async caller can index it in the fd cache.
+                /// cache hit), handed back so the async caller can index it in the
-                opened_fd: Option<Arc<std::fs::File>>,
+                /// fd cache.
+                opened_fd: Option<Arc<FdCacheEntry>>,
             }
 
             enum MmapCopyReadError {
@@ -3198,12 +3198,12 @@ impl LocalIoBackend for StdBackend {
                 (cache, key, gen_at_open)
             });
             #[cfg(target_os = "linux")]
-            let cached_fd: Option<Arc<std::fs::File>> = match &fd_lookup {
+            let cached_fd: Option<Arc<FdCacheEntry>> = match &fd_lookup {
                 Some((cache, key, _)) => cache.get(key).await,
                 None => None,
             };
             #[cfg(not(target_os = "linux"))]
-            let cached_fd: Option<Arc<std::fs::File>> = None;
+            let cached_fd: Option<Arc<FdCacheEntry>> = None;
 
             let blocking_wait_start = metrics_enabled.then(std::time::Instant::now);
             let read_result = tokio::task::spawn_blocking(move || {
@@ -3225,8 +3225,15 @@ impl LocalIoBackend for StdBackend {
                 // the read below is positioned (mmap offset argument / `read_exact_at`)
                 // and never depends on the descriptor's current offset. `cached_fd` being
                 // None also marks this call as a miss for the cache-insert side-channel.
-                let (file, access_check_duration) = if let Some(cached) = cached_fd.as_ref() {
+                // The cached length is the metadata snapshot captured at open time;
-                    (cached.as_ref().try_clone().map_err(DiskError::from)?, StdDuration::ZERO)
+                // all in-place/replacement writers invalidate this entry before
+                // publishing a mutation, so cache hits avoid a redundant fstat.
+                let (file, cached_len, access_check_duration) = if let Some(cached) = cached_fd.as_ref() {
+                    (
+                        cached.file.as_ref().try_clone().map_err(DiskError::from)?,
+                        Some(cached.len),
+                        StdDuration::ZERO,
+                    )
                 } else {
                     // Measure the volume access probe only — the part-path resolution
                     // above is accounted in `path_resolve_duration` (rustfs/backlog#1801).
@@ -3237,20 +3244,27 @@ impl LocalIoBackend for StdBackend {
                             .map_err(|e| DiskError::from(to_access_error(e, DiskError::VolumeAccessDenied)))?;
                     }
                     let access_check_duration = access_check_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
-                    (std::fs::File::open(&file_path).map_err(DiskError::from)?, access_check_duration)
+                    (std::fs::File::open(&file_path).map_err(DiskError::from)?, None, access_check_duration)
                 };
                 let file_open_duration = file_open_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
 
-                let metadata_lookup_start = metrics_enabled.then(StdInstant::now);
+                let (metadata_len, metadata_lookup_duration) = if let Some(len) = cached_len {
-                // On a cache hit this fstats the cached descriptor — the inode it was
+                    // Reuse the open-time metadata snapshot on a cache hit. The
-                // opened against, which invalidation keeps current for live entries. EC
+                    // generation fence and mutation invalidation keep this value
-                // shards are fixed-length, so a still-cached pre-heal length is benign.
+                    // tied to the inode held by `file`.
-                let meta = file.metadata().map_err(DiskError::from)?;
+                    (len, StdDuration::ZERO)
-                let metadata_lookup_duration = metadata_lookup_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
+                } else {
+                    let metadata_lookup_start = metrics_enabled.then(StdInstant::now);
+                    let meta = file.metadata().map_err(DiskError::from)?;
+                    let duration = metadata_lookup_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
+                    (meta.len(), duration)
+                };
 
                 let metadata_validate_start = metrics_enabled.then(StdInstant::now);
-                if meta.len() < end_offset_u64 {
+                if metadata_len < end_offset_u64 {
-                    return Err(MmapCopyReadError::OutOfBounds { actual_size: meta.len() });
+                    return Err(MmapCopyReadError::OutOfBounds {
+                        actual_size: metadata_len,
+                    });
                 }
                 let metadata_validate_duration =
                     metadata_validate_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
@@ -3396,9 +3410,14 @@ impl LocalIoBackend for StdBackend {
                 // Arc; `cached_fd.is_none()` is true exactly when this call did the open.
                 // Non-Linux has no fd cache, so skip the Arc allocation there.
                 #[cfg(target_os = "linux")]
-                let opened_fd: Option<Arc<std::fs::File>> = cached_fd.is_none().then(|| Arc::new(file));
+                let opened_fd: Option<Arc<FdCacheEntry>> = cached_fd.is_none().then(|| {
+                    Arc::new(FdCacheEntry {
+                        file: Arc::new(file),
+                        len: metadata_len,
+                    })
+                });
                 #[cfg(not(target_os = "linux"))]
-                let opened_fd: Option<Arc<std::fs::File>> = None;
+                let opened_fd: Option<Arc<FdCacheEntry>> = None;
 
                 Ok::<MmapCopyReadResult, MmapCopyReadError>(MmapCopyReadResult {
                     bytes,
@@ -3521,7 +3540,7 @@ impl LocalIoBackend for StdBackend {
                     }
                 }
             }
-            // Index the freshly opened descriptor for future cache hits
+            // Index the freshly opened descriptor and metadata snapshot for future cache hits
             // (rustfs/backlog#1801). `insert_if_fresh` refuses to cache if an
             // invalidation (heal/delete/rename) bumped the generation between the
             // open snapshot and now, so a stale pre-mutation inode is never served
@@ -3873,6 +3892,18 @@ struct FdKey {
     direct: bool,
 }
 
+/// Descriptor and immutable size snapshot retained for one cached shard inode.
+///
+/// The generation fence and explicit mutation invalidation keep the snapshot
+/// tied to the inode held by `file`, allowing cache hits to avoid a repeated
+/// metadata syscall without weakening replacement/heal semantics.
+struct FdCacheEntry {
+    /// An independently cloneable descriptor for the immutable shard inode.
+    file: Arc<std::fs::File>,
+    /// File length captured together with the descriptor.
+    len: u64,
+}
+
 /// Per-disk cache of open descriptors for io_uring reads (backlog#1145).
 ///
 /// Why this exists: `pread_uring` opened the file on the blocking pool for every
@@ -3902,7 +3933,7 @@ struct FdKey {
 /// the descriptor once no in-flight read still holds it.
 #[cfg(target_os = "linux")]
 struct FdCache {
-    cache: moka::future::Cache<FdKey, Arc<std::fs::File>>,
+    cache: moka::future::Cache<FdKey, Arc<FdCacheEntry>>,
     /// Bumped by every invalidation. A miss-path open snapshots this before it
     /// opens and refuses to insert if it moved, so an fd opened before a
     /// heal/delete commit can never be resurrected into the cache after the
@@ -3932,7 +3963,7 @@ impl FdCache {
         }
     }
 
-    async fn get(&self, key: &FdKey) -> Option<Arc<std::fs::File>> {
+    async fn get(&self, key: &FdKey) -> Option<Arc<FdCacheEntry>> {
         self.cache.get(key).await
     }
 
@@ -3947,11 +3978,11 @@ impl FdCache {
     /// open bumped the generation, so a stale pre-heal/pre-delete inode is never
     /// cached. The post-insert re-check closes the tiny window where an
     /// invalidate races the insert itself, by removing the entry we just added.
-    async fn insert_if_fresh(&self, key: FdKey, file: Arc<std::fs::File>, gen_at_open: u64) {
+    async fn insert_if_fresh(&self, key: FdKey, entry: Arc<FdCacheEntry>, gen_at_open: u64) {
         if self.generation.load(Ordering::Acquire) != gen_at_open {
             return;
         }
-        self.cache.insert(key.clone(), file).await;
+        self.cache.insert(key.clone(), entry).await;
         if self.generation.load(Ordering::Acquire) != gen_at_open {
             self.cache.invalidate(&key).await;
         }
@@ -3987,7 +4018,7 @@ impl FdCache {
         self.generation.fetch_add(1, Ordering::AcqRel);
         let volume = volume.to_owned();
         let prefix = prefix.trim_end_matches('/').to_owned();
-        let matches = move |k: &FdKey, _: &Arc<std::fs::File>| {
+        let matches = move |k: &FdKey, _: &Arc<FdCacheEntry>| {
             k.volume == volume && (k.path == prefix || k.path.strip_prefix(&prefix).is_some_and(|r| r.starts_with('/')))
         };
         if self.cache.invalidate_entries_if(matches).is_err() {
@@ -4003,7 +4034,7 @@ impl FdCache {
     fn invalidate_volume(&self, volume: &str) {
         self.generation.fetch_add(1, Ordering::AcqRel);
         let volume = volume.to_owned();
-        let matches = move |k: &FdKey, _: &Arc<std::fs::File>| k.volume == volume;
+        let matches = move |k: &FdKey, _: &Arc<FdCacheEntry>| k.volume == volume;
         if self.cache.invalidate_entries_if(matches).is_err() {
             self.cache.invalidate_all();
         }
@@ -4021,7 +4052,8 @@ impl FdCache {
     /// tests that drive the cache directly.
     #[cfg(test)]
     async fn insert(&self, key: FdKey, file: Arc<std::fs::File>) {
-        self.cache.insert(key, file).await;
+        let len = file.metadata().map(|metadata| metadata.len()).unwrap_or_default();
+        self.cache.insert(key, Arc::new(FdCacheEntry { file, len })).await;
     }
 
     #[cfg(test)]
@@ -4400,7 +4432,12 @@ impl UringBackend {
         };
 
         let file = match cached {
-            Some(file) => file,
+            Some(entry) => {
+                if entry.len < u64::try_from(end_offset).map_err(|_| DiskError::FileCorrupt)? {
+                    return Err(DiskError::FileCorrupt);
+                }
+                Arc::clone(&entry.file)
+            }
             None => {
                 // Snapshot the cache generation BEFORE opening (rustfs/backlog#1176):
                 // if a heal/delete invalidation runs while this open is in flight,
@@ -4410,7 +4447,7 @@ impl UringBackend {
                 let root = self.root.clone();
                 let volume_owned = volume.to_owned();
                 let path_owned = path.to_owned();
-                let file = tokio::task::spawn_blocking(move || -> Result<std::fs::File> {
+                let (file, len) = tokio::task::spawn_blocking(move || -> Result<(std::fs::File, u64)> {
                     let file_path = resolve_uring_object_path(&root, &volume_owned, &path_owned)?;
                     let file = std::fs::File::open(&file_path).map_err(DiskError::from)?;
                     let meta = file.metadata().map_err(DiskError::from)?;
@@ -4418,30 +4455,22 @@ impl UringBackend {
                     if meta.len() < end_offset_u64 {
                         return Err(DiskError::FileCorrupt);
                     }
-                    Ok(file)
+                    Ok((file, meta.len()))
                 })
                 .await
                 .map_err(|e| DiskError::other(format!("uring pread join error: {e}")))??;
-                let file = Arc::new(file);
+                let file = Arc::new(FdCacheEntry {
+                    file: Arc::new(file),
+                    len,
+                });
                 if let (Some((cache, key)), Some(gen_at_open)) = (cache_entry, gen_at_open) {
                     cache.insert_if_fresh(key, Arc::clone(&file), gen_at_open).await;
                 }
-                file
+                file.file.clone()
             }
         };
 
         if length == 0 {
-            // Parity with StdBackend and the miss path (rustfs/backlog#1173): a
-            // zero-length read still rejects an offset past EOF. The miss path
-            // validated `meta.len() < end_offset` (end_offset == offset here), but
-            // a cache hit skipped it — so fstat the descriptor and match. This is
-            // a rare path (callers do not issue zero-length reads), so the one
-            // extra fstat is negligible.
-            match file.metadata() {
-                Ok(meta) if offset_u64 > meta.len() => return Err(DiskError::FileCorrupt),
-                Ok(_) => {}
-                Err(e) => return Err(DiskError::from(e)),
-            }
             return Ok(Bytes::new());
         }
 
@@ -10410,8 +10439,14 @@ impl DiskAPI for LocalDisk {
                 fi.data = None;
             }
 
-            let inline = fi.transition_status.is_empty() && fi.data_dir.is_some() && fi.parts.len() == 1;
+            // Keep this compatibility read-ahead decision on the same policy
-            if inline && fi.shard_file_size(fi.parts[0].actual_size) < DEFAULT_INLINE_BLOCK as i64 {
+            // as PUT's inline admission. In particular, do not use the old
+            // fixed 128 KiB shard limit: a non-inline object in a wider EC
+            // layout can have a smaller shard and would otherwise be copied
+            // out of part.1 during every metadata read. Such objects remain
+            // fully readable through the normal EC reader below.
+            let storage_class_config = runtime_sources::storage_class_config_snapshot();
+            if should_read_legacy_inline_part(&fi, storage_class_config.as_ref()) {
                 let part_path = path_join_buf(&[
                     path,
                     fi.data_dir.map_or_else(|| "".to_string(), |dir| dir.to_string()).as_str(),
@@ -10913,6 +10948,21 @@ impl DiskAPI for LocalDisk {
     }
 }
 
+/// Whether a legacy object without the inline marker should have its external
+/// part materialized into `FileInfo.data` for compatibility with the old GET
+/// fast path. The marker-bearing path is handled by `read_raw`/`get_file_info`;
+/// this is only a conservative fallback for old metadata.
+fn should_read_legacy_inline_part(fi: &FileInfo, storage_class_config: &crate::config::storageclass::Config) -> bool {
+    if !fi.transition_status.is_empty() || fi.data_dir.is_none() || fi.parts.len() != 1 || fi.inline_data() {
+        return false;
+    }
+
+    let part = &fi.parts[0];
+    let shard_size = fi.shard_file_size(part.actual_size);
+    let versioned = fi.versioned || fi.version_id.is_some_and(|version_id| !version_id.is_nil());
+    storage_class_config.should_inline(shard_size, fi.erasure.data_blocks, versioned)
+}
+
 impl LocalDisk {
     pub(crate) async fn rename_data_borrowed(
         &self,
@@ -11049,6 +11099,46 @@ mod test {
         file_info
     }
 
+    #[test]
+    fn legacy_inline_read_ahead_matches_writer_policy_for_ec_layouts() {
+        let config = crate::config::storageclass::Config::default();
+        let object_sizes = [128 * 1024_i64, 256 * 1024, 512 * 1024, 1024 * 1024, 4 * 1024 * 1024];
+
+        for (data_shards, parity_shards) in [(8, 4), (12, 4)] {
+            for object_size in object_sizes {
+                let mut fi = FileInfo::new("object", data_shards, parity_shards);
+                fi.data_dir = Some(Uuid::from_u128(1));
+                fi.parts = vec![ObjectPartInfo {
+                    number: 1,
+                    size: usize::try_from(object_size).expect("test object size should fit usize"),
+                    actual_size: object_size,
+                    ..Default::default()
+                }];
+
+                let writer_decision = config.should_inline(fi.shard_file_size(object_size), data_shards, false);
+                assert_eq!(
+                    should_read_legacy_inline_part(&fi, &config),
+                    writer_decision,
+                    "legacy read-ahead must match PUT for EC{data_shards}+{parity_shards}, size={object_size}"
+                );
+            }
+        }
+
+        let mut ec12 = FileInfo::new("object", 12, 4);
+        ec12.data_dir = Some(Uuid::from_u128(1));
+        ec12.parts = vec![ObjectPartInfo {
+            number: 1,
+            size: 1024 * 1024,
+            actual_size: 1024 * 1024,
+            ..Default::default()
+        }];
+        assert!(
+            ec12.shard_file_size(1024 * 1024) < crate::config::storageclass::DEFAULT_INLINE_BLOCK as i64,
+            "the regression guard must exercise the old fixed 128 KiB read-ahead boundary"
+        );
+        assert!(!should_read_legacy_inline_part(&ec12, &config));
+    }
+
     fn test_meta(fi: FileInfo) -> Vec<u8> {
         let mut meta = FileMeta::default();
         meta.add_version(fi).expect("test metadata should accept file info");
@@ -21347,11 +21437,10 @@ mod test {
 
     /// Zero-length read bounds parity on the cache-HIT path (backlog#1173/#1180).
     /// A `length == 0` read past EOF must be rejected identically whether the
-    /// descriptor is freshly opened (miss path) or served from the cache: the
+    /// descriptor is freshly opened (miss path) or served from the cache. Seeds
-    /// cache-hit branch fstats the descriptor to reproduce the miss path's
+    /// the cache with a normal read so the zero-length reads reuse the same
-    /// `offset > len` check instead of returning empty unconditionally. Seeds
+    /// open-time size snapshot, then pins that UringBackend and StdBackend agree
-    /// the cache with a normal read so the zero-length reads are hits, then pins
+    /// on every case.
-    /// that UringBackend and StdBackend agree on every case.
     #[cfg(target_os = "linux")]
     #[tokio::test(flavor = "multi_thread")]
     async fn uring_zero_length_read_bounds_match_std_on_cache_hit() {

crates/ecstore/src/disk/mod.rs

@@ -677,15 +677,20 @@ impl DiskAPI for Disk {
 }
 
 impl Disk {
-    pub(crate) async fn delete_with_scanner_publication_lease(
+    pub async fn delete_with_scanner_publication_lease_and_guard(
         &self,
         volume: &str,
         path: &str,
         opts: DeleteOptions,
         scanner_publication_lease_token: Option<Uuid>,
+        external_guard: Option<Arc<dyn Send + Sync>>,
     ) -> Result<()> {
         match self {
-            Disk::Local(local_disk) => local_disk.delete(volume, path, opts).await,
+            Disk::Local(local_disk) => {
+                local_disk
+                    .delete_with_publication_guard(volume, path, opts, external_guard)
+                    .await
+            }
             Disk::Remote(remote_disk) => {
                 remote_disk
                     .delete_with_scanner_publication_lease(volume, path, opts, scanner_publication_lease_token)
@@ -714,11 +719,34 @@ impl Disk {
         dst_volume: &str,
         dst_path: &str,
         scanner_publication_lease_token: Option<Uuid>,
+    ) -> Result<RenameDataResp> {
+        self.rename_data_borrowed_with_fence_and_guard(
+            src_volume,
+            src_path,
+            fi,
+            dst_volume,
+            dst_path,
+            scanner_publication_lease_token,
+            None,
+        )
+        .await
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub async fn rename_data_borrowed_with_fence_and_guard(
+        &self,
+        src_volume: &str,
+        src_path: &str,
+        fi: &FileInfo,
+        dst_volume: &str,
+        dst_path: &str,
+        scanner_publication_lease_token: Option<Uuid>,
+        external_guard: Option<Arc<dyn Send + Sync>>,
     ) -> Result<RenameDataResp> {
         match self {
             Disk::Local(local_disk) => {
                 local_disk
-                    .rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
+                    .rename_data_borrowed_with_guard(src_volume, src_path, fi, dst_volume, dst_path, external_guard)
                     .await
             }
             Disk::Remote(remote_disk) => {

crates/ecstore/src/ecstore_validation_blackbox.rs

@@ -16,10 +16,143 @@ use rustfs_filemeta::{MetacacheReader, MetacacheWriter};
 use std::io::Cursor;
 use std::path::PathBuf;
 use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
 use tokio::fs;
 use tokio::io::AsyncReadExt;
 use tokio::sync::RwLock;
 
+/// Test-only lock client whose refresh path can be rejected independently of
+/// every other lock operation. The observed event is awaitable so lock-loss
+/// tests do not depend on sleeps or scheduler timing.
+#[derive(Debug)]
+pub(crate) struct RefreshLossLockClient {
+    inner: rustfs_lock::LocalClient,
+    reject_refresh: AtomicBool,
+    rejected_refresh: AtomicBool,
+    rejected_refresh_notify: tokio::sync::Notify,
+}
+
+impl RefreshLossLockClient {
+    pub(crate) fn with_manager(manager: Arc<rustfs_lock::GlobalLockManager>) -> Self {
+        Self {
+            inner: rustfs_lock::LocalClient::with_manager(manager),
+            reject_refresh: AtomicBool::new(false),
+            rejected_refresh: AtomicBool::new(false),
+            rejected_refresh_notify: tokio::sync::Notify::new(),
+        }
+    }
+
+    pub(crate) fn reject_refreshes(&self) {
+        self.reject_refresh.store(true, Ordering::Release);
+    }
+
+    pub(crate) fn refreshes_rejected(&self) -> bool {
+        self.rejected_refresh.load(Ordering::Acquire)
+    }
+
+    pub(crate) async fn wait_for_rejected_refresh(
+        &self,
+        timeout: std::time::Duration,
+    ) -> std::result::Result<(), tokio::time::error::Elapsed> {
+        tokio::time::timeout(timeout, async {
+            loop {
+                let notified = self.rejected_refresh_notify.notified();
+                if self.refreshes_rejected() {
+                    return;
+                }
+                notified.await;
+            }
+        })
+        .await
+    }
+}
+
+#[async_trait::async_trait]
+impl rustfs_lock::LockClient for RefreshLossLockClient {
+    async fn acquire_lock(&self, request: &rustfs_lock::LockRequest) -> rustfs_lock::Result<rustfs_lock::LockResponse> {
+        rustfs_lock::LockClient::acquire_lock(&self.inner, request).await
+    }
+
+    async fn release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
+        rustfs_lock::LockClient::release(&self.inner, lock_id).await
+    }
+
+    async fn refresh(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
+        if self.reject_refresh.load(Ordering::Acquire) {
+            self.rejected_refresh.store(true, Ordering::Release);
+            self.rejected_refresh_notify.notify_waiters();
+            return Ok(false);
+        }
+        rustfs_lock::LockClient::refresh(&self.inner, lock_id).await
+    }
+
+    async fn force_release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
+        rustfs_lock::LockClient::force_release(&self.inner, lock_id).await
+    }
+
+    async fn check_status(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<Option<rustfs_lock::LockInfo>> {
+        rustfs_lock::LockClient::check_status(&self.inner, lock_id).await
+    }
+
+    async fn list_lock_leases(&self) -> Vec<rustfs_lock::LockLeaseInfo> {
+        rustfs_lock::LockClient::list_lock_leases(&self.inner).await
+    }
+
+    async fn get_stats(&self) -> rustfs_lock::Result<rustfs_lock::LockStats> {
+        rustfs_lock::LockClient::get_stats(&self.inner).await
+    }
+
+    async fn close(&self) -> rustfs_lock::Result<()> {
+        rustfs_lock::LockClient::close(&self.inner).await
+    }
+
+    async fn is_online(&self) -> bool {
+        rustfs_lock::LockClient::is_online(&self.inner).await
+    }
+
+    async fn is_local(&self) -> bool {
+        rustfs_lock::LockClient::is_local(&self.inner).await
+    }
+}
+
+#[tokio::test]
+async fn refresh_loss_lock_client_keeps_rejection_observable_for_late_waiters() {
+    let manager = Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
+        rustfs_lock::FastObjectLockManager::new(),
+    )));
+    let client = RefreshLossLockClient::with_manager(manager);
+    let resource = rustfs_lock::ObjectKey::new("bucket", "object");
+    let response = rustfs_lock::LockClient::acquire_lock(
+        &client,
+        &rustfs_lock::LockRequest::new(resource, rustfs_lock::LockType::Shared, "refresh-loss-harness"),
+    )
+    .await
+    .expect("acquire should reach the inner local client");
+    let lock_id = response.lock_info.expect("the inner local client should acquire the lock").id;
+    assert_eq!(
+        rustfs_lock::LockClient::list_lock_leases(&client).await.len(),
+        1,
+        "lease diagnostics must remain transparent through the refresh wrapper"
+    );
+
+    client.reject_refreshes();
+    assert!(
+        !rustfs_lock::LockClient::refresh(&client, &lock_id)
+            .await
+            .expect("refresh should return a response")
+    );
+    client
+        .wait_for_rejected_refresh(std::time::Duration::from_millis(50))
+        .await
+        .expect("a waiter registered after rejection must still observe the event");
+    assert!(client.refreshes_rejected());
+    assert!(
+        rustfs_lock::LockClient::release(&client, &lock_id)
+            .await
+            .expect("release should reach the inner local client")
+    );
+}
+
 /// Returns the backing [`tempfile::TempDir`]s alongside the set so callers keep
 /// them alive for the test's duration and the directories are removed on drop.
 pub(crate) async fn make_local_set_disks(drive_count: usize, parity_count: usize) -> (Vec<tempfile::TempDir>, Arc<SetDisks>) {

crates/ecstore/src/erasure/coding/decode.rs

@@ -46,6 +46,7 @@ type ShardReadFuture<'a> = Pin<Box<dyn Future<Output = (usize, ShardReadCost, Re
 type OwnedShardReadFuture<'a, R> =
     Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<Vec<u8>, Error>, Option<BitrotReader<R>>, bool)> + Send + 'a>>;
 pub(crate) type DeferredReaderReopener<R> = Arc<dyn Fn(usize) -> Option<BitrotReader<R>> + Send + Sync>;
+pub(crate) type DecodeOutcome = (usize, Option<std::io::Error>, bool);
 
 type ShardIndexes = SmallVec<[usize; INLINE_SHARD_SLOTS]>;
 type ActiveReaders = SmallVec<[bool; INLINE_SHARD_SLOTS]>;
@@ -574,6 +575,7 @@ pub(crate) struct ParallelReader<R> {
     read_timeout: Duration,
     verify_reconstruction: bool,
     locality_preference_enabled: bool,
+    demand_bound_lockstep: bool,
     // Request-scoped shard buffers keyed by shard index. Keeping ownership in
     // `ParallelReader` avoids dropping unused parity/backup slot buffers between stripes.
     buffers: ShardBufferPool,
@@ -585,10 +587,8 @@ pub(crate) struct ParallelReader<R> {
     // it to the current stripe when it is engaged mid-object (backlog#923).
     engaged: SmallVec<[bool; INLINE_SHARD_SLOTS]>,
     deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
-    // Copy-source hedges use a fresh deferred reader so cancelling a hedge
+    // Demand-bound hedges use a fresh deferred reader so cancelling a hedge
-    // never consumes the unopened reader reserved for a later stripe. The
+    // never consumes the unopened reader reserved for a later stripe.
-    // vector is empty for callers that do not provide a reopen factory (tests
-    // and the ordinary GET path retain the handle-based behavior).
     deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
     stripe_index: usize,
 }
@@ -777,9 +777,9 @@ where
         // reads all live readers on every stripe — the pre-backlog#923
         // behavior. With the gate on, only data slots start engaged; parity is
         // engaged on demand, stripe-aligned through its deferred handle.
-        let data_shards_only = get_lockstep_data_shards_only_enabled();
+        let demand_bound_lockstep = get_lockstep_data_shards_only_enabled();
         let engaged: SmallVec<_> = (0..readers.len())
-            .map(|index| !data_shards_only || index < e.data_shards)
+            .map(|index| !demand_bound_lockstep || index < e.data_shards)
             .collect();
         ParallelReader {
             readers,
@@ -793,6 +793,7 @@ where
             read_timeout,
             verify_reconstruction,
             locality_preference_enabled: get_shard_locality_preference_enabled(),
+            demand_bound_lockstep,
             buffers: ShardBufferPool::new(e.data_shards + e.parity_shards),
             stripe_state: None,
             engaged,
@@ -1275,7 +1276,7 @@ where
     /// realigned (no pending deferred handle) is likewise retired instead of
     /// being read out of position.
     async fn read_lockstep(&mut self, state: &mut StripeReadState) {
-        if matches!(decode_read_policy(), DecodeReadPolicy::DemandBound) {
+        if self.demand_bound_lockstep {
             self.read_lockstep_demand_bound(state).await;
             return;
         }
@@ -1531,17 +1532,18 @@ where
         }
     }
 
-    /// Demand-bound lockstep stripe read used by server-side copy sources.
+    /// Demand-bound data-shards-only lockstep stripe read.
     ///
     /// The ordinary lockstep path can cancel every in-flight reader once it
     /// has a quorum because all of its parity readers are already engaged.
-    /// Copy sources keep parity unopened until a data reader is missing.  A
+    /// Copy sources and the data-shards-only rollout gate keep parity unopened
-    /// hedge therefore has to race the deferred parity reads against the
+    /// until a data reader is missing. A hedge therefore has to race the
-    /// original data reads and may retire the latter only after the parity has
+    /// deferred parity reads against the original data reads and may retire the
-    /// produced an actual decode-plus-verification quorum.  The futures own
+    /// latter only after parity has produced an actual decode-plus-verification
-    /// their readers so disjoint data/parity slots can be admitted while the
+    /// quorum. The futures own their readers so disjoint data/parity slots can
-    /// other group is still pending; dropping an abandoned future retires its
+    /// be admitted while the other group is still pending; dropping an
-    /// stream without leaving a borrowed slot behind.
+    /// abandoned future retires its stream without leaving a borrowed slot
+    /// behind.
     async fn read_lockstep_demand_bound(&mut self, state: &mut StripeReadState) {
         let num_readers = self.readers.len();
         state.reset(num_readers, self.data_shards);
@@ -1576,14 +1578,14 @@ where
         let mut completed = 0usize;
         let mut failed = 0usize;
         let mut first_shard_recorded = false;
-        let mut active = vec![false; num_readers];
+        let mut active: ActiveReaders = smallvec![false; num_readers];
-        let mut temporary_parity = vec![false; num_readers];
+        let mut temporary_parity: ActiveReaders = smallvec![false; num_readers];
         // A deferred parity slot is attempted at most once per stripe. A
         // failed disposable hedge keeps its unopened reserve for the next
         // stripe, but must not be relaunched in a tight same-stripe retry
         // loop (which would defeat the bounded fan-out and amplify a remote
         // outage).
-        let mut attempted_parity = vec![false; num_readers];
+        let mut attempted_parity: ActiveReaders = smallvec![false; num_readers];
         // Once a data reader has returned an error (or was already missing at
         // setup), the loss is permanent for lockstep alignment. Use the
         // deferred handle and keep parity engaged across subsequent stripes;
@@ -2189,8 +2191,10 @@ impl Erasure {
         W: AsyncWrite + Send + Sync + Unpin,
         R: crate::erasure::coding::ShardSource,
     {
-        self.decode_inner(writer, readers, offset, length, total_length, None, Vec::new(), Vec::new())
+        let (written, error, _) = self
-            .await
+            .decode_inner(writer, readers, offset, length, total_length, None, Vec::new(), Vec::new())
+            .await;
+        (written, error)
     }
 
     #[allow(dead_code, reason = "read-cost decode path asserted by this file's tests (backlog#1823)")]
@@ -2207,8 +2211,10 @@ impl Erasure {
         W: AsyncWrite + Send + Sync + Unpin,
         R: crate::erasure::coding::ShardSource,
     {
-        self.decode_inner(writer, readers, offset, length, total_length, Some(read_costs), Vec::new(), Vec::new())
+        let (written, error, _) = self
-            .await
+            .decode_inner(writer, readers, offset, length, total_length, Some(read_costs), Vec::new(), Vec::new())
+            .await;
+        (written, error)
     }
 
     /// GET decode entry point that also carries the deferred-parity stripe
@@ -2261,6 +2267,37 @@ impl Erasure {
         deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
         deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
     ) -> (usize, Option<std::io::Error>)
+    where
+        W: AsyncWrite + Send + Sync + Unpin,
+        R: crate::erasure::coding::ShardSource,
+    {
+        let (written, error, _) = self
+            .decode_inner(
+                writer,
+                readers,
+                offset,
+                length,
+                total_length,
+                read_costs,
+                deferred_handles,
+                deferred_reopeners,
+            )
+            .await;
+        (written, error)
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub(crate) async fn decode_with_stripe_handles_and_reopeners_with_diagnostics<W, R>(
+        &self,
+        writer: &mut W,
+        readers: Vec<Option<BitrotReader<R>>>,
+        offset: usize,
+        length: usize,
+        total_length: usize,
+        read_costs: Option<Vec<ShardReadCost>>,
+        deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
+        deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
+    ) -> DecodeOutcome
     where
         W: AsyncWrite + Send + Sync + Unpin,
         R: crate::erasure::coding::ShardSource,
@@ -2298,6 +2335,7 @@ impl Erasure {
         written: &mut usize,
         ret_err: &mut Option<std::io::Error>,
         stage_metrics_enabled: bool,
+        require_surplus_source: bool,
     ) -> StripeFlow
     where
         W: AsyncWrite + Send + Sync + Unpin,
@@ -2335,7 +2373,12 @@ impl Erasure {
         // missing data shard and an extra source shard was available, verify
         // the reconstructed data against that source before streaming bytes.
         let reconstruct_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
-        if let Err(e) = self.decode_data_with_reconstruction_verification(shards) {
+        let decode_result = if require_surplus_source {
+            self.decode_data_with_reconstruction_verification_for_lockstep(shards)
+        } else {
+            self.decode_data_with_reconstruction_verification(shards)
+        };
+        if let Err(e) = decode_result {
             record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_RECONSTRUCT, reconstruct_stage_start);
             let reason = GetObjectFailureReason::DecodeError;
             error!(
@@ -2404,36 +2447,48 @@ impl Erasure {
         read_costs: Option<Vec<ShardReadCost>>,
         deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
         deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
-    ) -> (usize, Option<std::io::Error>)
+    ) -> DecodeOutcome
     where
         W: AsyncWrite + Send + Sync + Unpin,
         R: crate::erasure::coding::ShardSource,
     {
         if readers.len() != self.data_shards + self.parity_shards {
             record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
-            return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")));
+            return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")), false);
         }
 
         // block_size/data_shards come from on-disk metadata; a corrupt FileInfo with a
         // zero here must surface as an error, not a divide-by-zero panic on every GET.
         if self.block_size == 0 || self.data_shards == 0 {
             record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
-            return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid erasure coding parameters")));
+            return (
+                0,
+                Some(io::Error::new(ErrorKind::InvalidInput, "Invalid erasure coding parameters")),
+                false,
+            );
         }
 
         let Some(end_offset) = offset.checked_add(length) else {
             record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
-            return (0, Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")));
+            return (
+                0,
+                Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
+                false,
+            );
         };
         if end_offset > total_length {
             record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
-            return (0, Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")));
+            return (
+                0,
+                Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
+                false,
+            );
         }
 
         let mut ret_err = None;
 
         if length == 0 {
-            return (0, ret_err);
+            return (0, ret_err, false);
         }
 
         let mut written = 0;
@@ -2473,6 +2528,7 @@ impl Erasure {
             }
         };
 
+        let mut exact_quorum = false;
         if legacy_stripe_prefetch_enabled() {
             // Depth-1 stripe prefetch (backlog#930 HP-9 step 2): while the current
             // stripe is reconstructed and emitted, the next stripe's shard reads
@@ -2515,6 +2571,7 @@ impl Erasure {
                     let Some((mut shards, errs)) = current.take() else {
                         break;
                     };
+                    exact_quorum |= shards.iter().filter(|shard| shard.is_some()).count() == self.data_shards;
 
                     if idx + 1 < blocks.len() {
                         // Overlap: read stripe idx+1 while reconstructing/emitting idx.
@@ -2546,6 +2603,7 @@ impl Erasure {
                         // `shards` are borrowed again below. In the `Stop` case that
                         // drop is what cancels the still-in-flight prefetch read.
                         let (flow, next): (Option<StripeFlow>, Option<StripeReadOutput>) = {
+                            let require_surplus_source = reader.demand_bound_lockstep;
                             let read_fut = read_stripe_timed(&mut reader, stage_metrics_enabled);
                             let emit_fut = self.emit_decoded_stripe(
                                 writer,
@@ -2556,6 +2614,7 @@ impl Erasure {
                                 &mut written,
                                 &mut ret_err,
                                 stage_metrics_enabled,
+                                require_surplus_source,
                             );
                             tokio::pin!(read_fut);
                             tokio::pin!(emit_fut);
@@ -2603,6 +2662,7 @@ impl Erasure {
                                 &mut written,
                                 &mut ret_err,
                                 stage_metrics_enabled,
+                                reader.demand_bound_lockstep,
                             )
                             .await
                         {
@@ -2626,6 +2686,7 @@ impl Erasure {
                 let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
                 let stripe_read_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
                 let (mut shards, errs) = reader.read().await;
+                exact_quorum |= shards.iter().filter(|shard| shard.is_some()).count() == self.data_shards;
                 record_get_stage_duration_if_enabled(
                     GET_OBJECT_PATH_LEGACY_DUPLEX,
                     GET_STAGE_STRIPE_READ,
@@ -2642,6 +2703,7 @@ impl Erasure {
                         &mut written,
                         &mut ret_err,
                         stage_metrics_enabled,
+                        reader.demand_bound_lockstep,
                     )
                     .await
                 {
@@ -2654,14 +2716,14 @@ impl Erasure {
         }
 
         if ret_err.is_some() {
-            return (written, ret_err);
+            return (written, ret_err, exact_quorum);
         }
 
         if written < length {
             ret_err = Some(Error::LessData.into());
         }
 
-        (written, ret_err)
+        (written, ret_err, exact_quorum)
     }
 }
 
@@ -2866,6 +2928,7 @@ mod tests {
             cursor: Cursor<Vec<u8>>,
             stall: Duration,
             sleep: Option<Pin<Box<Sleep>>>,
+            stall_polls: Arc<AtomicUsize>,
         },
     }
 
@@ -2904,7 +2967,12 @@ mod tests {
                 TestShardReader::TerminalFileNotFound => {
                     Poll::Ready(Err(crate::disk::error::terminal_read_error_to_io(Error::FileNotFound)))
                 }
-                TestShardReader::PrefixThenSlow { cursor, stall, sleep } => {
+                TestShardReader::PrefixThenSlow {
+                    cursor,
+                    stall,
+                    sleep,
+                    stall_polls,
+                } => {
                     let before = buf.filled().len();
                     match Pin::new(cursor).poll_read(cx, buf) {
                         // Cursor still has bytes for the current stripe: serve them.
@@ -2914,6 +2982,7 @@ mod tests {
                         // the task cleanly (no busy `wake_by_ref` spin), letting the
                         // `#[tokio::test(start_paused = true)]` clock auto-advance.
                         Poll::Ready(Ok(())) => {
+                            stall_polls.fetch_add(1, Ordering::SeqCst);
                             let stall = *stall;
                             let sleeper = sleep.get_or_insert_with(|| Box::pin(tokio::time::sleep(stall)));
                             let _ = sleeper.as_mut().poll(cx);
@@ -2942,6 +3011,29 @@ mod tests {
         }
     }
 
+    struct YieldOnceThenFailWriter {
+        yielded: bool,
+    }
+
+    impl AsyncWrite for YieldOnceThenFailWriter {
+        fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, _buf: &[u8]) -> Poll<io::Result<usize>> {
+            if !self.yielded {
+                self.yielded = true;
+                cx.waker().wake_by_ref();
+                return Poll::Pending;
+            }
+            Poll::Ready(Err(io::Error::new(ErrorKind::BrokenPipe, "injected emit failure after prefetch poll")))
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+    }
+
     struct DownstreamClosedWriter;
 
     impl AsyncWrite for DownstreamClosedWriter {
@@ -3878,6 +3970,7 @@ mod tests {
             (rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT, Some(READ_TIMEOUT_SECS)),
         ];
         temp_env::async_with_vars(vars, async {
+            let stall_polls = Arc::new(AtomicUsize::new(0));
             let readers: Vec<Option<BitrotReader<TestShardReader>>> = shard_bufs
                 .iter()
                 .map(|buf| {
@@ -3887,12 +3980,13 @@ mod tests {
                         cursor: Cursor::new(prefix),
                         stall: STALL,
                         sleep: None,
+                        stall_polls: Arc::clone(&stall_polls),
                     };
                     Some(BitrotReader::new(reader, shard_size, hash_algo.clone(), false))
                 })
                 .collect();
 
-            let mut writer = FailingEmitWriter;
+            let mut writer = YieldOnceThenFailWriter { yielded: false };
             let start = TokioInstant::now();
             let (written, err) = erasure.decode(&mut writer, readers, 0, total_len, total_len).await;
             let elapsed = start.elapsed();
@@ -3900,6 +3994,10 @@ mod tests {
             // Emit failed on stripe 0, so the GET fails with no bytes emitted.
             assert!(err.is_some(), "emit failure must surface as an error");
             assert_eq!(written, 0, "the failing writer accepts no bytes");
+            assert!(
+                stall_polls.load(Ordering::SeqCst) > 0,
+                "the speculative next-stripe read must be in flight before emit fails"
+            );
             // The decisive assertion: the prefetch read was cancelled rather than
             // awaited. Without cancel-safety this would take READ_TIMEOUT_SECS.
             assert!(
@@ -4911,6 +5009,24 @@ mod tests {
     /// read timeout even though both parity readers were available to engage.
     #[tokio::test]
     async fn test_demand_bound_lockstep_hedges_to_deferred_parity_quorum() {
+        with_decode_read_policy(DecodeReadPolicy::DemandBound, assert_deferred_parity_hedges_slow_data()).await;
+    }
+
+    /// The ordinary GET rollout gate must use the same bounded parity race as
+    /// CopySource. Leaving it on the legacy lockstep loop deadlocks the hedge:
+    /// that loop waits for a parity success before cancelling the slow data
+    /// read, but does not admit deferred parity until after the data read ends.
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn test_data_shards_only_gate_hedges_to_deferred_parity_quorum() {
+        temp_env::async_with_vars(
+            [(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, Some("true"))],
+            assert_deferred_parity_hedges_slow_data(),
+        )
+        .await;
+    }
+
+    async fn assert_deferred_parity_hedges_slow_data() {
         const NUM_SHARDS: usize = 1;
         const BLOCK_SIZE: usize = 64;
         const DATA_SHARDS: usize = 2;
@@ -4951,33 +5067,27 @@ mod tests {
         ];
 
         let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
-        let (bufs, errs, engaged, readers_remaining) = with_decode_read_policy(DecodeReadPolicy::DemandBound, async {
+        let mut parallel_reader = ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
-            let mut parallel_reader = ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
+            readers,
-                readers,
+            erasure,
-                erasure,
+            0,
-                0,
+            NUM_SHARDS * BLOCK_SIZE,
-                NUM_SHARDS * BLOCK_SIZE,
+            None,
-                None,
+            vec![ShardReadCost::Unknown; DATA_SHARDS + PARITY_SHARDS],
-                vec![ShardReadCost::Unknown; DATA_SHARDS + PARITY_SHARDS],
+            Duration::from_secs(60),
-                Duration::from_secs(60),
+            true,
-                true,
+        );
-            );
+        let (bufs, errs) = tokio::time::timeout(Duration::from_secs(2), parallel_reader.read())
-            let (bufs, errs) = tokio::time::timeout(Duration::from_secs(2), parallel_reader.read())
+            .await
-                .await
+            .expect("deferred parity must cover a hedged data shard without waiting for read_timeout");
-                .expect("deferred parity must cover a hedged data shard without waiting for read_timeout");
-            (
-                bufs,
-                errs,
-                parallel_reader.engaged.clone(),
-                parallel_reader.readers.iter().map(Option::is_some).collect::<Vec<_>>(),
-            )
-        })
-        .await;
 
         assert!(matches!(&errs[0], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut));
         assert_eq!(bufs.iter().filter(|buf| buf.is_some()).count(), DATA_SHARDS + 1);
-        assert_eq!(engaged.as_slice(), &[true, true, true, true]);
+        assert_eq!(parallel_reader.engaged.as_slice(), &[true, true, true, true]);
-        assert_eq!(readers_remaining, vec![false, true, true, true]);
+        assert_eq!(
+            parallel_reader.readers.iter().map(Option::is_some).collect::<Vec<_>>(),
+            vec![false, true, true, true]
+        );
     }
 
     /// A fast data failure must admit deferred parity immediately.  There is
@@ -5046,6 +5156,24 @@ mod tests {
 
     #[tokio::test]
     async fn test_demand_bound_canceled_hedge_preserves_deferred_parity_for_next_stripe() {
+        with_decode_read_policy(
+            DecodeReadPolicy::DemandBound,
+            assert_canceled_hedge_preserves_deferred_parity_for_next_stripe(),
+        )
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn test_data_shards_only_gate_canceled_hedge_preserves_deferred_parity_for_next_stripe() {
+        temp_env::async_with_vars(
+            [(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, Some("true"))],
+            assert_canceled_hedge_preserves_deferred_parity_for_next_stripe(),
+        )
+        .await;
+    }
+
+    async fn assert_canceled_hedge_preserves_deferred_parity_for_next_stripe() {
         const BLOCK_SIZE: usize = 64;
         const DATA_SHARDS: usize = 2;
         const PARITY_SHARDS: usize = 2;
@@ -5094,7 +5222,7 @@ mod tests {
             Some(BitrotReader::new(TestShardReader::Pending, SHARD_SIZE, hash_algo, false)),
         ];
 
-        let (first_parity_reserved, second_result) = with_decode_read_policy(DecodeReadPolicy::DemandBound, async {
+        let (first_parity_reserved, second_result) = {
             let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
             let mut parallel_reader = ParallelReader::new_with_metrics_path_read_timeout_and_reconstruction_verification(
                 readers,
@@ -5155,8 +5283,7 @@ mod tests {
                 parallel_reader.readers[2].is_some() && parallel_reader.readers[3].is_some(),
                 (third_buffers, third_errors),
             )
-        })
+        };
-        .await;
 
         assert!(first_parity_reserved);
         assert_eq!(parity_calls.load(Ordering::SeqCst), PARITY_SHARDS * 2);
@@ -5240,6 +5367,58 @@ mod tests {
         assert!(error.is_none(), "a failed disposable hedge must not fail a recovered stripe: {error:?}");
     }
 
+    /// Rollout guard for backlog#1308: when a data shard and the first parity
+    /// hedge both fail, the gate-on path must not settle at decode quorum and
+    /// emit an unverified body. The second parity can restore decode quorum but
+    /// cannot provide the extra source required for reconstruction verification,
+    /// so the stripe must fail before exposing bytes.
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn test_data_shards_only_gate_data_and_parity_failure_fails_before_output() {
+        const BLOCK_SIZE: usize = 64;
+        const DATA_SHARDS: usize = 2;
+        const PARITY_SHARDS: usize = 2;
+
+        temp_env::async_with_vars([(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, Some("true"))], async {
+            let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
+            let payload = (0..BLOCK_SIZE).map(|value| value as u8).collect::<Vec<_>>();
+            let shards = erasure.encode_data(&payload).expect("test payload should encode");
+            let shard_size = erasure.shard_size();
+
+            let readers = vec![
+                Some(BitrotReader::new(TestShardReader::TimedOut, shard_size, HashAlgorithm::None, false)),
+                Some(BitrotReader::new(
+                    TestShardReader::Ready(Cursor::new(shards[1].to_vec())),
+                    shard_size,
+                    HashAlgorithm::None,
+                    false,
+                )),
+                Some(BitrotReader::new(
+                    TestShardReader::TerminalFileNotFound,
+                    shard_size,
+                    HashAlgorithm::None,
+                    false,
+                )),
+                Some(BitrotReader::new(
+                    TestShardReader::Ready(Cursor::new(shards[3].to_vec())),
+                    shard_size,
+                    HashAlgorithm::None,
+                    false,
+                )),
+            ];
+
+            let mut output = Vec::new();
+            let (written, error) = erasure.decode(&mut output, readers, 0, payload.len(), payload.len()).await;
+
+            assert_eq!(written, 0, "an unverified stripe must not report body bytes");
+            assert!(output.is_empty(), "an unverified stripe must not expose a clean short body");
+            let error = error.expect("data plus parity loss must fail closed");
+            assert_eq!(error.kind(), ErrorKind::InvalidData);
+            assert!(error.to_string().contains("insufficient source shards"));
+        })
+        .await;
+    }
+
     /// Lockstep verification-quorum regression (backlog#1156). When a data shard is
     /// missing, the hedge must settle only at `data_shards + 1` (decode quorum plus
     /// a reconstruction-verification source), never at exactly `data_shards` — that

crates/ecstore/src/erasure/coding/decode_reader.rs

@@ -27,6 +27,8 @@ use std::io;
 use std::io::ErrorKind;
 use std::pin::Pin;
 use std::sync::Mutex;
+#[cfg(test)]
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::task::{Context, Poll, ready};
 use std::time::Instant;
 use tokio::io::{AsyncRead, ReadBuf};
@@ -38,6 +40,14 @@ const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT: usize = 2;
 const FILL_POLICY_SINGLE_INFLIGHT: &str = "single_inflight";
 const FILL_POLICY_DUAL_INFLIGHT: &str = "dual_inflight";
 
+#[cfg(test)]
+static SINGLE_INFLIGHT_CONSTRUCTIONS: AtomicU64 = AtomicU64::new(0);
+
+#[cfg(test)]
+pub(crate) fn test_single_inflight_construction_count() -> u64 {
+    SINGLE_INFLIGHT_CONSTRUCTIONS.load(Ordering::Relaxed)
+}
+
 type FillTask = oneshot::Receiver<FillResult>;
 
 struct FillWorker {
@@ -155,6 +165,23 @@ where
         Self::new_with_fill_policy_inner(source, engine, total_length, metrics_path, FillPolicy::from_env())
     }
 
+    /// Construct the bounded reader without lookahead.
+    ///
+    /// Mid-size GETs are latency-sensitive and are already gated to a single
+    /// plain part. Keeping one stripe in flight avoids retaining a second
+    /// decoded output buffer while preserving the same source, reconstruction,
+    /// bitrot and cancellation semantics as the general streaming reader.
+    pub(crate) fn new_single_inflight_with_metrics_path(
+        source: S,
+        engine: E,
+        total_length: usize,
+        metrics_path: &'static str,
+    ) -> io::Result<Self> {
+        #[cfg(test)]
+        SINGLE_INFLIGHT_CONSTRUCTIONS.fetch_add(1, Ordering::Relaxed);
+        Self::new_with_fill_policy_inner(source, engine, total_length, metrics_path, FillPolicy::SingleInFlight)
+    }
+
     fn new_with_fill_policy_inner(
         source: S,
         engine: E,
@@ -602,7 +629,8 @@ where
 
         loop {
             if self.output_pos < self.output_buf.len() {
-                if self.prefetched_bufs.len() < self.fill_policy.max_inflight()
+                if self.fill_policy == FillPolicy::DualInFlight
+                    && self.prefetched_bufs.len() < self.fill_policy.max_inflight()
                     && self.prefetch_error.is_none()
                     && self.remaining > 0
                     && let Poll::Ready(result) = self.poll_prefetch(cx)
@@ -1620,6 +1648,149 @@ mod tests {
         assert_eq!(decoded, data);
     }
 
+    #[tokio::test]
+    async fn single_inflight_reader_reads_full_body_without_lookahead() {
+        let erasure = Erasure::new(4, 2, 32);
+        let data = (0..96u8).collect::<Vec<_>>();
+        let read_count = Arc::new(AtomicUsize::new(0));
+        let mut source = source_from_data(&erasure, &data, &[]);
+        source.read_count = Some(Arc::clone(&read_count));
+        let engine = LegacyEcDecodeEngine::new(erasure);
+        let mut reader = ErasureDecodeReader::new_single_inflight_with_metrics_path(
+            source,
+            engine,
+            data.len(),
+            GET_OBJECT_PATH_CODEC_STREAMING,
+        )
+        .expect("single-inflight reader should be constructed");
+        let mut decoded = Vec::new();
+
+        reader
+            .read_to_end(&mut decoded)
+            .await
+            .expect("single-inflight reader should decode the complete body");
+
+        assert_eq!(decoded, data);
+        assert_eq!(
+            read_count.load(Ordering::SeqCst),
+            3,
+            "single-inflight must not read ahead after the final stripe"
+        );
+    }
+
+    #[tokio::test]
+    async fn single_inflight_reader_preserves_partial_reads() {
+        let erasure = Erasure::new(4, 2, 32);
+        let data = (0..83u8).collect::<Vec<_>>();
+        let engine = LegacyEcDecodeEngine::new(erasure.clone());
+        let mut reader = ErasureDecodeReader::new_single_inflight_with_metrics_path(
+            source_from_data(&erasure, &data, &[]),
+            engine,
+            data.len(),
+            GET_OBJECT_PATH_CODEC_STREAMING,
+        )
+        .expect("single-inflight reader should be constructed");
+        let mut decoded = Vec::with_capacity(data.len());
+        let mut chunk = [0u8; 3];
+
+        loop {
+            let read = reader.read(&mut chunk).await.expect("partial read should succeed");
+            if read == 0 {
+                break;
+            }
+            decoded.extend_from_slice(&chunk[..read]);
+        }
+
+        assert_eq!(decoded, data);
+    }
+
+    #[tokio::test]
+    async fn single_inflight_reader_does_not_prefetch_before_output_is_drained() {
+        let erasure = Erasure::new(4, 2, 32);
+        let data = (0..96u8).collect::<Vec<_>>();
+        let read_count = Arc::new(AtomicUsize::new(0));
+        let mut source = source_from_data(&erasure, &data, &[]);
+        source.read_count = Some(Arc::clone(&read_count));
+        let engine = LegacyEcDecodeEngine::new(erasure);
+        let mut reader = ErasureDecodeReader::new_single_inflight_with_metrics_path(
+            source,
+            engine,
+            data.len(),
+            GET_OBJECT_PATH_CODEC_STREAMING,
+        )
+        .expect("single-inflight reader should be constructed");
+        let mut first = [0u8; 3];
+
+        reader
+            .read_exact(&mut first)
+            .await
+            .expect("first partial read should succeed");
+
+        assert_eq!(
+            read_count.load(Ordering::SeqCst),
+            1,
+            "single-inflight must not prefetch while output remains"
+        );
+        assert_eq!(&first, &data[..3]);
+    }
+
+    #[tokio::test]
+    async fn single_inflight_reader_reconstructs_degraded_body() {
+        let erasure = Erasure::new(4, 2, 32);
+        let data = (0..97u16).map(|value| value as u8).collect::<Vec<_>>();
+        let engine = LegacyEcDecodeEngine::new(erasure.clone());
+        let mut reader = ErasureDecodeReader::new_single_inflight_with_metrics_path(
+            source_from_data(&erasure, &data, &[1]),
+            engine,
+            data.len(),
+            GET_OBJECT_PATH_CODEC_STREAMING,
+        )
+        .expect("single-inflight reader should be constructed");
+        let mut decoded = Vec::new();
+
+        reader
+            .read_to_end(&mut decoded)
+            .await
+            .expect("a readable degraded stripe should be reconstructed");
+
+        assert_eq!(decoded, data);
+    }
+
+    #[tokio::test]
+    async fn single_inflight_reader_surfaces_error_after_buffered_body() {
+        let erasure = Erasure::new(4, 2, 32);
+        let first_stripe = (0..32u8).collect::<Vec<_>>();
+        let first_state = source_from_data(&erasure, &first_stripe, &[])
+            .stripes
+            .pop_front()
+            .expect("first stripe should exist");
+        let source = VecStripeSource {
+            stripes: VecDeque::from([
+                first_state,
+                StripeReadState::from_parts(Vec::new(), Vec::new(), erasure.data_shards),
+            ]),
+            read_quorum: erasure.data_shards,
+            read_count: None,
+        };
+        let engine = LegacyEcDecodeEngine::new(erasure);
+        let mut reader = ErasureDecodeReader::new_single_inflight_with_metrics_path(
+            source,
+            engine,
+            first_stripe.len() + 1,
+            GET_OBJECT_PATH_CODEC_STREAMING,
+        )
+        .expect("single-inflight reader should be constructed");
+        let mut decoded = Vec::new();
+
+        let error = reader
+            .read_to_end(&mut decoded)
+            .await
+            .expect_err("short source error should be returned after buffered bytes");
+
+        assert_eq!(error.kind(), ErrorKind::Other);
+        assert_eq!(decoded, first_stripe);
+    }
+
     #[tokio::test]
     async fn erasure_decode_reader_stops_at_eof_for_empty_object() {
         let erasure = Erasure::new(4, 2, 32);
@@ -1882,14 +2053,9 @@ mod tests {
         };
         let engine = LegacyEcDecodeEngine::new(Erasure::new(1, 0, 32));
         let task = tokio::spawn(async move {
-            let mut reader = ErasureDecodeReader::new_with_fill_policy(
+            let mut reader =
-                source,
+                ErasureDecodeReader::new_single_inflight_with_metrics_path(source, engine, 1, GET_OBJECT_PATH_CODEC_STREAMING)
-                engine,
+                    .expect("reader should be constructed");
-                1,
-                GET_OBJECT_PATH_CODEC_STREAMING,
-                FillPolicy::SingleInFlight,
-            )
-            .expect("reader should be constructed");
             let mut first_read = [0u8; 1];
             let _ = reader.read(&mut first_read).await;
         });
@@ -2226,7 +2392,7 @@ mod tests {
             engine,
             data.len(),
             GET_OBJECT_PATH_CODEC_STREAMING,
-            FillPolicy::SingleInFlight,
+            FillPolicy::DualInFlight,
         )
         .expect("reader should be constructed");
         let mut first_read = [0u8; 1];
@@ -2235,13 +2401,11 @@ mod tests {
 
         assert_eq!(read, first_read.len());
         assert_eq!(first_read[0], data[0]);
-        timeout(Duration::from_secs(1), async {
+        assert_eq!(
-            while read_count.load(Ordering::SeqCst) < 2 {
+            read_count.load(Ordering::SeqCst),
-                yield_now().await;
+            2,
-            }
+            "dual-inflight reader should prefetch the next stripe before returning the first byte"
-        })
+        );
-        .await
-        .expect("reader should start reading the next stripe before the current output buffer is fully consumed");
     }
 
     #[tokio::test]

crates/ecstore/src/erasure/coding/encode.rs

@@ -321,6 +321,13 @@ impl<'a> MultiWriter<'a> {
         }
     }
 
+    pub(super) fn take_retryable_internode_write_failure(&mut self) -> Option<Error> {
+        self.errs
+            .iter_mut()
+            .find(|error| error.as_ref().is_some_and(Error::is_retryable_internode_write_failure))
+            .and_then(Option::take)
+    }
+
     /// Effective budget for one shard operation: the smaller of the per-shard
     /// stall timeout and the time remaining until the object's absolute cap.
     /// Returns `None` when neither deadline is configured (wait indefinitely).

crates/ecstore/src/erasure/coding/erasure.rs

@@ -933,8 +933,29 @@ impl Erasure {
     }
 
     pub(crate) fn decode_data_with_reconstruction_verification(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
+        self.decode_data_with_reconstruction_verification_policy(shards, false)
+    }
+
+    pub(crate) fn decode_data_with_reconstruction_verification_for_lockstep(
+        &self,
+        shards: &mut [Option<Vec<u8>>],
+    ) -> io::Result<()> {
+        self.decode_data_with_reconstruction_verification_policy(shards, true)
+    }
+
+    fn decode_data_with_reconstruction_verification_policy(
+        &self,
+        shards: &mut [Option<Vec<u8>>],
+        require_surplus_source: bool,
+    ) -> io::Result<()> {
         let missing_data_source = shards.iter().take(self.data_shards).any(|shard| shard.is_none());
         let available_shards = shards.iter().filter(|shard| shard.is_some()).count();
+        if require_surplus_source && missing_data_source && available_shards == self.data_shards {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidData,
+                "insufficient source shards to verify reconstructed data",
+            ));
+        }
         let source_parity = if missing_data_source && available_shards > self.data_shards {
             shards
                 .iter()
@@ -1868,6 +1889,31 @@ mod tests {
         assert_eq!(err.kind(), io::ErrorKind::InvalidData);
     }
 
+    #[test]
+    fn decode_data_with_verification_scopes_exact_quorum_to_lockstep() {
+        for uses_legacy in [false, true] {
+            let erasure = Erasure::new_with_options(3, 2, 128, uses_legacy);
+            let data = b"verified reads must not accept reconstruction without a surplus source";
+            let encoded = erasure.encode_data(data).expect("encode should succeed");
+            let mut exact_quorum = optional_shards(&encoded);
+            exact_quorum[0] = None;
+            exact_quorum[erasure.total_shard_count() - 1] = None;
+
+            let mut default_shards = exact_quorum.clone();
+            erasure
+                .decode_data_with_reconstruction_verification(&mut default_shards)
+                .expect("default decode must preserve exact-quorum reconstruction");
+            assert_eq!(default_shards[0].as_deref(), Some(encoded[0].as_ref()));
+
+            let err = erasure
+                .decode_data_with_reconstruction_verification_for_lockstep(&mut exact_quorum)
+                .expect_err("data-shards-only lockstep must reject an exact decode quorum");
+
+            assert_eq!(err.kind(), io::ErrorKind::InvalidData);
+            assert!(err.to_string().contains("insufficient source shards"));
+        }
+    }
+
     #[test]
     fn verify_data_and_parity_rejects_missing_and_mismatched_shards() {
         let erasure = Erasure::new(4, 2, 128);

crates/ecstore/src/erasure/coding/heal.rs

@@ -108,6 +108,13 @@ where
     (shards, errs)
 }
 
+fn heal_writer_failure(writers: &mut MultiWriter<'_>, error: io::Error) -> Error {
+    writers
+        .take_retryable_internode_write_failure()
+        .map(|error| Error::RemoteClientUnavailable(error.to_string()))
+        .unwrap_or_else(|| error.into())
+}
+
 impl super::Erasure {
     pub async fn heal<R>(
         &self,
@@ -202,10 +209,14 @@ impl super::Erasure {
                 .map(|s| Bytes::from(s.unwrap_or_default()))
                 .collect::<Vec<_>>();
 
-            writers.write(shards).await?;
+            if let Err(error) = writers.write(shards).await {
+                return Err(heal_writer_failure(&mut writers, error));
+            }
         }
 
-        writers.shutdown().await?;
+        if let Err(error) = writers.shutdown().await {
+            return Err(heal_writer_failure(&mut writers, error));
+        }
         Ok(())
     }
 }
@@ -246,6 +257,35 @@ mod tests {
         }
     }
 
+    struct InternodeFailureWriter {
+        fail_on_write: bool,
+        status: http::StatusCode,
+    }
+
+    impl InternodeFailureWriter {
+        fn error(&self) -> io::Error {
+            rustfs_rio::new_test_internode_http_io_error(rustfs_rio::InternodeHttpErrorKind::HttpStatus(self.status))
+        }
+    }
+
+    impl AsyncWrite for InternodeFailureWriter {
+        fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> {
+            Poll::Ready(if self.fail_on_write {
+                Err(self.error())
+            } else {
+                Ok(buf.len())
+            })
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Err(self.error()))
+        }
+    }
+
     struct PendingReader;
 
     impl AsyncRead for PendingReader {
@@ -331,6 +371,94 @@ mod tests {
         assert!(writers.iter().all(Option::is_some));
     }
 
+    #[tokio::test]
+    async fn heal_maps_put_file_epoch_conflict_to_retryable_remote_unavailable() {
+        for status in [http::StatusCode::CONFLICT, http::StatusCode::BAD_REQUEST] {
+            for (fail_on_write, data) in [
+                (false, b"".as_slice()),
+                (false, b"payload".as_slice()),
+                (true, b"payload".as_slice()),
+            ] {
+                let erasure = Erasure::new(2, 1, 64);
+                let encoded = erasure.encode_data(data).expect("source shards should encode");
+                let readers = encoded
+                    .iter()
+                    .enumerate()
+                    .map(|(index, shard)| {
+                        (index < erasure.data_shards).then(|| {
+                            BitrotReader::new(Cursor::new(shard.to_vec()), erasure.shard_size(), HashAlgorithm::None, false)
+                        })
+                    })
+                    .collect::<Vec<_>>();
+                let mut writers = (0..erasure.total_shard_count())
+                    .map(|index| {
+                        (index == erasure.data_shards).then(|| {
+                            BitrotWriterWrapper::new(
+                                CustomWriter::new_tokio_writer(InternodeFailureWriter { fail_on_write, status }),
+                                erasure.shard_size(),
+                                HashAlgorithm::None,
+                            )
+                        })
+                    })
+                    .collect::<Vec<_>>();
+                let error = erasure
+                    .heal(&mut writers, readers, data.len(), &[])
+                    .await
+                    .expect_err("failed sole target must not satisfy heal write quorum");
+                assert_eq!(
+                    matches!(error, Error::RemoteClientUnavailable(_)),
+                    status == http::StatusCode::CONFLICT,
+                    "status={status}, fail_on_write={fail_on_write}, len={}, error={error:?}",
+                    data.len()
+                );
+                assert!(writers.iter().all(Option::is_none), "failed target must not be committed");
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn heal_epoch_conflict_does_not_abort_healthy_target() {
+        for fail_on_write in [false, true] {
+            let erasure = Erasure::new(2, 2, 64);
+            let data = b"healthy target must retain exact reconstructed bytes";
+            let encoded = erasure.encode_data(data).expect("source shards should encode");
+            let readers = encoded
+                .iter()
+                .enumerate()
+                .map(|(index, shard)| {
+                    (index < erasure.data_shards)
+                        .then(|| BitrotReader::new(Cursor::new(shard.to_vec()), erasure.shard_size(), HashAlgorithm::None, false))
+                })
+                .collect::<Vec<_>>();
+            let mut writers = vec![
+                None,
+                None,
+                Some(BitrotWriterWrapper::new(
+                    CustomWriter::new_tokio_writer(InternodeFailureWriter {
+                        fail_on_write,
+                        status: http::StatusCode::CONFLICT,
+                    }),
+                    erasure.shard_size(),
+                    HashAlgorithm::None,
+                )),
+                Some(inline_writer(erasure.shard_size())),
+            ];
+            erasure
+                .heal(&mut writers, readers, data.len(), &[])
+                .await
+                .expect("one healthy target must still satisfy the existing heal quorum");
+            assert!(writers[2].is_none(), "conflicting target must be dropped");
+            assert_eq!(
+                writers[3]
+                    .take()
+                    .expect("healthy target remains")
+                    .into_inline_data()
+                    .expect("inline target data"),
+                encoded[3].to_vec()
+            );
+        }
+    }
+
     #[tokio::test]
     async fn heal_reconstructs_missing_parity_shard() {
         let erasure = Erasure::new(2, 2, 64);