Refactor code structure for improved readability and maintainability

fix(sse): separate SSE-S3 and KMS key providers
chore(deps): refresh cargo dependencies (#5132 )
2026-07-30 01:58:59 +00:00 · 2026-07-24 14:01:40 +08:00 · 2026-07-22 21:05:23 +08:00 · 2026-07-22 21:02:59 +08:00 · 2026-07-22 11:52:56 +00:00 · 2026-07-22 11:31:03 +00:00
124 changed files with 17746 additions and 2827 deletions
@@ -15,18 +15,37 @@
 # MinIO on-disk interop: prove RustFS reads MinIO-written erasure-coded SSE
 # objects with byte-identical data and correct logical size.
 #
-# This is NOT a PR gate. The fixtures are real MinIO backend trees generated on
-# the fly (they are gitignored, never committed), so the job regenerates them
-# each run with Docker and then runs the `#[ignore]` reader tests in
-# crates/ecstore/tests/minio_generated_read_test.rs.
+# The fixtures are real MinIO backend trees generated on the fly (they are
+# gitignored, never committed), so the job regenerates them each run with
+# Docker and then runs the `#[ignore]` reader tests in
+# crates/ecstore/tests/minio_generated_read_test.rs. SSE/RIO changes run this
+# as a path-filtered PR gate; nightly and manual runs retain broader coverage.
 #
 # Runner: GitHub-hosted `ubuntu-latest`. It reliably ships Docker + Python,
 # unlike the self-hosted fleet, whose pods drift in Docker/pip availability
-# (see the infra note in e2e-s3tests.yml). Nightly + manual only.
+# (see the infra note in e2e-s3tests.yml).
 name: minio-interop

 on:
  workflow_dispatch:
+  pull_request:
+    paths:
+      - ".github/workflows/minio-interop.yml"
+      - "Cargo.lock"
+      - "Cargo.toml"
+      - "crates/ecstore/Cargo.toml"
+      - "crates/ecstore/src/io_support/rio.rs"
+      - "crates/ecstore/src/client/api_put_object_streaming.rs"
+      - "crates/ecstore/src/object_api/readers.rs"
+      - "crates/ecstore/src/set_disk/ops/object.rs"
+      - "crates/ecstore/src/sse/**"
+      - "crates/ecstore/src/store/object.rs"
+      - "crates/ecstore/tests/minio_generated_read_test.rs"
+      - "crates/kms/**"
+      - "crates/rio-v2/**"
+      - "rustfs/src/app/object_usecase.rs"
+      - "rustfs/Cargo.toml"
+      - "rustfs/src/storage/sse.rs"
  schedule:
    # Nightly at 03:17 UTC (offset from other nightly jobs).
    - cron: "17 3 * * *"
@@ -41,13 +60,13 @@ permissions:
 jobs:
  minio-interop:
    name: MinIO interop (EC + SSE read parity)
-    # Skip on forks: needs the repo's runners and is not a contributor gate.
+    # Skip on forks because this job depends on the repository's CI setup.
    if: github.repository == 'rustfs/rustfs'
    runs-on: ubuntu-latest
    timeout-minutes: 40
    env:
      # Fixed 32-byte test KMS key baked into the fixture lab; not a secret.
-      RUSTFS_MINIO_STATIC_KMS_KEY_B64: IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g=
+      RUSTFS_MINIO_STATIC_KMS_KEY: minio-default-key:IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g=
    steps:
      - name: Checkout repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -61,7 +80,10 @@ jobs:
          cache-save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: Generate real MinIO fixtures via Docker
-        run: bash crates/rio-v2/tests/minio_fixture_lab/capture_via_docker.sh
+        run: bash crates/rio-v2/tests/minio_fixture_lab/capture_via_docker.sh all
+
+      - name: Generate real RustFS beta.5 KMS fixture
+        run: uv run python crates/rio-v2/tests/minio_fixture_lab/capture_rustfs_beta5.py

      - name: Run MinIO interop reader tests
        run: |
@@ -132,6 +132,29 @@
                "rust"
            ],
        },
+        {
+            "name": "Debug executable target/debug/rustfs with sse-s3",
+            "type": "lldb",
+            "request": "launch",
+            "program": "${workspaceFolder}/target/debug/rustfs",
+            "args": [],
+            "cwd": "${workspaceFolder}",
+            "env": {
+                "RUSTFS_ACCESS_KEY": "rustfsadmin",
+                "RUSTFS_SECRET_KEY": "rustfsadmin",
+                "RUSTFS_VOLUMES": "./target/volumes/test{1...4}",
+                "RUSTFS_ADDRESS": ":9000",
+                "RUSTFS_CONSOLE_ENABLE": "true",
+                "RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001",
+                "RUSTFS_OBS_LOG_DIRECTORY": "./target/logs",
+                "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true",
+                "RUSTFS_SSE_S3_MASTER_KEY": "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=",
+                "RUST_LOG": "rustfs=debug,ecstore=debug,s3s=debug,iam=debug"
+            },
+            "sourceLanguages": [
+                "rust"
+            ]
+        },
        {
            "type": "lldb",
            "request": "launch",
@@ -698,6 +698,7 @@ version = "0.50.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d83a251fa1a4c9d0fe6e816b7acd60549e473e08d14f27a1d992c2675abff05f"
 dependencies = [
+ "aws-lc-rs",
 "base64 0.22.1",
 "bytes",
 "futures-util",
@@ -708,7 +709,6 @@ dependencies = [
 "portable-atomic",
 "rand 0.10.2",
 "regex",
- "ring",
 "rustls-native-certs",
 "rustls-pki-types",
 "rustls-webpki",
@@ -769,7 +769,7 @@ checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 3.0.2",
+ "syn 3.0.3",
 ]

 [[package]]
@@ -850,9 +850,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"

 [[package]]
 name = "aws-config"
-version = "1.9.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47712fde1909402600ccfbb26e47d482d2e58bb9e9e603d9f17e67cc435a6319"
+checksum = "701418aa459dac33e50a0f8e818e5662a16bc018a6ac7423659b70f3799d67a8"
 dependencies = [
 "aws-credential-types",
 "aws-runtime",
@@ -917,9 +917,9 @@ dependencies = [

 [[package]]
 name = "aws-runtime"
-version = "1.8.1"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7816e98ee912159f45d307e5ee6bfea4a335a55aee15f7f3e32f81a6f3000f1d"
+checksum = "a6b50a43f3ccdf331521c6d6c68b7cc9668b6e09d439ebda9569df5722324d76"
 dependencies = [
 "aws-credential-types",
 "aws-sigv4",
@@ -945,9 +945,9 @@ dependencies = [

 [[package]]
 name = "aws-sdk-s3"
-version = "1.138.1"
+version = "1.139.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fa59420755799fec8aac71f562aa37ba82263f5b1926bba8b5d3465d2f7a73f"
+checksum = "a159b9721a6a41468f967d1029bece78f410b0beb0594498435deb6ff72bfe48"
 dependencies = [
 "arc-swap",
 "aws-credential-types",
@@ -982,9 +982,9 @@ dependencies = [

 [[package]]
 name = "aws-sdk-sso"
-version = "1.103.0"
+version = "1.104.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0469f435f645ad2162cfb463b15bde37115966ee3acf2d87fb4871ee309b8401"
+checksum = "b53416d16c278234845392e38d93bd4481d2f09daa0f005a2277f0aa91f59c22"
 dependencies = [
 "arc-swap",
 "aws-credential-types",
@@ -1008,9 +1008,9 @@ dependencies = [

 [[package]]
 name = "aws-sdk-ssooidc"
-version = "1.105.0"
+version = "1.106.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "085faefb253f770655e162b9304321e62a1e71adf7f019ee1f4454228a377b3a"
+checksum = "cc9b706c3305ed0285d5b1b696c747aa34950f830fb03e3e6c76890f99b9f188"
 dependencies = [
 "arc-swap",
 "aws-credential-types",
@@ -1034,9 +1034,9 @@ dependencies = [

 [[package]]
 name = "aws-sdk-sts"
-version = "1.108.0"
+version = "1.109.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c72b08911d8128dd360fe1b22a9fec0fa8b552dde8ec828dcf20ef5ec974e9f"
+checksum = "32d214cdfa5bbe17f117e76a7643fadf32a5234fb597322ef8b1fb4b2f17dbbd"
 dependencies = [
 "arc-swap",
 "aws-credential-types",
@@ -1197,11 +1197,14 @@ dependencies = [

 [[package]]
 name = "aws-smithy-query"
-version = "0.61.1"
+version = "0.62.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd22a6ba36e3f113cb8d5b3d1fe0ed31c76ee608ef63322d753bb8d2c9479e77"
+checksum = "512346c7212ab7436df2d77a16d976a468ae44a418835511d2a69269810aaf62"
 dependencies = [
+ "aws-smithy-runtime-api",
+ "aws-smithy-schema",
 "aws-smithy-types",
+ "aws-smithy-xml",
 "urlencoding",
 ]

@@ -1299,9 +1302,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-xml"
-version = "0.61.1"
+version = "0.62.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea3f68eec3607f02acd24067969ce2abc6ba16aa7d5ce59ca450ed2fb5f78957"
+checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8"
 dependencies = [
 "aws-smithy-runtime-api",
 "aws-smithy-schema",
@@ -1311,9 +1314,9 @@ dependencies = [

 [[package]]
 name = "aws-types"
-version = "1.4.0"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e957a6c6dbce82b7a91f44231c09273159703769f447cbe85e854dfe9cf67f86"
+checksum = "eec1cd5469f328c782dc3e33d4153cf118a54e33cbb3356d60d16f89883e1f94"
 dependencies = [
 "aws-credential-types",
 "aws-smithy-async",
@@ -1519,7 +1522,7 @@ version = "0.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
 ]

 [[package]]
@@ -1538,7 +1541,7 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
 ]

 [[package]]
@@ -1861,7 +1864,7 @@ version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
 dependencies = [
- "crypto-common 0.1.7",
+ "crypto-common 0.1.6",
 "inout 0.1.4",
 ]

@@ -1879,9 +1882,9 @@ dependencies = [

 [[package]]
 name = "clap"
-version = "4.6.3"
+version = "4.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fb99565819980999fb7b4a1796046a5c949e6d4ff132cf5fadf5a641e20d776"
+checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7"
 dependencies = [
 "clap_builder",
 "clap_derive",
@@ -1901,14 +1904,14 @@ dependencies = [

 [[package]]
 name = "clap_derive"
-version = "4.6.3"
+version = "4.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32f2392eae7f16557a3d727ef3a12e57b2b2ca6f98566a5f4fb41ffe305df077"
+checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061"
 dependencies = [
 "heck",
 "proc-macro2",
 "quote",
- "syn 2.0.119",
+ "syn 3.0.3",
 ]

 [[package]]
@@ -2319,7 +2322,7 @@ version = "0.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
 "rand_core 0.6.4",
 "subtle",
 "zeroize",
@@ -2344,11 +2347,11 @@ dependencies = [

 [[package]]
 name = "crypto-common"
-version = "0.1.7"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
+checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
 "typenum",
 ]

@@ -3545,7 +3548,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
 dependencies = [
 "block-buffer 0.10.4",
 "const-oid 0.9.6",
- "crypto-common 0.1.7",
+ "crypto-common 0.1.6",
 "subtle",
 ]

@@ -3798,7 +3801,7 @@ dependencies = [
 "crypto-bigint 0.5.5",
 "digest 0.10.7",
 "ff 0.13.1",
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
 "group 0.13.0",
 "hkdf 0.12.4",
 "pem-rfc7468 0.7.0",
@@ -4244,9 +4247,9 @@ dependencies = [

 [[package]]
 name = "generic-array"
-version = "0.14.7"
+version = "0.14.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
+checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
 dependencies = [
 "typenum",
 "version_check",
@@ -4259,7 +4262,7 @@ version = "1.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
 "rustversion",
 "typenum",
 ]
@@ -4340,9 +4343,9 @@ dependencies = [

 [[package]]
 name = "glob"
-version = "0.3.3"
+version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
+checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"

 [[package]]
 name = "gloo-timers"
@@ -5259,7 +5262,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
 dependencies = [
 "block-padding 0.3.3",
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
 ]

 [[package]]
@@ -5710,9 +5713,9 @@ checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c"

 [[package]]
 name = "libc"
-version = "0.2.187"
+version = "0.2.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7743783ea728ef5c31194c6590797eed286449b4a4e87d626d8a51f0a94e732"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"

 [[package]]
 name = "libflate"
@@ -6080,9 +6083,9 @@ dependencies = [

 [[package]]
 name = "metrique"
-version = "0.1.27"
+version = "0.1.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d6737991308ea171744eac9bb26885d6caa0b747d3cca72f7a69f38e689a1886"
+checksum = "aa466af30a9fe0b1db1dae097e0ce7ddac4b666b1d3a9f5c682e889272511dd8"
 dependencies = [
 "itoa",
 "jiff",
@@ -6110,9 +6113,9 @@ dependencies = [

 [[package]]
 name = "metrique-macro"
-version = "0.1.18"
+version = "0.1.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6512443e1f72c1faa670da0e7cc347dbdd6f17b8e97c3a10d0de56365887f18d"
+checksum = "1f5febfaf14fea234b60e0ad43c728db274033893a38d0fa1f87d9b7056d3d61"
 dependencies = [
 "Inflector",
 "darling 0.23.0",
@@ -7947,9 +7950,9 @@ dependencies = [

 [[package]]
 name = "pyroscope"
-version = "2.1.0"
+version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b9adfbc87341b5976c0bbab196ed2e2c188e2acf9f4d434d46eb9433f0f529c"
+checksum = "59b1989a61e856eba4ef903eadad8313f8376e88e7c9cc5d5db8217d832a64d9"
 dependencies = [
 "aligned-vec",
 "backtrace",
@@ -8269,8 +8272,8 @@ version = "0.14.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055"
 dependencies = [
+ "aws-lc-rs",
 "pem",
- "ring",
 "rustls-pki-types",
 "time",
 "x509-parser",
@@ -8394,7 +8397,7 @@ checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 3.0.2",
+ "syn 3.0.3",
 ]

 [[package]]
@@ -8949,6 +8952,7 @@ dependencies = [
 "url",
 "urlencoding",
 "uuid",
+ "zeroize",
 "zip",
 "zstd",
 ]
@@ -9090,6 +9094,7 @@ dependencies = [
 "byteorder",
 "bytes",
 "bytesize",
+ "chacha20",
 "chacha20poly1305",
 "chrono",
 "criterion",
@@ -9186,6 +9191,7 @@ dependencies = [
 "urlencoding",
 "uuid",
 "xxhash-rust",
+ "zeroize",
 ]

 [[package]]
@@ -9537,8 +9543,10 @@ dependencies = [
 "serde",
 "serde_json",
 "starshard",
+ "tempfile",
 "thiserror 2.0.19",
 "tokio",
+ "tokio-util",
 "tracing",
 "tracing-subscriber",
 "url",
@@ -9955,6 +9963,7 @@ dependencies = [
 "chrono",
 "criterion",
 "deadpool-postgres",
+ "futures-util",
 "hashbrown 0.17.1",
 "hyper",
 "hyper-rustls",
@@ -9964,6 +9973,7 @@ dependencies = [
 "mysql_async",
 "parking_lot",
 "pulsar",
+ "rayon",
 "rcgen",
 "redis",
 "regex",
@@ -9988,6 +9998,7 @@ dependencies = [
 "tokio",
 "tokio-postgres",
 "tokio-postgres-rustls",
+ "tokio-util",
 "tracing",
 "tracing-subscriber",
 "url",
@@ -10184,7 +10195,6 @@ dependencies = [
 "aws-lc-rs",
 "log",
 "once_cell",
- "ring",
 "rustls-pki-types",
 "rustls-webpki",
 "subtle",
@@ -10455,7 +10465,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
 dependencies = [
 "base16ct 0.2.0",
 "der 0.7.10",
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
 "pkcs8 0.10.2",
 "subtle",
 "zeroize",
@@ -10561,7 +10571,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 3.0.2",
+ "syn 3.0.3",
 ]

 [[package]]
@@ -10615,7 +10625,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 3.0.2",
+ "syn 3.0.3",
 ]

 [[package]]
@@ -11320,9 +11330,9 @@ dependencies = [

 [[package]]
 name = "syn"
-version = "3.0.2"
+version = "3.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3"
+checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -11541,7 +11551,7 @@ checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 3.0.2",
+ "syn 3.0.3",
 ]

 [[package]]
@@ -11742,9 +11752,9 @@ dependencies = [

 [[package]]
 name = "tokio-stream"
-version = "0.1.18"
+version = "0.1.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70"
+checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b"
 dependencies = [
 "futures-core",
 "pin-project-lite",
@@ -11765,15 +11775,16 @@ dependencies = [

 [[package]]
 name = "tokio-util"
-version = "0.7.18"
+version = "0.7.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098"
+checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52"
 dependencies = [
 "bytes",
 "futures-core",
 "futures-io",
 "futures-sink",
 "futures-util",
+ "libc",
 "pin-project-lite",
 "tokio",
 ]
@@ -11784,6 +11795,7 @@ version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f591660438b3038dd04d16c938271c79e7e06260ad2ea2885a4861bfb238605d"
 dependencies = [
+ "aws-lc-rs",
 "base64 0.22.1",
 "bytes",
 "futures-core",
@@ -11791,7 +11803,6 @@ dependencies = [
 "http 1.4.2",
 "httparse",
 "rand 0.8.7",
- "ring",
 "rustls-pki-types",
 "tokio",
 "tokio-rustls",
@@ -12823,12 +12834,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202"
 dependencies = [
 "asn1-rs",
+ "aws-lc-rs",
 "data-encoding",
 "der-parser",
 "lazy_static",
 "nom 7.1.3",
 "oid-registry",
- "ring",
 "rusticata-macros",
 "thiserror 2.0.19",
 "time",
@@ -12876,9 +12887,9 @@ dependencies = [

 [[package]]
 name = "xxhash-rust"
-version = "0.8.17"
+version = "0.8.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "985eec839aaf2a1270af8f4ebcf63cf9401cfd90f0902f97c28d9f104ffbde72"
+checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6"

 [[package]]
 name = "yansi"
@@ -140,7 +140,7 @@ mysql_async = { default-features = false, version = "0.37" }
 async-compression = { version = "0.4.42" }
 async-recursion = "1.1.1"
 async-trait = "0.1.91"
-async-nats = "0.50.0"
+async-nats = { version = "0.50.0", default-features = false }
 axum = "0.8.9"
 futures = "0.3.33"
 futures-core = "0.3.33"
@@ -161,9 +161,9 @@ rustfs-kafka-async = { version = "1.2.0" }
 socket2 = { version = "0.6.5" }
 tokio = { version = "1.53.1" }
 tokio-rustls = { default-features = false, version = "0.26.4" }
-tokio-stream = { version = "0.1.18" }
+tokio-stream = { version = "0.1.19" }
 tokio-test = "0.4.5"
-tokio-util = { version = "0.7.18" }
+tokio-util = { version = "0.7.19" }
 tonic = { version = "0.14.6" }
 tonic-prost = { version = "0.14.6" }
 tonic-prost-build = { version = "0.14.6" }
@@ -194,6 +194,7 @@ aes-gcm = { version = "=0.11.0" }
 argon2 = { version = "=0.6.0-rc.8" }
 blake2 = "=0.11.0-rc.6"
 chacha20poly1305 = { version = "=0.11.0" }
+chacha20 = { version = "=0.10.1", features = ["xchacha"] }
 crc-fast = "1.10.0"
 hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "10.4.0" }
@@ -225,16 +226,16 @@ arc-swap = "1.9.2"
 astral-tokio-tar = "0.6.4"
 atoi = "3.1.0"
 atomic_enum = "0.3.0"
-aws-config = { version = "1.9.0" }
+aws-config = { version = "1.10.0" }
 aws-credential-types = { version = "1.3.0" }
-aws-sdk-s3 = { default-features = false, version = "1.138.1" }
+aws-sdk-s3 = { default-features = false, version = "1.139.0" }
 aws-smithy-http-client = { default-features = false, version = "1.2.0" }
 aws-smithy-runtime-api = { version = "1.13.0" }
 aws-smithy-types = { version = "1.6.1" }
 base64 = "0.22.1"
 base64-simd = "0.8.0"
 brotli = "8.0.4"
-clap = { version = "4.6.3" }
+clap = { version = "4.6.4" }
 const-str = { version = "1.1.0" }
 convert_case = "0.11.0"
 criterion = { version = "0.8" }
@@ -247,7 +248,7 @@ derive_builder = "0.20.2"
 enumset = "1.1.14"
 faster-hex = "0.10.0"
 flate2 = "1.1.9"
-glob = "0.3.3"
+glob = "0.3.4"
 google-cloud-storage = "1.16.0"
 google-cloud-auth = "1.14.0"
 hashbrown = { version = "0.17.1" }
@@ -256,7 +257,7 @@ hex-simd = "0.8.0"
 highway = { version = "1.3.0" }
 ipnetwork = { version = "0.21.1" }
 lazy_static = "1.5.0"
-libc = "0.2.187"
+libc = "0.2.189"
 libsystemd = "0.7.2"
 local-ip-address = "0.6.13"
 memmap2 = "0.9.11"
@@ -313,7 +314,7 @@ vaultrs = { version = "0.8.0" }
 tar = "0.4.46"
 walkdir = "2.5.0"
 windows = { version = "0.62.2" }
-xxhash-rust = { version = "0.8.17" }
+xxhash-rust = { version = "0.8.18" }
 zip = "8.6.0"
 zstd = "0.13.3"

@@ -326,13 +327,13 @@ opentelemetry-otlp = { version = "0.32.0" }
 opentelemetry_sdk = { version = "0.32.1" }
 opentelemetry-semantic-conventions = { version = "0.32.1" }
 opentelemetry-stdout = { version = "0.32.0" }
-pyroscope = { version = "2.1.0" }
+pyroscope = { version = "2.1.1" }

 # FTP and SFTP
 libunftp = { version = "0.23.0" }
 unftp-core = "0.1.0"
 suppaftp = { version = "10.0.1" }
-rcgen = "0.14.8"
+rcgen = { version = "0.14.8", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
 russh = { version = "0.62.3" }
 russh-sftp = "2.3.0"

@@ -292,8 +292,8 @@ impl AuditPipeline {
    }

    pub async fn snapshot_target_health(&self) -> Vec<rustfs_targets::RuntimeTargetHealthSnapshot> {
-        let registry = self.registry.lock().await;
-        registry.runtime_manager().health_snapshots().await
+        let targets = self.registry.lock().await.list_target_values();
+        rustfs_targets::health_snapshots_for_targets(targets).await
    }
 }

@@ -570,7 +570,7 @@ mod tests {
    use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
    use rustfs_targets::{StoreError, Target, TargetError};
    use std::sync::Arc;
-    use tokio::sync::Mutex;
+    use tokio::sync::{Mutex, Notify};

    /// Mock target whose `save()` outcome is fixed at construction so tests can
    /// force full-success / full-failure / partial-failure fan-outs.
@@ -578,6 +578,7 @@ mod tests {
    struct MockTarget {
        id: TargetID,
        fail: bool,
+        health_gate: Option<(Arc<Notify>, Arc<Notify>)>,
    }

    impl MockTarget {
@@ -585,8 +586,14 @@ mod tests {
            Self {
                id: TargetID::new(id.to_string(), "webhook".to_string()),
                fail,
+                health_gate: None,
            }
        }
+
+        fn with_health_gate(mut self, started: Arc<Notify>, release: Arc<Notify>) -> Self {
+            self.health_gate = Some((started, release));
+            self
+        }
    }

    #[async_trait]
@@ -599,6 +606,10 @@ mod tests {
        }

        async fn is_active(&self) -> Result<bool, TargetError> {
+            if let Some((started, release)) = &self.health_gate {
+                started.notify_one();
+                release.notified().await;
+            }
            Ok(true)
        }

@@ -673,6 +684,24 @@ mod tests {
        pipeline.dispatch(entry()).await.expect("no targets should return Ok");
    }

+    #[tokio::test]
+    async fn health_probe_does_not_hold_the_registry_lock() {
+        let started = Arc::new(Notify::new());
+        let release = Arc::new(Notify::new());
+        let pipeline = pipeline_with(vec![MockTarget::new("blocked", false).with_health_gate(started.clone(), release.clone())]);
+        let registry = Arc::clone(&pipeline.registry);
+        let snapshot_task = tokio::spawn(async move { pipeline.snapshot_target_health().await });
+        started.notified().await;
+
+        let guard = tokio::time::timeout(std::time::Duration::from_secs(1), registry.lock())
+            .await
+            .expect("network health probe must not retain the audit registry lock");
+        drop(guard);
+        release.notify_one();
+
+        assert_eq!(snapshot_task.await.expect("snapshot task should finish").len(), 1);
+    }
+
    // backlog#962: dispatch_batch must mirror dispatch and propagate a
    // whole-batch loss instead of returning Ok.
    #[tokio::test]
@@ -1068,7 +1068,8 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_s3() -> Result<(), B

 #[tokio::test]
 #[serial]
-async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+async fn test_anonymous_post_object_bucket_default_sse_kms_requires_kms() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
+{
    init_logging();

    let mut env = RustFSTestEnvironment::new().await?;
@@ -1128,14 +1129,23 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(),
        .send()
        .await?;

-    assert_eq!(post_resp.status(), reqwest::StatusCode::NO_CONTENT);
-
-    let head = admin_client.head_object().bucket(bucket).key(object_key).send().await?;
-    assert_eq!(head.server_side_encryption().map(|value| value.as_str()), Some("aws:kms"));
-
-    let uploaded = admin_client.get_object().bucket(bucket).key(object_key).send().await?;
-    let uploaded = uploaded.body.collect().await?.into_bytes();
-    assert_eq!(uploaded.as_ref(), expected_body.as_slice());
+    let status = post_resp.status();
+    let response_body = post_resp.text().await?;
+    assert_eq!(status, reqwest::StatusCode::BAD_REQUEST);
+    assert!(
+        response_body.contains("configured KMS service"),
+        "unexpected response body: {response_body}"
+    );
+    assert!(
+        admin_client
+            .head_object()
+            .bucket(bucket)
+            .key(object_key)
+            .send()
+            .await
+            .is_err(),
+        "failed SSE-KMS POST must not create an object"
+    );

    Ok(())
 }
@@ -156,7 +156,43 @@ async fn spawn_event_collector() -> Result<(String, mpsc::UnboundedReceiver<Valu
    Ok((format!("http://{endpoint_ip}.nip.io:{port}/events"), rx, handle))
 }

-fn spawn_https_event_collector(ca_path: &Path) -> Result<(String, Arc<AtomicBool>, thread::JoinHandle<()>), BoxError> {
+struct HttpsEventCollector {
+    endpoint: String,
+    running: Arc<AtomicBool>,
+    handle: Option<thread::JoinHandle<()>>,
+    events: mpsc::UnboundedReceiver<Value>,
+}
+
+impl HttpsEventCollector {
+    fn endpoint(&self) -> &str {
+        &self.endpoint
+    }
+
+    fn events_mut(&mut self) -> &mut mpsc::UnboundedReceiver<Value> {
+        &mut self.events
+    }
+
+    fn shutdown(&mut self) -> TestResult {
+        self.running.store(false, Ordering::Relaxed);
+        if let Ok(parsed) = self.endpoint.parse::<reqwest::Url>()
+            && let Some(port) = parsed.port()
+        {
+            let _ = std::net::TcpStream::connect(("127.0.0.1", port));
+        }
+        if let Some(handle) = self.handle.take() {
+            handle.join().map_err(|_| "https event collector thread panicked")?;
+        }
+        Ok(())
+    }
+}
+
+impl Drop for HttpsEventCollector {
+    fn drop(&mut self) {
+        let _ = self.shutdown();
+    }
+}
+
+fn spawn_https_event_collector(ca_path: &Path) -> Result<HttpsEventCollector, BoxError> {
    use rustls::{
        ServerConfig,
        pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer},
@@ -188,48 +224,103 @@ fn spawn_https_event_collector(ca_path: &Path) -> Result<(String, Arc<AtomicBool

    let running = Arc::new(AtomicBool::new(true));
    let server_running = Arc::clone(&running);
+    let (tx, events) = mpsc::unbounded_channel();
    let handle = thread::spawn(move || {
+        let mut connections = Vec::new();
        while server_running.load(Ordering::Relaxed) {
            match listener.accept() {
                Ok((stream, _)) => {
                    let config = Arc::clone(&server_config);
-                    handle_https_probe(stream, config);
+                    let tx = tx.clone();
+                    connections.push(thread::spawn(move || {
+                        let _ = handle_https_request(stream, config, tx);
+                    }));
                }
                Err(err) if err.kind() == ErrorKind::WouldBlock => thread::sleep(Duration::from_millis(20)),
                Err(_) => break,
            }
        }
+        for connection in connections {
+            let _ = connection.join();
+        }
    });

-    Ok((format!("https://{endpoint_host}:{}/events", addr.port()), running, handle))
+    Ok(HttpsEventCollector {
+        endpoint: format!("https://{endpoint_host}:{}/events", addr.port()),
+        running,
+        handle: Some(handle),
+        events,
+    })
 }

-fn handle_https_probe(stream: std::net::TcpStream, server_config: Arc<rustls::ServerConfig>) {
-    use std::io::{Read, Write};
-
-    let _ = stream.set_read_timeout(Some(Duration::from_secs(5)));
-    let _ = stream.set_write_timeout(Some(Duration::from_secs(5)));
-    let Ok(connection) = rustls::ServerConnection::new(server_config) else {
-        return;
+fn read_sync_http_message<R: std::io::Read>(stream: &mut R) -> Result<(String, Vec<u8>), BoxError> {
+    let mut buffer = Vec::new();
+    let mut chunk = [0_u8; 4096];
+    let header_end = loop {
+        let read = stream.read(&mut chunk)?;
+        if read == 0 {
+            return Err("connection closed before request headers were complete".into());
+        }
+        buffer.extend_from_slice(&chunk[..read]);
+        if let Some(pos) = buffer.windows(4).position(|window| window == b"\r\n\r\n") {
+            break pos;
+        }
    };
-    let mut tls_stream = rustls::StreamOwned::new(connection, stream);
-    let mut buf = [0u8; 1024];
-    if tls_stream.read(&mut buf).is_err() {
-        return;
+
+    let header_text = std::str::from_utf8(&buffer[..header_end])?;
+    let mut lines = header_text.split("\r\n");
+    let method = lines
+        .next()
+        .and_then(|line| line.split_whitespace().next())
+        .ok_or("missing request method")?
+        .to_string();
+    let mut content_length = 0usize;
+    for line in lines {
+        if let Some((name, value)) = line.split_once(':')
+            && name.trim().eq_ignore_ascii_case("content-length")
+        {
+            content_length = value.trim().parse()?;
+        }
    }
-    let response = "HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n";
-    let _ = tls_stream.write_all(response.as_bytes());
-    let _ = tls_stream.flush();
+
+    let body_offset = header_end + 4;
+    while buffer.len().saturating_sub(body_offset) < content_length {
+        let read = stream.read(&mut chunk)?;
+        if read == 0 {
+            return Err("connection closed before request body was complete".into());
+        }
+        buffer.extend_from_slice(&chunk[..read]);
+    }
+    Ok((method, buffer[body_offset..body_offset + content_length].to_vec()))
 }

-fn stop_https_event_collector(endpoint: &str, running: Arc<AtomicBool>, handle: thread::JoinHandle<()>) -> TestResult {
-    running.store(false, Ordering::Relaxed);
-    if let Ok(parsed) = endpoint.parse::<reqwest::Url>()
-        && let Some(port) = parsed.port()
-    {
-        let _ = std::net::TcpStream::connect(("127.0.0.1", port));
+fn handle_https_request(
+    stream: std::net::TcpStream,
+    server_config: Arc<rustls::ServerConfig>,
+    tx: mpsc::UnboundedSender<Value>,
+) -> Result<(), BoxError> {
+    use std::io::Write;
+
+    stream.set_nonblocking(false)?;
+    stream.set_read_timeout(Some(Duration::from_secs(5)))?;
+    stream.set_write_timeout(Some(Duration::from_secs(5)))?;
+    let connection = rustls::ServerConnection::new(server_config)?;
+    let mut tls_stream = rustls::StreamOwned::new(connection, stream);
+    let (method, body) = read_sync_http_message(&mut tls_stream)?;
+    let response = "HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n";
+    tls_stream.write_all(response.as_bytes())?;
+    tls_stream.flush()?;
+    tls_stream.conn.send_close_notify();
+    while tls_stream.conn.wants_write() {
+        tls_stream.conn.write_tls(&mut tls_stream.sock)?;
+    }
+    let _ = tls_stream.sock.shutdown(std::net::Shutdown::Write);
+    if method == "POST"
+        && !body.is_empty()
+        && let Ok(event) = serde_json::from_slice(&body)
+    {
+        let _ = tx.send(event);
    }
-    handle.join().map_err(|_| "https event collector thread panicked")?;
    Ok(())
 }

@@ -415,17 +506,20 @@ async fn wait_for_target_registered(env: &RustFSTestEnvironment, target_name: &s
    Err(format!("target {target_name} was not registered in admin ARNs").into())
 }

-async fn wait_for_target_listed(env: &RustFSTestEnvironment, target_name: &str) -> TestResult {
+async fn wait_for_target_online(env: &RustFSTestEnvironment, target_name: &str) -> TestResult {
    let url = format!("{}/rustfs/admin/v3/target/list", env.url);
    for _ in 0..40 {
        let response = signed_admin_request(env, http::Method::GET, &url, None).await?;
        if response.status() == StatusCode::OK {
            let body: Value = serde_json::from_slice(&response.bytes().await?)?;
+            if body["notify_enabled"].as_bool() != Some(true) {
+                return Err(format!("admin target list did not report notify_enabled=true: {body}").into());
+            }
            let listed = body["notification_endpoints"].as_array().is_some_and(|endpoints| {
                endpoints.iter().any(|endpoint| {
                    endpoint["account_id"].as_str() == Some(target_name)
                        && endpoint["service"].as_str() == Some("webhook")
-                        && endpoint["status"].as_str().is_some()
+                        && endpoint["status"].as_str() == Some("online")
                })
            });
            if listed {
@@ -434,7 +528,7 @@ async fn wait_for_target_listed(env: &RustFSTestEnvironment, target_name: &str)
        }
        tokio::time::sleep(Duration::from_millis(250)).await;
    }
-    Err(format!("target {target_name} was not listed in admin targets").into())
+    Err(format!("target {target_name} did not become online in admin targets").into())
 }

 /// Binds a bucket to a webhook target for ObjectCreated:*/ObjectRemoved:* events,
@@ -484,11 +578,11 @@ fn trimmed_etag(value: Option<&str>) -> Option<String> {
 // ---------------------------------------------------------------------------

 /// Regression for rustfs#5052: with the notify module enabled through
-/// RUSTFS_NOTIFY_ENABLE, an HTTPS webhook using a configured CA must be accepted
-/// and remain visible in the admin target list.
+/// RUSTFS_NOTIFY_ENABLE, an HTTPS webhook using a configured CA must become
+/// online and receive a real S3 event POST.
 #[tokio::test]
 #[serial]
-async fn test_https_webhook_target_lists_with_notify_env_enabled() -> TestResult {
+async fn test_https_webhook_target_delivers_event_with_notify_env_enabled() -> TestResult {
    init_logging();

    let mut env = RustFSTestEnvironment::new().await?;
@@ -496,22 +590,40 @@ async fn test_https_webhook_target_lists_with_notify_env_enabled() -> TestResult
        .await?;

    let ca_path = Path::new(&env.temp_dir).join("https-webhook-ca.pem");
-    let (endpoint, running, handle) = spawn_https_event_collector(&ca_path)?;
+    let mut collector = spawn_https_event_collector(&ca_path)?;
    let target = "peri1https";
+    let bucket = "peri1-https-events";
+    let key = "uploads/https.dat";
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;

    configure_webhook_target_with_key_values(
        &env,
        target,
        vec![
-            ("endpoint", endpoint.clone()),
+            ("endpoint", collector.endpoint().to_string()),
            ("client_ca", ca_path.to_string_lossy().into_owned()),
        ],
    )
    .await?;
-    wait_for_target_listed(&env, target).await?;
+    wait_for_target_online(&env, target).await?;
+    wait_for_target_registered(&env, target).await?;
+    put_notification_config(&client, bucket, target, "uploads/", ".dat").await?;
+
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"https webhook event body"))
+        .send()
+        .await?;
+    let event = wait_for_event(collector.events_mut(), key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
+    assert_eq!(event["EventName"].as_str(), Some("s3:ObjectCreated:Put"));
+    assert_eq!(event["Records"][0]["s3"]["bucket"]["name"].as_str(), Some(bucket));
+    assert_eq!(event_key(&event).as_deref(), Some(key));

    env.stop_server();
-    stop_https_event_collector(&endpoint, running, handle)?;
+    collector.shutdown()?;
    Ok(())
 }

@@ -126,6 +126,8 @@ rustfs-madmin.workspace = true
 reqwest = { workspace = true }
 aes-gcm = { workspace = true, features = ["rand_core"] }
 chacha20poly1305.workspace = true
+chacha20.workspace = true
+zeroize.workspace = true
 aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 urlencoding = { workspace = true }
 smallvec = { workspace = true, features = ["serde"] }
@@ -241,8 +241,10 @@ pub mod config {
        pub use crate::config::com::{
            COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS,
            ServerConfigCorruptError, delete_config, is_server_config_corrupt_error, lookup_configs, read_config,
-            read_config_no_lock, read_config_with_metadata, read_config_without_migrate, save_config, save_config_with_opts,
-            save_server_config, try_migrate_server_config,
+            read_config_no_lock, read_config_with_metadata, read_config_without_migrate, read_config_without_migrate_no_lock,
+            read_existing_server_config_no_lock, save_config, save_config_no_lock, save_config_with_opts, save_server_config,
+            save_server_config_no_lock, try_migrate_server_config, with_config_object_read_lock, with_config_object_write_lock,
+            with_server_config_read_lock, with_server_config_write_lock,
        };
    }

@@ -359,9 +361,10 @@ pub mod notification {
 pub mod object {
    pub use crate::object_api::{
        BLOCK_SIZE_V2, ERASURE_ALGORITHM, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource,
-        GetObjectReader, ObjectInfo, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, StreamConsumer,
-        get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
-        register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
+        GetObjectReader, GetObjectSse, ObjectInfo, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader,
+        StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
+        register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
+        unregister_object_mutation_hook,
    };
    pub use crate::store::PreparedGetObjectReader;
 }
@@ -383,9 +386,9 @@ pub mod rio {

 pub mod rpc {
    pub use crate::cluster::rpc::{
-        LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
-        SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers,
-        gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
+        LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client,
+        SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor,
+        gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
        node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
        sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof,
        verify_tonic_rpc_signature,
@@ -396,6 +399,13 @@ pub mod set_disk {
    pub use crate::set_disk::{DEFAULT_READ_BUFFER_SIZE, SetDisks, get_lock_acquire_timeout, is_valid_storage_class};
 }

+pub mod sse {
+    pub use crate::sse::{
+        ManagedDekProvider, ManagedSseScheme, PersistedEncryptionError, PersistedManagedEncryption,
+        classify_persisted_managed_encryption, decrypt_minio_static_kms_dek,
+    };
+}
+
 pub mod store_list {
    pub use crate::store::list_objects::{ListPathOptions, max_keys_plus_one};
 }
@@ -2809,6 +2809,8 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
        Err(err) => return Err(std::io::Error::other(err)),
    };

+    tgt_client.validate_remote_version_id(&oi.transitioned_object.version_id)?;
+
    let ret = new_getobjectreader(rs, oi, opts, h);
    if let Err(err) = ret {
        return Err(error_resp_to_object_err(err, vec![bucket, object]));
@@ -4046,6 +4048,56 @@ mod tests {
        );
    }

+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    async fn transitioned_get_rejects_nonempty_remote_version_before_backend_io() {
+        let manager = TierConfigMgr::new();
+        let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+        let backend = register_mock_tier(&manager, &tier).await;
+        let remote_object = format!("remote/{}", Uuid::new_v4());
+        let body = Bytes::from_static(b"transitioned object body");
+        let remote_version = backend
+            .put(
+                &remote_object,
+                ReaderImpl::Body(body.clone()),
+                i64::try_from(body.len()).expect("body length should fit"),
+            )
+            .await
+            .expect("mock remote object should be stored");
+        backend.set_reject_non_empty_remote_versions(true);
+        let object_info = ObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "object".to_string(),
+            size: i64::try_from(body.len()).expect("body length should fit"),
+            transitioned_object: TransitionedObject {
+                name: remote_object,
+                version_id: remote_version,
+                status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
+                tier,
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+
+        let err = match get_transitioned_object_reader_with_tier_manager(
+            &object_info.bucket,
+            &object_info.name,
+            &None,
+            &HeaderMap::new(),
+            &object_info,
+            &ObjectOptions::default(),
+            &manager,
+        )
+        .await
+        {
+            Ok(_) => panic!("a provider that rejects versioned GET must fail before remote IO"),
+            Err(err) => err,
+        };
+
+        assert!(err.to_string().contains("requires an unversioned remote object"));
+        assert_eq!(backend.get_count().await, 0);
+    }
+
    #[cfg(feature = "test-util")]
    #[tokio::test]
    async fn free_version_remote_delete_requires_persisted_destination_identity() {
@@ -152,13 +152,13 @@ pub(crate) fn tier_delete_journal_object_name(je: &Jentry) -> String {
    )
 }

-fn decode_tier_delete_journal_entry(data: &[u8]) -> Result<Jentry> {
+pub(crate) fn decode_tier_delete_journal_entry(data: &[u8]) -> Result<Jentry> {
    let persisted: PersistedTierDeleteJournalEntry =
        serde_json::from_slice(data).map_err(|err| Error::other(format!("decode tier delete journal failed: {err}")))?;
    persisted.into_jentry()
 }

-fn encode_tier_delete_journal_entry(je: &Jentry) -> Result<Vec<u8>> {
+pub(crate) fn encode_tier_delete_journal_entry(je: &Jentry) -> Result<Vec<u8>> {
    serde_json::to_vec(&PersistedTierDeleteJournalEntry::from_jentry(je))
        .map_err(|err| Error::other(format!("encode tier delete journal failed: {err}")))
 }
@@ -339,6 +339,8 @@ async fn delete_object_from_remote_tier_raw_with_lease(
    lease: &TierOperationLease,
    version_id_exact: bool,
 ) -> Result<(), std::io::Error> {
+    lease.validate_remote_version_id(rv_id)?;
+
    if remote_delete_breaker_is_open(Instant::now()).await {
        metrics::counter!(METRIC_DELETE_REMOTE_BREAKER_TOTAL).increment(1);
        return Err(std::io::Error::other(ERR_REMOTE_DELETE_BREAKER_OPEN));
@@ -622,6 +624,46 @@ mod test {
        assert_eq!(backend.remove_count().await, 1);
    }

+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    async fn journal_delete_rejects_nonempty_remote_version_before_backend_io() {
+        let manager = crate::services::tier::tier::TierConfigMgr::new();
+        let backend = crate::services::tier::test_util::register_mock_tier(&manager, "WARM").await;
+        let lease = crate::services::tier::tier::TierConfigMgr::acquire_operation_lease(&manager, "WARM")
+            .await
+            .expect("test tier lease should be available");
+        let identity = lease.backend_identity();
+        drop(lease);
+        backend.set_reject_non_empty_remote_versions(true);
+
+        let err = delete_object_from_remote_tier_idempotent_with_manager_and_identity(
+            "remote/object",
+            "remote-version",
+            "WARM",
+            identity,
+            &manager,
+            true,
+        )
+        .await
+        .expect_err("a provider that rejects a versioned delete must fail before remote IO");
+
+        assert!(err.to_string().contains("requires an unversioned remote object"));
+        assert_eq!(backend.remove_count().await, 0);
+
+        delete_object_from_remote_tier_idempotent_with_manager_and_identity(
+            "remote/object",
+            "",
+            "WARM",
+            identity,
+            &manager,
+            false,
+        )
+        .await
+        .expect("unversioned remote delete should continue without a version ID");
+
+        assert_eq!(backend.remove_versions().await, vec![("remote/object".to_string(), String::new())]);
+    }
+
    #[test]
    fn breaker_opens_at_threshold_and_recovers_after_window() {
        let mut breaker = RemoteDeleteBreaker::new(3, Duration::from_secs(30));
@@ -26,6 +26,7 @@ use crate::bucket::lifecycle::tier_sweeper::delete_object_from_remote_tier_idemp
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result as EcstoreResult};
 use crate::object_api::ObjectOptions;
+use crate::services::tier::{tier::TierConfigMgr, warm_backend::TransitionCandidateProbe};
 use crate::storage_api_contracts::{list::ListOperations as _, object::ObjectOperations as _};
 use crate::store::ECStore;

@@ -612,6 +613,11 @@ pub enum TransitionTransactionRecoveryOutcome {
    Retained,
 }

+pub(crate) fn decode_transition_transaction_record(object: &str, data: &[u8]) -> Result<TransitionTransaction> {
+    let transaction_id = transition_transaction_id_from_record_object_name(object)?;
+    TransitionTransaction::decode(transaction_id, data)
+}
+
 fn transition_transaction_id_from_record_object_name(object: &str) -> Result<Uuid> {
    let prefix = format!("{TRANSITION_TRANSACTION_RECORD_PREFIX}/");
    let suffix = object
@@ -658,20 +664,84 @@ pub async fn process_transition_transaction_record(
            }
            Err(err) => Err(err),
        },
-        TransitionTransactionState::LocalCommitStarted if local_commit_matches_transaction(api.clone(), transaction).await? => {
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
-            Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
+        TransitionTransactionState::LocalCommitStarted => {
+            match local_commit_matches_transaction(api.clone(), transaction).await {
+                Ok(true) => {
+                    delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                    Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
+                }
+                Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained),
+                Err(err) if transition_source_is_missing(&err) => Ok(TransitionTransactionRecoveryOutcome::Retained),
+                Err(err) => Err(err),
+            }
        }
        TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => {
            delete_transition_transaction_record(api, transaction.transaction_id).await?;
            Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
        }
-        TransitionTransactionState::UploadStarted
-        | TransitionTransactionState::UploadOutcomeUnknown
-        | TransitionTransactionState::LocalCommitStarted => Ok(TransitionTransactionRecoveryOutcome::Retained),
+        TransitionTransactionState::UploadOutcomeUnknown => recover_unknown_upload_outcome(api, transaction).await,
+        TransitionTransactionState::UploadStarted => Ok(TransitionTransactionRecoveryOutcome::Retained),
    }
 }

+async fn recover_unknown_upload_outcome(
+    api: Arc<ECStore>,
+    transaction: &TransitionTransaction,
+) -> EcstoreResult<TransitionTransactionRecoveryOutcome> {
+    let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
+        &api.tier_config_mgr(),
+        &transaction.tier_name,
+        transaction.backend_fingerprint,
+    )
+    .await
+    .map_err(Error::other)?;
+
+    match lease
+        .probe_transition_candidate(&transaction.remote_object)
+        .await
+        .map_err(Error::other)?
+    {
+        TransitionCandidateProbe::Missing => {
+            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
+        }
+        TransitionCandidateProbe::UnversionedPresent => {
+            cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::unversioned()).await
+        }
+        TransitionCandidateProbe::VersionedPresent(version_id) => {
+            cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await
+        }
+        TransitionCandidateProbe::Ambiguous | TransitionCandidateProbe::Unsupported => {
+            Ok(TransitionTransactionRecoveryOutcome::Retained)
+        }
+    }
+}
+
+async fn cleanup_recovered_unknown_upload_candidate(
+    api: Arc<ECStore>,
+    transaction: &TransitionTransaction,
+    remote_version: TransitionRemoteVersion,
+) -> EcstoreResult<TransitionTransactionRecoveryOutcome> {
+    let mut cleanup = transaction.clone();
+    cleanup
+        .mark_cleanup_pending(
+            transaction.fence(),
+            TransitionCleanupProof {
+                transaction_id: transaction.transaction_id,
+                write_id: transaction.write_id,
+                remote_object: transaction.remote_object.clone(),
+                remote_version,
+                backend_fingerprint: transaction.backend_fingerprint,
+                decision: TransitionCleanupDecision::RemoteVersionRecoveredAfterCancellation,
+            },
+        )
+        .map_err(transition_transaction_store_error)?;
+    save_transition_transaction_record(api.clone(), &cleanup).await?;
+    delete_transition_remote_candidate(api.clone(), &cleanup).await?;
+    delete_transition_transaction_record(api, cleanup.transaction_id).await?;
+    Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
+}
+
 fn transition_source_is_missing(err: &Error) -> bool {
    matches!(
        err,
@@ -1625,6 +1625,7 @@ mod tests {
                    ..Default::default()
                },
                buffered_body: None,
+                resolved_sse: None,
                body_source: Default::default(),
            })
        }
@@ -124,6 +124,24 @@ fn classify_listing_quorum_failure(errors: &[DiskError]) -> DiskError {
    DiskError::ErasureReadQuorum
 }

+/// Returns true when a metacache listing missed quorum purely because the
+/// volume or path is absent on a quorum of drives, i.e. every recorded failure
+/// is [`DiskError::VolumeNotFound`] or [`DiskError::FileNotFound`].
+///
+/// This is a benign, expected outcome: the caller is handed
+/// `VolumeNotFound`/`FileNotFound` and decides how to react. The most common
+/// trigger is a startup race where the system bucket has not yet been created
+/// on every drive when an early reader (e.g. the IAM config loader) lists it.
+/// It must be distinguished from a listing that failed for a real reason (I/O,
+/// timeout, corruption) so the former is not surfaced at `error`. See
+/// rustfs/rustfs#5076.
+fn is_benign_not_found_listing_failure(errors: &[DiskError]) -> bool {
+    !errors.is_empty()
+        && errors
+            .iter()
+            .all(|err| matches!(err, DiskError::VolumeNotFound | DiskError::FileNotFound))
+}
+
 struct PublishedBytesWriter<W> {
    inner: W,
    published: bool,
@@ -840,17 +858,38 @@ async fn list_path_raw_inner(
                    _ => {}
                });

-                error!(
-                    event = EVENT_METACACHE_LISTING,
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_METACACHE,
-                    bucket = %opts.bucket,
-                    path = %opts.path,
-                    state = "quorum_failed",
-                    error = %combined_err.join(", "),
-                    "Metacache listing quorum failed"
-                );
                let failures = errs.iter().flatten().cloned().collect::<Vec<_>>();
+                // A listing that misses quorum purely because the volume/path is
+                // absent on a quorum of drives is benign and expected — the caller
+                // receives VolumeNotFound/FileNotFound and decides how to react.
+                // The common trigger is a startup race where the system bucket is
+                // not yet created on every drive when an early reader (e.g. the IAM
+                // config loader) lists it, so surfacing it at `error` is misleading
+                // noise (rustfs/rustfs#5076). Keep `error` for listings that failed
+                // for a real reason (I/O, timeout, corruption).
+                if is_benign_not_found_listing_failure(&failures) {
+                    debug!(
+                        event = EVENT_METACACHE_LISTING,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_METACACHE,
+                        bucket = %opts.bucket,
+                        path = %opts.path,
+                        state = "quorum_not_found",
+                        error = %combined_err.join(", "),
+                        "Metacache listing quorum not reached (volume/path absent)"
+                    );
+                } else {
+                    error!(
+                        event = EVENT_METACACHE_LISTING,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_METACACHE,
+                        bucket = %opts.bucket,
+                        path = %opts.path,
+                        state = "quorum_failed",
+                        error = %combined_err.join(", "),
+                        "Metacache listing quorum failed"
+                    );
+                }
                return Err(classify_listing_quorum_failure(&failures));
            }

@@ -1037,6 +1076,21 @@ mod tests {
    use time::OffsetDateTime;
    use uuid::Uuid;

+    #[test]
+    fn benign_not_found_listing_failure_detection() {
+        // Pure not-found quorum misses are benign (the volume/path simply does
+        // not exist on a quorum of drives) and must not be logged at ERROR.
+        assert!(is_benign_not_found_listing_failure(&[DiskError::VolumeNotFound]));
+        assert!(is_benign_not_found_listing_failure(
+            &[DiskError::VolumeNotFound, DiskError::FileNotFound,]
+        ));
+        // No recorded failure is not a not-found case.
+        assert!(!is_benign_not_found_listing_failure(&[]));
+        // Any real error must keep the failure at ERROR severity.
+        assert!(!is_benign_not_found_listing_failure(&[DiskError::VolumeNotFound, DiskError::Timeout,]));
+        assert!(!is_benign_not_found_listing_failure(&[DiskError::DiskNotFound]));
+    }
+
    #[tokio::test]
    async fn list_path_raw_empty_disks_returns_read_quorum() {
        let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
@@ -24,7 +24,7 @@ use crate::client::{
        ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ListVersionsResult, ObjectPart,
    },
    credentials,
-    transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
+    transition_api::{ReaderImpl, RequestMetadata, TransitionClient, collect_response_body},
 };
 use crate::storage_api_contracts::bucket::BucketInfo;
 use http::{HeaderMap, StatusCode};
@@ -88,7 +88,7 @@ impl TransitionClient {
            url_values.insert("max-keys".to_string(), max_keys.to_string());
        }

-        let mut resp = self
+        let resp = self
            .execute_method(
                http::Method::GET,
                &mut RequestMetadata {
@@ -164,7 +164,7 @@ impl TransitionClient {
        Ok(list_bucket_result)
    }

-    pub fn list_object_versions_query(
+    pub async fn list_object_versions_query(
        &self,
        bucket_name: &str,
        opts: &ListObjectsOptions,
@@ -172,93 +172,88 @@ impl TransitionClient {
        version_id_marker: &str,
        delimiter: &str,
    ) -> Result<ListVersionsResult, std::io::Error> {
-        /*if err := s3utils.CheckValidBucketName(bucketName); err != nil {
-          return ListVersionsResult{}, err
+        let mut url_values = HashMap::new();
+        url_values.insert("versions".to_string(), "".to_string());
+        url_values.insert("prefix".to_string(), opts.prefix.clone());
+        url_values.insert("delimiter".to_string(), delimiter.to_string());
+        url_values.insert("encoding-type".to_string(), "url".to_string());
+
+        if !key_marker.is_empty() {
+            url_values.insert("key-marker".to_string(), key_marker.to_string());
        }
-        if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil {
-          return ListVersionsResult{}, err
-        }
-        urlValues := make(url.Values)
-
-        urlValues.Set("versions", "")
-
-        urlValues.Set("prefix", opts.Prefix)
-
-        urlValues.Set("delimiter", delimiter)
-
-        if keyMarker != "" {
-          urlValues.Set("key-marker", keyMarker)
-        }
-
        if opts.max_keys > 0 {
-          urlValues.Set("max-keys", fmt.Sprintf("%d", opts.max_keys))
+            url_values.insert("max-keys".to_string(), opts.max_keys.to_string());
+        }
+        if !version_id_marker.is_empty() {
+            url_values.insert("version-id-marker".to_string(), version_id_marker.to_string());
+        }
+        if opts.with_metadata {
+            url_values.insert("metadata".to_string(), "true".to_string());
        }

-        if versionIDMarker != "" {
-          urlValues.Set("version-id-marker", versionIDMarker)
+        let mut resp = self
+            .execute_method(
+                http::Method::GET,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: "".to_string(),
+                    query_values: url_values,
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    custom_header: opts.headers.clone(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+
+        let resp_status = resp.status();
+        let headers = resp.headers().clone();
+        let body = collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE).await?;
+        if resp_status != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(
+                resp_status,
+                &headers,
+                body,
+                bucket_name,
+                "",
+            )));
        }

-        if opts.WithMetadata {
-          urlValues.Set("metadata", "true")
+        let mut versions = quick_xml::de::from_reader::<_, ListVersionsResult>(body.as_slice())
+            .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?;
+        for version in &mut versions.versions {
+            version.key = decode_s3_name(&version.key, &versions.encoding_type)?;
+        }
+        for marker in &mut versions.delete_markers {
+            marker.key = decode_s3_name(&marker.key, &versions.encoding_type)?;
+        }
+        for prefix in &mut versions.common_prefixes {
+            prefix.prefix = decode_s3_name(&prefix.prefix, &versions.encoding_type)?;
+        }
+        if !versions.next_key_marker.is_empty() {
+            versions.next_key_marker = decode_s3_name(&versions.next_key_marker, &versions.encoding_type)?;
        }

-        urlValues.Set("encoding-type", "url")
-
-        let resp = self.executeMethod(http::Method::GET, &mut RequestMetadata{
-            bucketName:       bucketName,
-            queryValues:      urlValues,
-            contentSHA256Hex: emptySHA256Hex,
-            customHeader:     opts.headers,
-        }).await?;
-        defer closeResponse(resp)
-        if err != nil {
-          return ListVersionsResult{}, err
-        }
-        if resp != nil {
-          if resp.StatusCode != http.StatusOK {
-            return ListVersionsResult{}, httpRespToErrorResponse(resp, bucketName, "")
-          }
-        }
-
-        listObjectVersionsOutput := ListVersionsResult{}
-        err = xml_decoder(resp.Body, &listObjectVersionsOutput)
-        if err != nil {
-          return ListVersionsResult{}, err
-        }
-
-        for i, obj := range listObjectVersionsOutput.Versions {
-          listObjectVersionsOutput.Versions[i].Key, err = decode_s3_name(obj.Key, listObjectVersionsOutput.EncodingType)
-          if err != nil {
-            return listObjectVersionsOutput, err
-          }
-        }
-
-        for i, obj := range listObjectVersionsOutput.CommonPrefixes {
-          listObjectVersionsOutput.CommonPrefixes[i].Prefix, err = decode_s3_name(obj.Prefix, listObjectVersionsOutput.EncodingType)
-          if err != nil {
-            return listObjectVersionsOutput, err
-          }
-        }
-
-        if listObjectVersionsOutput.NextKeyMarker != "" {
-          listObjectVersionsOutput.NextKeyMarker, err = decode_s3_name(listObjectVersionsOutput.NextKeyMarker, listObjectVersionsOutput.EncodingType)
-          if err != nil {
-            return listObjectVersionsOutput, err
-          }
-        }
-
-        Ok(listObjectVersionsOutput)*/
-        Err(std::io::Error::new(
-            ErrorKind::Unsupported,
-            credentials::ErrorResponse {
+        if versions.is_truncated && versions.next_key_marker.is_empty() {
+            return Err(std::io::Error::other(credentials::ErrorResponse {
                sts_error: credentials::STSError {
                    r#type: "".to_string(),
                    code: "NotImplemented".to_string(),
-                    message: format!("list_object_versions_query is not implemented for bucket {bucket_name}"),
+                    message: "Truncated ListObjectVersions response should have next key marker set".to_string(),
                },
                request_id: "".to_string(),
-            },
-        ))
+            }));
+        }
+
+        Ok(versions)
    }

    pub fn list_objects_query(
@@ -364,6 +359,7 @@ impl TransitionClient {
    }
 }

+#[derive(Default)]
 #[allow(dead_code)]
 pub struct ListObjectsOptions {
    reverse_versions: bool,
@@ -431,3 +427,51 @@ fn decode_s3_name(name: &str, encoding_type: &str) -> Result<String, std::io::Er
        }
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn list_versions_xml_preserves_versions_and_delete_markers() {
+        let xml = br#"
+            <ListVersionsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+                <Name>tier-bucket</Name>
+                <Prefix>archive/object</Prefix>
+                <KeyMarker></KeyMarker>
+                <VersionIdMarker></VersionIdMarker>
+                <MaxKeys>2</MaxKeys>
+                <IsTruncated>true</IsTruncated>
+                <NextKeyMarker>archive/object</NextKeyMarker>
+                <NextVersionIdMarker>version-a</NextVersionIdMarker>
+                <Version>
+                    <Key>archive/object</Key>
+                    <VersionId>version-a</VersionId>
+                    <IsLatest>true</IsLatest>
+                    <LastModified>2026-07-22T00:00:00Z</LastModified>
+                    <ETag>&quot;etag-a&quot;</ETag>
+                    <Size>5</Size>
+                    <StorageClass>STANDARD</StorageClass>
+                </Version>
+                <DeleteMarker>
+                    <Key>archive/object</Key>
+                    <VersionId>marker-a</VersionId>
+                    <IsLatest>false</IsLatest>
+                    <LastModified>2026-07-22T00:00:01Z</LastModified>
+                </DeleteMarker>
+            </ListVersionsResult>
+        "#;
+
+        let parsed =
+            quick_xml::de::from_reader::<_, ListVersionsResult>(xml.as_slice()).expect("ListObjectVersions XML should parse");
+
+        assert!(parsed.is_truncated);
+        assert_eq!(parsed.next_key_marker, "archive/object");
+        assert_eq!(parsed.next_version_id_marker, "version-a");
+        assert_eq!(parsed.versions.len(), 1);
+        assert_eq!(parsed.versions[0].key, "archive/object");
+        assert_eq!(parsed.versions[0].version_id, "version-a");
+        assert_eq!(parsed.delete_markers.len(), 1);
+        assert_eq!(parsed.delete_markers[0].version_id, "marker-a");
+    }
+}
@@ -657,6 +657,7 @@ mod tests {
            stream: Box::new(r),
            object_info: Default::default(),
            buffered_body: None,
+            resolved_sse: None,
            body_source: Default::default(),
        });

@@ -687,6 +688,7 @@ mod tests {
            stream: Box::new(r),
            object_info: Default::default(),
            buffered_body: None,
+            resolved_sse: None,
            body_source: Default::default(),
        });
        let buf = read_multipart_part(&mut reader, 100).await.unwrap();
@@ -55,34 +55,38 @@ pub struct ListBucketV2Result {
    pub start_after: String,
 }

-#[allow(dead_code)]
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
 pub struct Version {
-    etag: String,
-    is_latest: bool,
-    key: String,
-    last_modified: OffsetDateTime,
-    owner: Owner,
-    size: i64,
-    storage_class: String,
-    version_id: String,
-    user_metadata: HashMap<String, String>,
-    user_tags: HashMap<String, String>,
-    is_delete_marker: bool,
+    #[serde(rename = "ETag")]
+    pub etag: String,
+    pub is_latest: bool,
+    pub key: String,
+    pub size: i64,
+    pub storage_class: String,
+    pub version_id: String,
+    pub user_metadata: HashMap<String, String>,
+    pub user_tags: HashMap<String, String>,
 }

+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
 pub struct ListVersionsResult {
-    versions: Vec<Version>,
-    common_prefixes: Vec<CommonPrefix>,
-    name: String,
-    prefix: String,
-    delimiter: String,
-    max_keys: i64,
-    encoding_type: String,
-    is_truncated: bool,
-    key_marker: String,
-    version_id_marker: String,
-    next_key_marker: String,
-    next_version_id_marker: String,
+    #[serde(rename = "Version")]
+    pub versions: Vec<Version>,
+    #[serde(rename = "DeleteMarker")]
+    pub delete_markers: Vec<Version>,
+    pub common_prefixes: Vec<CommonPrefix>,
+    pub name: String,
+    pub prefix: String,
+    pub delimiter: String,
+    pub max_keys: i64,
+    pub encoding_type: String,
+    pub is_truncated: bool,
+    pub key_marker: String,
+    pub version_id_marker: String,
+    pub next_key_marker: String,
+    pub next_version_id_marker: String,
 }

 pub struct ListBucketResult {
@@ -153,6 +153,7 @@ pub fn new_getobjectreader<'a>(
                object_info: oi.clone(),
                stream: Box::new(input_reader),
                buffered_body: None,
+                resolved_sse: None,
                body_source: Default::default(),
            };
            r
@@ -166,6 +167,7 @@ pub fn new_getobjectreader<'a>(
            object_info: oi.clone(),
            stream: Box::new(input_reader),
            buffered_body: None,
+            resolved_sse: None,
            body_source: Default::default(),
        });

@@ -37,8 +37,8 @@ pub use http_auth::{
 pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
 pub use internode_data_transport::build_internode_data_transport_from_env;
 pub use peer_rest_client::{
-    PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
-    ScannerPeerActivity,
+    PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
+    SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity,
 };
 pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
 pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, S3PeerSys};
@@ -17,12 +17,15 @@ use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
 use crate::error::{Error, Result};
 use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
 use crate::runtime::sources as runtime_sources;
+use crate::set_disk::get_lock_acquire_timeout;
 use crate::storage_api_contracts::{
    admin::StorageAdminApi,
    heal::HealOperations,
+    namespace::NamespaceLocking,
    object::{DeletedObject, EcstoreObjectIO, ObjectIO, ObjectOperations, ObjectToDelete},
    range::HTTPRangeSpec,
 };
+use crate::store::ECStore;
 use http::HeaderMap;
 use rustfs_common::heal_channel::{HealOpts, HealScanMode};
 use rustfs_config::audit::{
@@ -45,10 +48,94 @@ use serde_json::{Map, Value};
 use std::collections::{HashMap, HashSet};
 use std::sync::LazyLock;
 use std::sync::{Arc, RwLock};
+use tokio::sync::RwLock as AsyncRwLock;
 use tracing::{debug, error, info, instrument, warn};

 pub const CONFIG_PREFIX: &str = "config";
-const CONFIG_FILE: &str = "config.json";
+const SERVER_CONFIG_OBJECT: &str = "config/config.json";
+
+// Server-config lock order: SERVER_CONFIG_LOCK -> distributed namespace lock
+// for SERVER_CONFIG_OBJECT. Readers and writers must never reverse this order.
+static SERVER_CONFIG_LOCK: LazyLock<AsyncRwLock<()>> = LazyLock::new(|| AsyncRwLock::new(()));
+
+fn config_task_join_error(operation: &'static str, error: tokio::task::JoinError) -> Error {
+    let outcome = if error.is_cancelled() { "cancelled" } else { "panicked" };
+    Error::other(format!("{operation} task {outcome}"))
+}
+
+/// Runs one complete server-config transaction while holding both the local
+/// process guard and the distributed namespace write lock for `config.json`.
+/// The operation must use the corresponding no-lock read/save functions.
+pub async fn with_server_config_write_lock<F, Fut, T>(store: Arc<ECStore>, operation: F) -> Result<T>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    tokio::spawn(async move {
+        // Lock order: SERVER_CONFIG_LOCK -> namespace write lock.
+        let _local_guard = SERVER_CONFIG_LOCK.write().await;
+        let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?;
+        let _write_guard = namespace_lock.get_write_lock(get_lock_acquire_timeout()).await?;
+        Ok(operation().await)
+    })
+    .await
+    .map_err(|error| config_task_join_error("server config write", error))?
+}
+
+/// Reads and synchronously publishes one server-config snapshot while holding
+/// a shared namespace lock. Concurrent peer reloads may proceed together, but
+/// no writer can interleave between the snapshot read and generation accept.
+pub async fn with_server_config_read_lock<F, Fut, T>(store: Arc<ECStore>, operation: F) -> Result<T>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    tokio::spawn(async move {
+        // Lock order: SERVER_CONFIG_LOCK -> namespace read lock.
+        let _local_guard = SERVER_CONFIG_LOCK.read().await;
+        let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?;
+        let _read_guard = namespace_lock.get_read_lock(get_lock_acquire_timeout()).await?;
+        Ok(operation().await)
+    })
+    .await
+    .map_err(|error| config_task_join_error("server config read", error))?
+}
+
+/// Runs a cancellation-safe transaction under the distributed write lock for
+/// one metadata config object. The operation must use no-lock object I/O.
+pub async fn with_config_object_write_lock<F, Fut, T>(store: Arc<ECStore>, object: String, operation: F) -> Result<T>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    tokio::spawn(async move {
+        let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
+        let _write_guard = namespace_lock.get_write_lock(get_lock_acquire_timeout()).await?;
+        Ok(operation().await)
+    })
+    .await
+    .map_err(|error| config_task_join_error("config object write", error))?
+}
+
+/// Runs one read-and-publish transaction under the distributed read lock for
+/// a metadata config object. The operation must use no-lock object I/O.
+pub async fn with_config_object_read_lock<F, Fut, T>(store: Arc<ECStore>, object: String, operation: F) -> Result<T>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    tokio::spawn(async move {
+        let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
+        let _read_guard = namespace_lock.get_read_lock(get_lock_acquire_timeout()).await?;
+        Ok(operation().await)
+    })
+    .await
+    .map_err(|error| config_task_join_error("config object read", error))?
+}

 /// Environment variable gating the startup fallback to the default server
 /// config when the persisted `config.json` object is corrupt beyond repair
@@ -393,6 +480,31 @@ where
    .await
 }

+pub async fn save_config_no_lock<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
+where
+    S: ObjectIO<
+            Error = Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        >,
+{
+    save_config_with_opts(
+        api,
+        file,
+        data,
+        &ObjectOptions {
+            max_parity: true,
+            no_lock: true,
+            ..Default::default()
+        },
+    )
+    .await
+}
+
 #[instrument(skip(api))]
 pub async fn delete_config<S>(api: Arc<S>, file: &str) -> Result<()>
 where
@@ -462,8 +574,17 @@ where
    Ok(cfg)
 }

+async fn new_and_save_server_config_no_lock<S>(api: Arc<S>) -> Result<Config>
+where
+    S: EcstoreObjectIO + StorageAdminApi,
+{
+    let cfg = new_server_config();
+    save_server_config_no_lock(api, &cfg).await?;
+    Ok(cfg)
+}
+
 fn get_config_file() -> String {
-    format!("{CONFIG_PREFIX}{SLASH_SEPARATOR}{CONFIG_FILE}")
+    SERVER_CONFIG_OBJECT.to_string()
 }

 fn storage_class_kvs_mut(cfg: &mut Config) -> &mut KVS {
@@ -1267,13 +1388,17 @@ where
 }

 /// Handle the situation where the configuration file does not exist, create and save a new configuration
-async fn handle_missing_config<S>(api: Arc<S>, context: &str) -> Result<Config>
+async fn handle_missing_config<S>(api: Arc<S>, context: &str, namespace_lock_held: bool) -> Result<Config>
 where
    S: EcstoreObjectIO + StorageAdminApi,
 {
    warn!("Configuration not found ({}): Start initializing new configuration", context);
    let cfg = if runtime_sources::first_cluster_node_is_local().await {
-        new_and_save_server_config(api.clone()).await?
+        if namespace_lock_held {
+            new_and_save_server_config_no_lock(api.clone()).await?
+        } else {
+            new_and_save_server_config(api.clone()).await?
+        }
    } else {
        new_server_config()
    };
@@ -1288,6 +1413,46 @@ fn handle_config_read_error(err: Error, file_path: &str) -> Result<Config> {
 }

 pub async fn read_config_without_migrate<S>(api: Arc<S>) -> Result<Config>
+where
+    S: ObjectIO<
+            Error = Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        > + StorageAdminApi,
+{
+    read_config_without_migrate_inner(api, false).await
+}
+
+/// Reads the server config while an upper layer holds the namespace write lock
+/// for [`SERVER_CONFIG_OBJECT`]. Missing-config initialization uses the matching
+/// no-lock save path and therefore cannot recursively acquire the same lock.
+pub async fn read_config_without_migrate_no_lock<S>(api: Arc<S>) -> Result<Config>
+where
+    S: ObjectIO<
+            Error = Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        > + StorageAdminApi,
+{
+    read_config_without_migrate_inner(api, true).await
+}
+
+/// Reads an already-initialized server config while a caller owns a namespace
+/// read lock. This never initializes or migrates a missing object.
+pub async fn read_existing_server_config_no_lock(api: Arc<ECStore>) -> Result<Config> {
+    let data = read_config_no_lock(api, SERVER_CONFIG_OBJECT).await?;
+    Ok(decode_persisted_server_config(&data)?.merge())
+}
+
+async fn read_config_without_migrate_inner<S>(api: Arc<S>, namespace_lock_held: bool) -> Result<Config>
 where
    S: ObjectIO<
            Error = Error,
@@ -1303,13 +1468,13 @@ where

    // Try to read the configuration file
    match read_config_no_lock(api.clone(), &config_file).await {
-        Ok(data) => read_server_config(api, &data).await,
-        Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration").await,
+        Ok(data) => read_server_config(api, &data, namespace_lock_held).await,
+        Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration", namespace_lock_held).await,
        Err(err) => handle_config_read_error(err, &config_file),
    }
 }

-async fn read_server_config<S>(api: Arc<S>, data: &[u8]) -> Result<Config>
+async fn read_server_config<S>(api: Arc<S>, data: &[u8], namespace_lock_held: bool) -> Result<Config>
 where
    S: EcstoreObjectIO + StorageAdminApi,
 {
@@ -1324,7 +1489,9 @@ where
                let cfg = decode_persisted_server_config(&cfg_data)?;
                return Ok(cfg.merge());
            }
-            Err(Error::ConfigNotFound) => return handle_missing_config(api, "Read alternate configuration").await,
+            Err(Error::ConfigNotFound) => {
+                return handle_missing_config(api, "Read alternate configuration", namespace_lock_held).await;
+            }
            Err(err) => return handle_config_read_error(err, &config_file),
        }
    }
@@ -1534,6 +1701,38 @@ fn fallback_server_config_after_corruption(err: Error, config_file: &str, recove
 }

 pub async fn save_server_config<S>(api: Arc<S>, cfg: &Config) -> Result<()>
+where
+    S: ObjectIO<
+            Error = Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        >,
+{
+    save_server_config_inner(api, cfg, false).await
+}
+
+/// Saves the server config while an upper layer holds the namespace write
+/// lock for [`SERVER_CONFIG_OBJECT`].
+pub async fn save_server_config_no_lock<S>(api: Arc<S>, cfg: &Config) -> Result<()>
+where
+    S: ObjectIO<
+            Error = Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        >,
+{
+    save_server_config_inner(api, cfg, true).await
+}
+
+async fn save_server_config_inner<S>(api: Arc<S>, cfg: &Config, no_lock: bool) -> Result<()>
 where
    S: ObjectIO<
            Error = Error,
@@ -1546,7 +1745,11 @@ where
        >,
 {
    let config_file = get_config_file();
-    let existing = match read_config(api.clone(), &config_file).await {
+    let existing = match if no_lock {
+        read_config_no_lock(api.clone(), &config_file).await
+    } else {
+        read_config(api.clone(), &config_file).await
+    } {
        Ok(v) => Some(v),
        Err(Error::ConfigNotFound) => None,
        Err(err) => {
@@ -1570,7 +1773,21 @@ where
        return Ok(());
    }

-    save_config(api, &config_file, data).await
+    if no_lock {
+        save_config_with_opts(
+            api,
+            &config_file,
+            data,
+            &ObjectOptions {
+                max_parity: true,
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+    } else {
+        save_config(api, &config_file, data).await
+    }
 }

 pub async fn lookup_configs<S>(cfg: &mut Config, api: Arc<S>) -> Result<()>
@@ -1630,9 +1847,9 @@ where
 #[cfg(test)]
 mod tests {
    use super::{
-        apply_dynamic_config_for_sub_sys_with, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob,
-        is_standard_object_server_config, lookup_configs, read_config, read_config_preserve_empty, read_config_with_metadata,
-        storage_class_kvs_mut,
+        apply_dynamic_config_for_sub_sys_with, config_task_join_error, configs_semantically_equal, decode_server_config_blob,
+        encode_server_config_blob, is_standard_object_server_config, lookup_configs, read_config, read_config_preserve_empty,
+        read_config_with_metadata, storage_class_kvs_mut,
    };
    use crate::config::{audit, notify, oidc};
    use crate::disk::endpoint::Endpoint;
@@ -1652,6 +1869,17 @@ mod tests {
    use rustfs_config::{
        DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MYSQL_DSN_STRING, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE,
    };
+
+    #[tokio::test]
+    async fn config_task_join_error_does_not_expose_panic_payload() {
+        let join_error = tokio::spawn(async { panic!("do-not-expose-payload") })
+            .await
+            .expect_err("test task should panic");
+
+        let rendered = config_task_join_error("server config write", join_error).to_string();
+        assert!(rendered.contains("panicked"));
+        assert!(!rendered.contains("do-not-expose-payload"));
+    }
    use rustfs_lock::client::LockClient;
    use rustfs_lock::client::local::LocalClient;
    use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats};
@@ -1861,6 +2089,7 @@ mod tests {
                }),
                object_info: self.object_info(bucket, object),
                buffered_body: None,
+                resolved_sse: None,
                body_source: Default::default(),
            })
        }
@@ -3087,6 +3316,7 @@ mod tests {
                stream: Box::new(Cursor::new(data)),
                object_info,
                buffered_body: None,
+                resolved_sse: None,
                body_source: Default::default(),
            })
        }
@@ -1240,6 +1240,7 @@ mod tests {
            stream: Box::new(Cursor::new(raw_payload.clone())),
            object_info: object_info.clone(),
            buffered_body: None,
+            resolved_sse: None,
            body_source: Default::default(),
        };

@@ -406,6 +406,9 @@ impl WritePlan {
                    false,
                )?,
                WriteEncryptionMode::Singlepart { base_nonce } => HashReader::from_reader(
+                    #[cfg(feature = "rio-v2")]
+                    rustfs_rio::EncryptReader::new(reader, encryption.key_bytes, base_nonce),
+                    #[cfg(not(feature = "rio-v2"))]
                    EncryptReader::new(reader, encryption.key_bytes, base_nonce),
                    HashReader::SIZE_PRESERVE_LAYER,
                    actual_size,
@@ -417,6 +420,9 @@ impl WritePlan {
                    base_nonce,
                    multipart_part_number,
                } => HashReader::from_reader(
+                    #[cfg(feature = "rio-v2")]
+                    rustfs_rio::EncryptReader::new_multipart(reader, encryption.key_bytes, base_nonce, multipart_part_number),
+                    #[cfg(not(feature = "rio-v2"))]
                    EncryptReader::new_multipart(reader, encryption.key_bytes, base_nonce, multipart_part_number),
                    HashReader::SIZE_PRESERVE_LAYER,
                    actual_size,
@@ -509,6 +515,10 @@ mod tests {
            .await
            .expect("read transformed ciphertext");

+        #[cfg(feature = "rio-v2")]
+        let decrypt_reader =
+            rustfs_rio::DecryptReader::new_multipart(Cursor::new(ciphertext), key_bytes, base_nonce, vec![part_number]);
+        #[cfg(not(feature = "rio-v2"))]
        let decrypt_reader = DecryptReader::new_multipart(Cursor::new(ciphertext), key_bytes, base_nonce, vec![part_number]);
        let mut decompressed = DecompressReader::new(Box::new(decrypt_reader), CompressionAlgorithm::default());

@@ -705,7 +715,7 @@ mod tests {
            .expect("read transformed ciphertext");

        let mut decrypted_compressed = Vec::new();
-        DecryptReader::new(Cursor::new(ciphertext), key_bytes, base_nonce)
+        rustfs_rio::DecryptReader::new(Cursor::new(ciphertext), key_bytes, base_nonce)
            .read_to_end(&mut decrypted_compressed)
            .await
            .expect("decrypt compressed stream");
@@ -49,6 +49,7 @@ mod object_api;
 mod runtime;
 mod services;
 mod set_disk;
+mod sse;
 mod storage_api_contracts;
 mod store;

@@ -14,6 +14,19 @@

 use super::*;
 #[cfg(feature = "rio-v2")]
+use crate::sse::PersistedManagedEncryption;
+#[cfg(feature = "rio-v2")]
+use crate::sse::{
+    AMZ_SERVER_SIDE_ENCRYPTION_KMS_KEY_ID, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
+    MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
+    MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
+    MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM, decrypt_minio_static_kms_dek,
+};
+use crate::sse::{
+    DEFAULT_SSE_ALGORITHM, INTERNAL_ENCRYPTION_IV_HEADER, INTERNAL_ENCRYPTION_KEY_HEADER, INTERNAL_ENCRYPTION_KEY_ID_HEADER,
+    MINIO_INTERNAL_ENCRYPTION_IV_HEADER, ManagedDekProvider, classify_persisted_managed_encryption,
+};
+#[cfg(feature = "rio-v2")]
 use aes_gcm::aead::Payload;
 use aes_gcm::{
    Aes256Gcm, Key, Nonce,
@@ -26,46 +39,28 @@ use chacha20poly1305::ChaCha20Poly1305;
 use hmac::{Hmac, Mac};
 use md5::{Digest, Md5};
 use rustfs_kms::types::ObjectEncryptionContext;
+#[cfg(feature = "rio-v2")]
+use rustfs_kms::{MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, decode_managed_kms_context};
 use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
 use rustfs_utils::path::path_join_buf;
 #[cfg(feature = "rio-v2")]
-use serde::Deserialize;
-#[cfg(feature = "rio-v2")]
 use sha2::Sha256;
 use std::collections::HashMap;
 use std::env;
+#[cfg(feature = "rio-v2")]
+use zeroize::Zeroizing;

 use crate::io_support::rio::Index;

-const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id";
-const INTERNAL_ENCRYPTION_KEY_HEADER: &str = "x-rustfs-encryption-key";
-const INTERNAL_ENCRYPTION_CONTEXT_HEADER: &str = "x-rustfs-encryption-context";
-const INTERNAL_ENCRYPTION_IV_HEADER: &str = "x-rustfs-encryption-iv";
 const INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER: &str = "x-rustfs-encryption-original-size";
 const SSEC_ORIGINAL_SIZE_HEADER: &str = "x-amz-server-side-encryption-customer-original-size";
-const DEFAULT_SSE_ALGORITHM: &str = "AES256";
 #[cfg(feature = "rio-v2")]
 const DARE_PAYLOAD_SIZE: i64 = 64 * 1024;
 #[cfg(feature = "rio-v2")]
 const DARE_PACKAGE_SIZE: i64 = DARE_PAYLOAD_SIZE + 32;
-const MINIO_INTERNAL_ENCRYPTION_IV_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Iv";
-#[cfg(feature = "rio-v2")]
-const MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm";
-#[cfg(feature = "rio-v2")]
-const MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key";
-#[cfg(feature = "rio-v2")]
-const MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key";
-#[cfg(feature = "rio-v2")]
-const MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id";
-#[cfg(feature = "rio-v2")]
-const MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key";
-#[cfg(feature = "rio-v2")]
-const MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Context";
 #[cfg(feature = "rio-v2")]
 const MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key";
 #[cfg(feature = "rio-v2")]
-const MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM: &str = "DAREv2-HMAC-SHA256";
-#[cfg(feature = "rio-v2")]
 const DARE_VERSION_20: u8 = 0x20;
 #[cfg(feature = "rio-v2")]
 const DARE_CIPHER_AES_256_GCM: u8 = 0x00;
@@ -79,13 +74,6 @@ const DARE_TAG_SIZE: usize = 16;
 const SEALED_KEY_IV_SIZE: usize = 32;
 #[cfg(feature = "rio-v2")]
 const SEALED_KEY_SIZE: usize = DARE_HEADER_SIZE + 32 + DARE_TAG_SIZE;
-#[cfg(feature = "rio-v2")]
-const MINIO_SECRET_KEY_RANDOM_SIZE: usize = 28;
-#[cfg(feature = "rio-v2")]
-const MINIO_SECRET_KEY_IV_SIZE: usize = 16;
-#[cfg(feature = "rio-v2")]
-const MINIO_SECRET_KEY_NONCE_SIZE: usize = 12;
-
 #[cfg(feature = "rio-v2")]
 type HmacSha256 = Hmac<Sha256>;

@@ -110,14 +98,6 @@ fn build_object_encryption_context(
    object_context
 }

-#[cfg(feature = "rio-v2")]
-fn is_legacy_rustfs_managed_metadata(metadata: &HashMap<String, String>) -> bool {
-    metadata_get(metadata, INTERNAL_ENCRYPTION_KEY_HEADER).is_some()
-        && metadata_get(metadata, INTERNAL_ENCRYPTION_IV_HEADER).is_some()
-        && metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER).is_none()
-        && metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER).is_none()
-}
-
 fn part_plaintext_size(part: &ObjectPartInfo) -> i64 {
    if part.actual_size > 0 {
        part.actual_size
@@ -301,11 +281,19 @@ pub struct GetObjectReader {
    pub stream: Box<dyn AsyncRead + Unpin + Send + Sync>,
    pub object_info: ObjectInfo,
    pub buffered_body: Option<Bytes>,
+    pub resolved_sse: Option<GetObjectSse>,
    /// Cache-hook provenance; defaults to [`GetObjectBodySource::Unprobed`] for
    /// every reader that never passed through the app-layer cache probe.
    pub body_source: GetObjectBodySource,
 }

+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum GetObjectSse {
+    SseC { customer_key_md5: String },
+    SseS3,
+    SseKms { key_id: String },
+}
+
 impl GetObjectReader {
    /// Builds a fully materialized reader from a cache-coordinated body.
    pub fn from_cache_body(mut object_info: ObjectInfo, body: Bytes) -> Result<Self> {
@@ -314,6 +302,7 @@ impl GetObjectReader {
            stream: Box::new(std::io::Cursor::new(body.clone())),
            object_info,
            buffered_body: Some(body),
+            resolved_sse: None,
            body_source: GetObjectBodySource::HookServed,
        })
    }
@@ -331,12 +320,13 @@ impl GetObjectReader {
    }
 }

-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone)]
 struct EncryptionMaterial {
    key_bytes: [u8; 32],
    base_nonce: [u8; 12],
    key_kind: EncryptionKeyKind,
    reader_backend: crate::io_support::rio::ReadEncryptionBackend,
+    resolved_sse: Option<GetObjectSse>,
 }

 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -582,6 +572,7 @@ impl ReadPlan {
                    stream: reader,
                    object_info: oi.clone(),
                    buffered_body: None,
+                    resolved_sse: None,
                    body_source: GetObjectBodySource::Unprobed,
                },
                self.storage_offset,
@@ -637,6 +628,7 @@ impl ReadPlan {
                        stream: final_reader,
                        object_info,
                        buffered_body: None,
+                        resolved_sse: None,
                        body_source: GetObjectBodySource::Unprobed,
                    },
                    self.storage_offset,
@@ -733,12 +725,14 @@ impl ReadPlan {

                let mut object_info = oi.clone();
                object_info.size = self.object_size;
+                let resolved_sse = material.resolved_sse;

                Ok((
                    GetObjectReader {
                        stream: final_reader,
                        object_info,
                        buffered_body: None,
+                        resolved_sse,
                        body_source: GetObjectBodySource::Unprobed,
                    },
                    self.storage_offset,
@@ -1128,19 +1122,19 @@ fn is_supported_sealed_object_key_cipher(cipher: u8) -> bool {
 }

 #[cfg(feature = "rio-v2")]
-fn decrypt_sealed_object_key_payload(sealing_key: [u8; 32], header: &[u8], sealed_key: &[u8]) -> Result<Vec<u8>> {
+fn decrypt_sealed_object_key_payload(sealing_key: &[u8; 32], header: &[u8], sealed_key: &[u8]) -> Result<Vec<u8>> {
    let nonce = &header[4..16];
    let ciphertext = &sealed_key[DARE_HEADER_SIZE..];
    let aad = &header[..4];
    match header[1] {
        DARE_CIPHER_AES_256_GCM => {
-            let cipher = Aes256Gcm::new_from_slice(&sealing_key)
+            let cipher = Aes256Gcm::new_from_slice(sealing_key)
                .map_err(|err| Error::other(format!("invalid AES-GCM sealing key: {err}")))?;
            let nonce = Nonce::try_from(nonce).map_err(|_| Error::other("invalid sealed object-key package nonce"))?;
            cipher.decrypt(&nonce, Payload { msg: ciphertext, aad })
        }
        DARE_CIPHER_CHACHA20_POLY1305 => {
-            let cipher = ChaCha20Poly1305::new_from_slice(&sealing_key)
+            let cipher = ChaCha20Poly1305::new_from_slice(sealing_key)
                .map_err(|err| Error::other(format!("invalid ChaCha20-Poly1305 sealing key: {err}")))?;
            let nonce =
                chacha20poly1305::Nonce::try_from(nonce).map_err(|_| Error::other("invalid sealed object-key package nonce"))?;
@@ -1287,8 +1281,8 @@ fn try_unseal_minio_object_key(
        return Err(Error::other("invalid sealed object-key payload header"));
    }

-    let sealing_key = derive_sealing_key(external_key, iv, managed_sse_domain(metadata), bucket, object);
-    let plaintext = decrypt_sealed_object_key_payload(sealing_key, header, &sealed_key)?;
+    let sealing_key = Zeroizing::new(derive_sealing_key(external_key, iv, managed_sse_domain(metadata), bucket, object));
+    let plaintext = Zeroizing::new(decrypt_sealed_object_key_payload(&sealing_key, header, &sealed_key)?);
    let object_key: [u8; 32] = plaintext
        .as_slice()
        .try_into()
@@ -1336,12 +1330,19 @@ fn resolve_ssec_material(oi: &ObjectInfo, headers: &HeaderMap<HeaderValue>) -> R
    }

    #[cfg(feature = "rio-v2")]
-    if let Some(object_key) = try_unseal_minio_object_key(&oi.user_defined, &oi.bucket, &oi.name, key_bytes)? {
+    if metadata_get(&oi.user_defined, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER).is_some()
+        || metadata_get(&oi.user_defined, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER).is_some()
+    {
+        let object_key = try_unseal_minio_object_key(&oi.user_defined, &oi.bucket, &oi.name, key_bytes)?
+            .ok_or_else(|| Error::other("incomplete or invalid MinIO SSE-C sealed object-key metadata"))?;
        return Ok(EncryptionMaterial {
            key_bytes: object_key,
            base_nonce: [0u8; 12],
            key_kind: EncryptionKeyKind::Object,
            reader_backend: crate::io_support::rio::ReadEncryptionBackend::V2,
+            resolved_sse: Some(GetObjectSse::SseC {
+                customer_key_md5: expected_md5,
+            }),
        });
    }

@@ -1350,6 +1351,9 @@ fn resolve_ssec_material(oi: &ObjectInfo, headers: &HeaderMap<HeaderValue>) -> R
        base_nonce: read_stored_ssec_nonce(&oi.user_defined, &oi.bucket, &oi.name),
        key_kind: EncryptionKeyKind::Direct,
        reader_backend: crate::io_support::rio::ReadEncryptionBackend::Legacy,
+        resolved_sse: Some(GetObjectSse::SseC {
+            customer_key_md5: expected_md5,
+        }),
    })
 }

@@ -1372,28 +1376,65 @@ fn read_stored_ssec_nonce(metadata: &HashMap<String, String>, bucket: &str, key:
 }

 async fn resolve_managed_material(bucket: &str, object: &str, metadata: &HashMap<String, String>) -> Result<EncryptionMaterial> {
-    let normalized_metadata = normalize_managed_metadata(metadata);
-    let encrypted_dek = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_KEY_HEADER)
-        .ok_or_else(|| Error::other("missing managed encrypted DEK"))?;
-    let encrypted_dek = BASE64_STANDARD
-        .decode(encrypted_dek)
-        .map_err(|e| Error::other(format!("failed to decode managed encrypted DEK: {e}")))?;
-
-    let kms_key_id = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_KEY_ID_HEADER).unwrap_or("default");
+    let encrypted_dek = metadata_get(metadata, INTERNAL_ENCRYPTION_KEY_HEADER);
    #[cfg(feature = "rio-v2")]
-    let kms_context = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_CONTEXT_HEADER)
-        .map(|value| {
-            serde_json::from_str::<HashMap<String, String>>(value)
-                .map_err(|e| Error::other(format!("failed to parse managed KMS context: {e}")))
-        })
-        .transpose()?;
+    let encrypted_dek = encrypted_dek.or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER));
+    let encrypted_dek = match encrypted_dek {
+        Some(encrypted_dek) => BASE64_STANDARD
+            .decode(encrypted_dek)
+            .map_err(|e| Error::other(format!("failed to decode managed encrypted DEK: {e}")))?,
+        None => Vec::new(),
+    };
+    let persisted_format =
+        classify_persisted_managed_encryption(metadata, &encrypted_dek).map_err(|err| Error::other(err.to_string()))?;
+
+    let kms_key_id = metadata_get(metadata, INTERNAL_ENCRYPTION_KEY_ID_HEADER).filter(|value| !value.is_empty());
+    #[cfg(feature = "rio-v2")]
+    let kms_key_id = kms_key_id
+        .or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER).filter(|value| !value.is_empty()))
+        .or_else(|| metadata_get(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_KEY_ID).filter(|value| !value.is_empty()));
+    let kms_key_id = kms_key_id.unwrap_or("default");
+    let resolved_sse = match persisted_format.scheme() {
+        crate::sse::ManagedSseScheme::SseS3 => GetObjectSse::SseS3,
+        crate::sse::ManagedSseScheme::SseKms => GetObjectSse::SseKms {
+            key_id: kms_key_id.to_string(),
+        },
+    };
+    #[cfg(feature = "rio-v2")]
+    let kms_context = decode_managed_kms_context(metadata).map_err(|err| Error::other(err.to_string()))?;
    #[cfg(not(feature = "rio-v2"))]
    let kms_context: Option<HashMap<String, String>> = None;
    let object_context = build_object_encryption_context(bucket, object, kms_context.as_ref());

-    let decrypted_key = if let Some(service) = crate::runtime::sources::object_encryption_service().await {
+    let provider = persisted_format.provider();
+    #[cfg(feature = "rio-v2")]
+    let minio_static_key = if matches!(provider, ManagedDekProvider::Kms) {
+        decrypt_minio_static_kms_dek(kms_key_id, &encrypted_dek, &object_context.encryption_context)
+            .map_err(|err| Error::other(err.to_string()))?
+    } else {
+        None
+    };
+    #[cfg(not(feature = "rio-v2"))]
+    let minio_static_key: Option<[u8; 32]> = None;
+    let service = match provider {
+        ManagedDekProvider::LocalSseS3 | ManagedDekProvider::MinioKeyValue => None,
+        ManagedDekProvider::Kms if minio_static_key.is_some() => None,
+        ManagedDekProvider::Kms => Some(
+            crate::runtime::sources::object_encryption_service()
+                .await
+                .ok_or_else(|| Error::other("KMS encryption service is required to decrypt this object"))?,
+        ),
+    };
+    let decrypted_key = if let Some(key) = minio_static_key {
+        key
+    } else if let Some(service) = service {
        #[cfg(feature = "rio-v2")]
-        let data_key = if is_legacy_rustfs_managed_metadata(&normalized_metadata) {
+        let data_key = if matches!(
+            persisted_format,
+            PersistedManagedEncryption::LegacySseS3Local
+                | PersistedManagedEncryption::LegacySseS3Kms
+                | PersistedManagedEncryption::LegacySseKms
+        ) {
            service.decrypt_legacy_data_key(&encrypted_dek).await
        } else {
            service.decrypt_data_key(&encrypted_dek, &object_context).await
@@ -1405,20 +1446,24 @@ async fn resolve_managed_material(bucket: &str, object: &str, metadata: &HashMap
            .map_err(|e| Error::other(format!("failed to decrypt managed data key: {e}")))?
            .plaintext_key
    } else {
-        decrypt_local_sse_dek(&encrypted_dek, kms_key_id, &object_context)?
+        decrypt_local_sse_dek(&encrypted_dek)?
    };

    #[cfg(feature = "rio-v2")]
-    if let Some(object_key) = try_unseal_minio_object_key(&normalized_metadata, bucket, object, decrypted_key)? {
+    if persisted_format.uses_object_key() {
+        let object_key = try_unseal_minio_object_key(metadata, bucket, object, decrypted_key)?
+            .ok_or_else(|| Error::other("MinIO managed SSE metadata is missing a valid sealed object key"))?;
        return Ok(EncryptionMaterial {
            key_bytes: object_key,
            base_nonce: [0u8; 12],
            key_kind: EncryptionKeyKind::Object,
            reader_backend: crate::io_support::rio::ReadEncryptionBackend::V2,
+            resolved_sse: Some(resolved_sse),
        });
    }

-    let iv_b64 = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_IV_HEADER)
+    let iv_b64 = metadata_get(metadata, INTERNAL_ENCRYPTION_IV_HEADER)
+        .or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_IV_HEADER))
        .ok_or_else(|| Error::other("missing managed encryption IV"))?;
    let iv = BASE64_STANDARD
        .decode(iv_b64)
@@ -1433,66 +1478,15 @@ async fn resolve_managed_material(bucket: &str, object: &str, metadata: &HashMap
        base_nonce,
        key_kind: EncryptionKeyKind::Direct,
        reader_backend: crate::io_support::rio::ReadEncryptionBackend::Legacy,
+        resolved_sse: Some(resolved_sse),
    })
 }

-fn normalize_managed_metadata(metadata: &HashMap<String, String>) -> HashMap<String, String> {
-    #[cfg(feature = "rio-v2")]
-    {
-        let mut normalized = metadata.clone();
-        if metadata_get(&normalized, INTERNAL_ENCRYPTION_KEY_HEADER).is_none()
-            && let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER)
-                .or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER))
-                .or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER))
-        {
-            normalized.insert(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), value.to_string());
-        }
-
-        if metadata_get(&normalized, INTERNAL_ENCRYPTION_IV_HEADER).is_none()
-            && let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_IV_HEADER)
-        {
-            normalized.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), value.to_string());
-        }
-
-        if metadata_get(&normalized, INTERNAL_ENCRYPTION_KEY_ID_HEADER).is_none()
-            && let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER)
-        {
-            normalized.insert(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), value.to_string());
-        }
-
-        if metadata_get(&normalized, INTERNAL_ENCRYPTION_CONTEXT_HEADER).is_none()
-            && let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)
-            && let Ok(decoded) = BASE64_STANDARD.decode(value)
-            && let Ok(context) = serde_json::from_slice::<HashMap<String, String>>(&decoded)
-            && let Ok(encoded) = serde_json::to_string(&context)
-        {
-            normalized.insert(INTERNAL_ENCRYPTION_CONTEXT_HEADER.to_string(), encoded);
-        }
-
-        normalized
-    }
-
-    #[cfg(not(feature = "rio-v2"))]
-    {
-        metadata.clone()
-    }
-}
-
-fn decrypt_local_sse_dek(encrypted_dek: &[u8], _kms_key_id: &str, object_context: &ObjectEncryptionContext) -> Result<[u8; 32]> {
-    if let Ok(plaintext) = decrypt_rustfs_local_sse_dek(encrypted_dek) {
-        return Ok(plaintext);
-    }
-
-    #[cfg(feature = "rio-v2")]
-    {
-        decrypt_minio_secret_key_dek(encrypted_dek, object_context)
-    }
-
-    #[cfg(not(feature = "rio-v2"))]
-    {
-        let _ = object_context;
-        Err(Error::other("invalid managed DEK format"))
+fn decrypt_local_sse_dek(encrypted_dek: &[u8]) -> Result<[u8; 32]> {
+    if encrypted_dek.is_empty() {
+        return local_sse_master_key();
    }
+    decrypt_rustfs_local_sse_dek(encrypted_dek)
 }

 fn decrypt_rustfs_local_sse_dek(encrypted_dek: &[u8]) -> Result<[u8; 32]> {
@@ -1526,113 +1520,22 @@ fn decrypt_rustfs_local_sse_dek(encrypted_dek: &[u8]) -> Result<[u8; 32]> {
        .map_err(|_| Error::other("managed DEK has invalid plaintext length"))
 }

-#[cfg(feature = "rio-v2")]
-#[derive(Deserialize)]
-struct MinioLegacyCiphertext {
-    #[serde(rename = "aead")]
-    algorithm: String,
-    iv: Vec<u8>,
-    nonce: Vec<u8>,
-    bytes: Vec<u8>,
-}
-
-#[cfg(feature = "rio-v2")]
-fn decrypt_minio_secret_key_dek(encrypted_dek: &[u8], object_context: &ObjectEncryptionContext) -> Result<[u8; 32]> {
-    let key = local_sse_master_key()?;
-    let (ciphertext, iv, nonce) = parse_minio_secret_key_ciphertext(encrypted_dek)?;
-    let associated_data = marshal_minio_kms_context(&object_context.encryption_context);
-
-    let mut mac = HmacSha256::new_from_slice(&key).map_err(|err| Error::other(format!("invalid local SSE master key: {err}")))?;
-    mac.update(&iv);
-    let sealing_key = mac.finalize().into_bytes();
-    let cipher = Aes256Gcm::new_from_slice(sealing_key.as_slice())
-        .map_err(|err| Error::other(format!("invalid MinIO sealing key: {err}")))?;
-    let nonce = Nonce::try_from(&nonce[..]).map_err(|_| Error::other("invalid MinIO managed DEK nonce"))?;
-    let plaintext = cipher
-        .decrypt(
-            &nonce,
-            aes_gcm::aead::Payload {
-                msg: &ciphertext,
-                aad: &associated_data,
-            },
-        )
-        .map_err(|err| Error::other(format!("failed to decrypt MinIO managed DEK: {err}")))?;
-
-    plaintext
-        .as_slice()
-        .try_into()
-        .map_err(|_| Error::other("MinIO managed DEK has invalid plaintext length"))
-}
-
-#[cfg(feature = "rio-v2")]
-fn parse_minio_secret_key_ciphertext(
-    encrypted_dek: &[u8],
-) -> Result<(Vec<u8>, [u8; MINIO_SECRET_KEY_IV_SIZE], [u8; MINIO_SECRET_KEY_NONCE_SIZE])> {
-    if encrypted_dek.first() == Some(&b'{') && encrypted_dek.last() == Some(&b'}') {
-        let legacy: MinioLegacyCiphertext = serde_json::from_slice(encrypted_dek)
-            .map_err(|err| Error::other(format!("failed to parse MinIO legacy managed DEK: {err}")))?;
-        if legacy.algorithm != "AES-256-GCM-HMAC-SHA-256" {
-            return Err(Error::other(format!(
-                "unsupported MinIO legacy managed DEK algorithm {}",
-                legacy.algorithm
-            )));
-        }
-        let iv = legacy
-            .iv
-            .as_slice()
-            .try_into()
-            .map_err(|_| Error::other("invalid MinIO legacy managed DEK IV length"))?;
-        let nonce = legacy
-            .nonce
-            .as_slice()
-            .try_into()
-            .map_err(|_| Error::other("invalid MinIO legacy managed DEK nonce length"))?;
-        return Ok((legacy.bytes, iv, nonce));
-    }
-
-    if encrypted_dek.len() <= MINIO_SECRET_KEY_RANDOM_SIZE {
-        return Err(Error::other("invalid MinIO managed DEK length"));
-    }
-
-    let split_at = encrypted_dek.len() - MINIO_SECRET_KEY_RANDOM_SIZE;
-    let (ciphertext, random) = encrypted_dek.split_at(split_at);
-    let iv = random[..MINIO_SECRET_KEY_IV_SIZE]
-        .try_into()
-        .map_err(|_| Error::other("invalid MinIO managed DEK IV length"))?;
-    let nonce = random[MINIO_SECRET_KEY_IV_SIZE..]
-        .try_into()
-        .map_err(|_| Error::other("invalid MinIO managed DEK nonce length"))?;
-    Ok((ciphertext.to_vec(), iv, nonce))
-}
-
-#[cfg(feature = "rio-v2")]
-fn marshal_minio_kms_context(context: &HashMap<String, String>) -> Vec<u8> {
-    let mut entries: Vec<_> = context.iter().collect();
-    entries.sort_by_key(|(left, _)| *left);
-
-    let mut json = String::from("{");
-    for (index, (key, value)) in entries.into_iter().enumerate() {
-        if index > 0 {
-            json.push(',');
-        }
-        json.push_str(&serde_json::to_string(key).expect("string key serializes"));
-        json.push(':');
-        json.push_str(&serde_json::to_string(value).expect("string value serializes"));
-    }
-    json.push('}');
-    json.into_bytes()
-}
-
 fn local_sse_master_key() -> Result<[u8; 32]> {
+    #[cfg(test)]
    if let Some(key) = decode_master_key_env("__RUSTFS_SSE_SIMPLE_CMK")? {
        return Ok(key);
    }

    if let Some(key) = decode_master_key_env("RUSTFS_SSE_S3_MASTER_KEY")? {
+        if key == [0u8; 32] {
+            return Err(Error::other("RUSTFS_SSE_S3_MASTER_KEY must not be an all-zero key"));
+        }
        return Ok(key);
    }

-    Ok([0u8; 32])
+    Err(Error::other(
+        "SSE-S3 requires RUSTFS_SSE_S3_MASTER_KEY to decrypt locally managed objects",
+    ))
 }

 fn decode_master_key_env(name: &str) -> Result<Option<[u8; 32]>> {
@@ -1707,6 +1610,7 @@ mod tests {

        assert_eq!(reader.body_source, GetObjectBodySource::HookServed);
        assert_eq!(reader.buffered_body.as_ref(), Some(&body));
+        assert_eq!(reader.resolved_sse, None);
        assert_eq!(reader.object_info.size, 11);
        assert_eq!(reader.object_info.actual_size, 11);
        assert!(reader.object_info.is_compressed());
@@ -1767,20 +1671,38 @@ mod tests {

    #[cfg(feature = "rio-v2")]
    #[test]
-    fn test_legacy_managed_metadata_excludes_sealed_keys() {
-        let legacy_metadata = HashMap::from([
-            (INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), "encrypted-dek".to_string()),
-            (INTERNAL_ENCRYPTION_IV_HEADER.to_string(), "nonce".to_string()),
-        ]);
-        assert!(is_legacy_rustfs_managed_metadata(&legacy_metadata));
+    fn resolve_ssec_material_rejects_partial_object_key_metadata() {
+        let customer_key = [0x42u8; 32];
+        let headers = ssec_headers_from_key(customer_key);
+        for partial in [
+            HashMap::from([(
+                MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER.to_string(),
+                BASE64_STANDARD.encode([0x11u8; SEALED_KEY_SIZE]),
+            )]),
+            HashMap::from([(
+                MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
+                MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
+            )]),
+        ] {
+            let mut metadata = HashMap::from([
+                (SSEC_ALGORITHM_HEADER.to_string(), DEFAULT_SSE_ALGORITHM.to_string()),
+                (SSEC_KEY_MD5_HEADER.to_string(), BASE64_STANDARD.encode(md5_bytes(customer_key))),
+            ]);
+            metadata.extend(partial);
+            let object_info = ObjectInfo {
+                bucket: "bucket".to_string(),
+                name: "object".to_string(),
+                user_defined: Arc::new(metadata),
+                ..Default::default()
+            };

-        let sealed_metadata = HashMap::from([
-            (INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), "encrypted-dek".to_string()),
-            (INTERNAL_ENCRYPTION_IV_HEADER.to_string(), "nonce".to_string()),
-            (MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), "sealed-key".to_string()),
-        ]);
-
-        assert!(!is_legacy_rustfs_managed_metadata(&sealed_metadata));
+            assert!(
+                resolve_ssec_material(&object_info, &headers)
+                    .expect_err("partial MinIO SSE-C metadata must fail closed")
+                    .to_string()
+                    .contains("incomplete or invalid")
+            );
+        }
    }

    #[cfg(feature = "rio-v2")]
@@ -2125,9 +2047,21 @@ mod tests {
        data_key: [u8; 32],
        object_key: [u8; 32],
        cipher_id: u8,
+    ) -> ([u8; 32], Vec<u8>) {
+        seal_managed_object_key_for_test(bucket, object, data_key, object_key, "SSE-S3", cipher_id)
+    }
+
+    #[cfg(feature = "rio-v2")]
+    fn seal_managed_object_key_for_test(
+        bucket: &str,
+        object: &str,
+        data_key: [u8; 32],
+        object_key: [u8; 32],
+        domain: &str,
+        cipher_id: u8,
    ) -> ([u8; 32], Vec<u8>) {
        let iv = [0x24u8; SEALED_KEY_IV_SIZE];
-        let sealing_key = derive_sealing_key(data_key, iv, "SSE-S3", bucket, object);
+        let sealing_key = derive_sealing_key(data_key, iv, domain, bucket, object);

        let mut header = [0u8; DARE_HEADER_SIZE];
        header[0] = DARE_VERSION_20;
@@ -2203,7 +2137,7 @@ mod tests {
    #[tokio::test]
    async fn resolve_managed_material_accepts_chacha20_poly1305_header_variant() {
        async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async {
-            let data_key = [0x24; 32];
+            let data_key = [0u8; 32];
            let object_key = [0x33; 32];
            let (iv, sealed_key) = seal_managed_s3_object_key_for_test_with_cipher(
                "bucket",
@@ -2213,7 +2147,6 @@ mod tests {
                DARE_CIPHER_CHACHA20_POLY1305,
            );

-            let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]);
            let metadata = HashMap::from([
                (
                    MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(),
@@ -2224,11 +2157,6 @@ mod tests {
                    MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
                    MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
                ),
-                (
-                    MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(),
-                    BASE64_STANDARD.encode(encrypted_dek.as_bytes()),
-                ),
-                (MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()),
            ]);

            let material = resolve_managed_material("bucket", "object", &metadata)
@@ -2458,11 +2386,52 @@ mod tests {
        ));
    }

+    #[tokio::test]
+    async fn test_zero_length_ssec_read_still_requires_customer_key() {
+        let object_info = ObjectInfo {
+            size: 0,
+            user_defined: Arc::new(HashMap::from([
+                (SSEC_ALGORITHM_HEADER.to_string(), DEFAULT_SSE_ALGORITHM.to_string()),
+                (SSEC_KEY_MD5_HEADER.to_string(), BASE64_STANDARD.encode([0x11; 16])),
+                (SSEC_ORIGINAL_SIZE_HEADER.to_string(), "0".to_string()),
+            ])),
+            ..Default::default()
+        };
+
+        let result = ReadPlan::build(None, &object_info, &ObjectOptions::default(), &HeaderMap::new()).await;
+
+        assert!(result.is_err(), "zero-length SSE-C reads must not bypass customer-key authorization");
+    }
+
+    #[tokio::test]
+    async fn test_zero_length_sse_kms_read_still_requires_kms() {
+        let object_info = ObjectInfo {
+            size: 0,
+            bucket: "bucket".to_string(),
+            name: "zero-kms".to_string(),
+            user_defined: Arc::new(HashMap::from([
+                ("x-amz-server-side-encryption".to_string(), "aws:kms".to_string()),
+                (
+                    INTERNAL_ENCRYPTION_KEY_HEADER.to_string(),
+                    BASE64_STANDARD.encode(b"opaque-kms-ciphertext"),
+                ),
+                (INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([0x12; 12])),
+                (INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER.to_string(), "0".to_string()),
+            ])),
+            ..Default::default()
+        };
+
+        let result = ReadPlan::build(None, &object_info, &ObjectOptions::default(), &HeaderMap::new()).await;
+
+        assert!(result.is_err(), "zero-length SSE-KMS reads must not bypass KMS authorization");
+    }
+
    #[tokio::test]
    async fn test_get_object_reader_allows_encrypted_full_object_passthrough() {
        async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async {
            let plaintext = b"managed-full-object".to_vec();
-            let data_key = [0x21; 32];
+            let data_key = [0u8; 32];
+            #[cfg(not(feature = "rio-v2"))]
            let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]);
            let bucket = "bucket";
            let object = "managed-full-object";
@@ -2478,7 +2447,6 @@ mod tests {
                    .expect("encrypt managed object");
                HashMap::from([
                    ("x-amz-server-side-encryption".to_string(), "AES256".to_string()),
-                    ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())),
                    ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()),
                    (
                        MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
@@ -2539,7 +2507,8 @@ mod tests {
    async fn test_get_object_reader_decrypts_managed_sse_range_on_plaintext_semantics() {
        async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async {
            let plaintext = b"0123456789abcdefghijklmnopqrstuvwxyz".to_vec();
-            let data_key = [0x23; 32];
+            let data_key = [0u8; 32];
+            #[cfg(not(feature = "rio-v2"))]
            let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]);
            let bucket = "bucket";
            let object = "managed-range-object";
@@ -2555,7 +2524,6 @@ mod tests {
                    .expect("encrypt managed ranged object");
                HashMap::from([
                    ("x-amz-server-side-encryption".to_string(), "AES256".to_string()),
-                    ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())),
                    ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()),
                    (
                        MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
@@ -2622,12 +2590,13 @@ mod tests {
        async_with_vars(
            [
                ("__RUSTFS_SSE_SIMPLE_CMK", None::<String>),
-                ("RUSTFS_SSE_S3_MASTER_KEY", Some(BASE64_STANDARD.encode([0u8; 32]))),
+                ("RUSTFS_SSE_S3_MASTER_KEY", Some(BASE64_STANDARD.encode([0x33u8; 32]))),
            ],
            async {
                let plaintext = b"managed-local-fallback".to_vec();
-                let data_key = [0x22; 32];
-                let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]);
+                let data_key = [0x33; 32];
+                #[cfg(not(feature = "rio-v2"))]
+                let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0x33; 32]);
                let bucket = "bucket";
                let object = "managed-local-fallback";

@@ -2642,7 +2611,6 @@ mod tests {
                        .expect("encrypt managed object with local fallback key");
                    HashMap::from([
                        ("x-amz-server-side-encryption".to_string(), "AES256".to_string()),
-                        ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())),
                        ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()),
                        (
                            MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
@@ -2703,8 +2671,7 @@ mod tests {
    async fn test_get_object_reader_accepts_minio_only_managed_metadata() {
        async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async {
            let plaintext = b"managed-minio-metadata".to_vec();
-            let data_key = [0x23; 32];
-            let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]);
+            let data_key = [0u8; 32];
            let bucket = "bucket";
            let object = "managed-minio-metadata";
            let object_key = [0x44; 32];
@@ -2722,10 +2689,6 @@ mod tests {
                size: encrypted.len() as i64,
                user_defined: Arc::new(HashMap::from([
                    ("x-amz-server-side-encryption".to_string(), "AES256".to_string()),
-                    (
-                        MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(),
-                        BASE64_STANDARD.encode(encrypted_dek.as_bytes()),
-                    ),
                    (
                        MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(),
                        BASE64_STANDARD.encode(sealed_key),
@@ -2735,7 +2698,6 @@ mod tests {
                        MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
                        MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
                    ),
-                    (MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()),
                    ("x-minio-internal-actual-size".to_string(), plaintext.len().to_string()),
                ])),
                ..Default::default()
@@ -2751,6 +2713,7 @@ mod tests {
            .await
            .expect("managed encrypted reads should accept MinIO-style metadata");

+            assert_eq!(reader.resolved_sse, Some(GetObjectSse::SseS3));
            let mut actual = Vec::new();
            reader.read_to_end(&mut actual).await.expect("read managed plaintext");

@@ -2762,6 +2725,128 @@ mod tests {
        .await;
    }

+    #[cfg(feature = "rio-v2")]
+    #[tokio::test]
+    async fn test_get_object_reader_accepts_minio_legacy_key_value_metadata() {
+        let external_key = [0x23; 32];
+        async_with_vars([("RUSTFS_SSE_S3_MASTER_KEY", Some(BASE64_STANDARD.encode(external_key)))], async {
+            let plaintext = b"managed-minio-key-value-metadata".to_vec();
+            let bucket = "bucket";
+            let object = "managed-minio-key-value-metadata";
+            let object_key = [0x45; 32];
+            let (sealing_iv, sealed_key) = seal_managed_s3_object_key_for_test(bucket, object, external_key, object_key);
+
+            let mut encrypted = Vec::new();
+            crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key)
+                .read_to_end(&mut encrypted)
+                .await
+                .expect("encrypt managed object");
+
+            let object_info = ObjectInfo {
+                bucket: bucket.to_string(),
+                name: object.to_string(),
+                size: encrypted.len() as i64,
+                user_defined: Arc::new(HashMap::from([
+                    ("x-amz-server-side-encryption".to_string(), "AES256".to_string()),
+                    (
+                        MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(),
+                        BASE64_STANDARD.encode(sealed_key),
+                    ),
+                    (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)),
+                    (
+                        MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
+                        MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
+                    ),
+                    ("x-minio-internal-actual-size".to_string(), plaintext.len().to_string()),
+                ])),
+                ..Default::default()
+            };
+
+            let (mut reader, _, _) = GetObjectReader::new(
+                Box::new(Cursor::new(encrypted)),
+                None,
+                &object_info,
+                &ObjectOptions::default(),
+                &HeaderMap::new(),
+            )
+            .await
+            .expect("MinIO historical K/V metadata should remain readable");
+            let mut actual = Vec::new();
+            reader
+                .read_to_end(&mut actual)
+                .await
+                .expect("read MinIO historical K/V object");
+
+            assert_eq!(reader.resolved_sse, Some(GetObjectSse::SseS3));
+            assert_eq!(actual, plaintext);
+        })
+        .await;
+    }
+
+    #[cfg(feature = "rio-v2")]
+    #[tokio::test]
+    async fn test_get_object_reader_accepts_minio_legacy_kms_key_value_metadata() {
+        let external_key = [0x26; 32];
+        async_with_vars([("RUSTFS_SSE_S3_MASTER_KEY", Some(BASE64_STANDARD.encode(external_key)))], async {
+            let plaintext = b"managed-minio-kms-key-value-metadata".to_vec();
+            let bucket = "bucket";
+            let object = "managed-minio-kms-key-value-metadata";
+            let object_key = [0x48; 32];
+            let (sealing_iv, sealed_key) =
+                seal_managed_object_key_for_test(bucket, object, external_key, object_key, "SSE-KMS", DARE_CIPHER_AES_256_GCM);
+
+            let mut encrypted = Vec::new();
+            crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key)
+                .read_to_end(&mut encrypted)
+                .await
+                .expect("encrypt managed KMS object");
+
+            let object_info = ObjectInfo {
+                bucket: bucket.to_string(),
+                name: object.to_string(),
+                size: encrypted.len() as i64,
+                user_defined: Arc::new(HashMap::from([
+                    ("x-amz-server-side-encryption".to_string(), "aws:kms".to_string()),
+                    (
+                        MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(),
+                        BASE64_STANDARD.encode(sealed_key),
+                    ),
+                    (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)),
+                    (
+                        MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
+                        MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
+                    ),
+                    ("x-minio-internal-actual-size".to_string(), plaintext.len().to_string()),
+                ])),
+                ..Default::default()
+            };
+
+            let (mut reader, _, _) = GetObjectReader::new(
+                Box::new(Cursor::new(encrypted)),
+                None,
+                &object_info,
+                &ObjectOptions::default(),
+                &HeaderMap::new(),
+            )
+            .await
+            .expect("MinIO historical SSE-KMS K/V metadata should remain readable");
+            let mut actual = Vec::new();
+            reader
+                .read_to_end(&mut actual)
+                .await
+                .expect("read MinIO historical SSE-KMS K/V object");
+
+            assert_eq!(
+                reader.resolved_sse,
+                Some(GetObjectSse::SseKms {
+                    key_id: "default".to_string(),
+                })
+            );
+            assert_eq!(actual, plaintext);
+        })
+        .await;
+    }
+
    #[tokio::test]
    async fn test_get_object_reader_compressed_range_returns_physical_offset_from_index() {
        let mut index = Index::new();
@@ -3034,6 +3119,12 @@ mod tests {
        .await
        .expect("ssec read should be supported");

+        assert_eq!(
+            reader.resolved_sse,
+            Some(GetObjectSse::SseC {
+                customer_key_md5: BASE64_STANDARD.encode(md5_bytes(key_bytes)),
+            })
+        );
        let mut actual = Vec::new();
        reader.read_to_end(&mut actual).await.expect("read decrypted ssec object");

@@ -3096,6 +3187,12 @@ mod tests {
        .await
        .expect("rio-v2 ssec sealed-object-key read should be supported");

+        assert_eq!(
+            reader.resolved_sse,
+            Some(GetObjectSse::SseC {
+                customer_key_md5: BASE64_STANDARD.encode(md5_bytes(customer_key)),
+            })
+        );
        let mut actual = Vec::new();
        reader
            .read_to_end(&mut actual)
@@ -253,7 +253,7 @@ impl NotificationSys {
        join_all(futures).await
    }

-    pub async fn reload_dynamic_config(&self, sub_sys: &str) -> Vec<NotificationPeerErr> {
+    async fn signal_dynamic_config(&self, sub_sys: &str, dry_run: bool) -> Vec<NotificationPeerErr> {
        let mut futures = Vec::with_capacity(self.peer_clients.len());
        for client in self.peer_clients.iter() {
            let sub_sys = sub_sys.to_string();
@@ -263,7 +263,7 @@ impl NotificationSys {
                        .signal_service(
                            crate::cluster::rpc::SERVICE_SIGNAL_RELOAD_DYNAMIC,
                            &sub_sys,
-                            false,
+                            dry_run,
                            SystemTime::UNIX_EPOCH,
                        )
                        .await
@@ -288,6 +288,14 @@ impl NotificationSys {
        join_all(futures).await
    }

+    pub async fn preflight_dynamic_config(&self, sub_sys: &str) -> Vec<NotificationPeerErr> {
+        self.signal_dynamic_config(sub_sys, true).await
+    }
+
+    pub async fn reload_dynamic_config(&self, sub_sys: &str) -> Vec<NotificationPeerErr> {
+        self.signal_dynamic_config(sub_sys, false).await
+    }
+
    pub async fn refresh_config_snapshot(&self) -> Vec<NotificationPeerErr> {
        let mut futures = Vec::with_capacity(self.peer_clients.len());
        for client in self.peer_clients.iter() {
@@ -141,6 +141,7 @@ impl MigrationBackendSpy {
            stream: Box::new(Cursor::new(vec![0_u8; 3])),
            object_info: ObjectInfo::default(),
            buffered_body: None,
+            resolved_sse: None,
            body_source: Default::default(),
        }
    }
@@ -74,7 +74,9 @@ use crate::disk::format::FormatV3;
 use crate::disk::{DiskAPI, DiskOption, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE, new_disk};
 use crate::services::tier::tier::TierConfigMgr;
 use crate::services::tier::tier_config::{TierConfig, TierMinIO, TierType};
-use crate::services::tier::warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options};
+use crate::services::tier::warm_backend::{
+    TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
+};
 use rustfs_filemeta::FileMeta;
 use rustfs_utils::path::path_join_buf;

@@ -142,6 +144,7 @@ pub enum MockWarmOp {
    Put { object: String },
    Get { object: String },
    Remove { object: String },
+    Probe { object: String },
    ExternalRemove { object: String },
    InUse,
 }
@@ -160,7 +163,10 @@ struct MockWarmBackendInner {
    faults: Mutex<FaultConfig>,
    put_read_limit: Mutex<Option<usize>>,
    put_remote_version: Mutex<Option<String>>,
+    response_loss_after_put: AtomicBool,
+    transition_candidate_probe_override: Mutex<Option<TransitionCandidateProbe>>,
    reject_non_empty_remote_versions: AtomicBool,
+    reject_non_empty_remote_version_validations: AtomicUsize,
    fail_remove: AtomicBool,
    exact_remove_count: AtomicUsize,
    op_log: Mutex<Vec<MockWarmOp>>,
@@ -378,11 +384,29 @@ impl MockWarmBackend {
        *self.inner.put_remote_version.lock().await = remote_version;
    }

+    /// Make the next PUT persist its body remotely but lose its response.
+    pub fn lose_next_put_response(&self) {
+        self.inner.response_loss_after_put.store(true, Ordering::Release);
+    }
+
+    /// Override candidate probing for transition-recovery fail-closed tests.
+    pub async fn set_transition_candidate_probe_override(&self, probe: Option<TransitionCandidateProbe>) {
+        *self.inner.transition_candidate_probe_override.lock().await = probe;
+    }
+
    /// Reject non-empty remote versions before transition metadata is committed.
    pub fn set_reject_non_empty_remote_versions(&self, reject: bool) {
        self.inner.reject_non_empty_remote_versions.store(reject, Ordering::Release);
    }

+    /// Reject the next non-empty remote version validation without changing
+    /// subsequent exact-version backend cleanup behavior.
+    pub fn reject_next_non_empty_remote_version_validation(&self) {
+        self.inner
+            .reject_non_empty_remote_version_validations
+            .fetch_add(1, Ordering::AcqRel);
+    }
+
    /// Enable or disable a persistent remove failure for durability tests.
    pub fn set_remove_failure(&self, fail: bool) {
        self.inner.fail_remove.store(fail, Ordering::Release);
@@ -444,6 +468,11 @@ impl MockWarmBackend {
        self.inner.remove_versions.lock().await.clone()
    }

+    /// Return the provider-authoritative view of a remote transition candidate.
+    pub async fn probe_transition_candidate_state(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        self.probe_transition_candidate(object).await
+    }
+
    /// Number of `get` calls recorded — useful to assert restore reads hit the
    /// local copy rather than the remote tier.
    pub async fn get_count(&self) -> usize {
@@ -575,12 +604,30 @@ impl MockWarmBackend {
            }
        }
    }
+
+    fn maybe_lose_put_response(&self) -> Result<(), std::io::Error> {
+        if self.inner.response_loss_after_put.swap(false, Ordering::AcqRel) {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::ConnectionReset,
+                "mock warm backend lost PUT response after storing remote object",
+            ));
+        }
+        Ok(())
+    }
 }

 #[async_trait]
 impl WarmBackend for MockWarmBackend {
    fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> {
-        if self.inner.reject_non_empty_remote_versions.load(Ordering::Acquire) && !remote_version_id.is_empty() {
+        if remote_version_id.is_empty() {
+            return Ok(());
+        }
+        let reject_once = self
+            .inner
+            .reject_non_empty_remote_version_validations
+            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| remaining.checked_sub(1))
+            .is_ok();
+        if reject_once || self.inner.reject_non_empty_remote_versions.load(Ordering::Acquire) {
            return Err(std::io::Error::other("mock warm backend requires an unversioned remote object"));
        }
        Ok(())
@@ -599,6 +646,7 @@ impl WarmBackend for MockWarmBackend {
            object: object.to_string(),
        })
        .await;
+        self.maybe_lose_put_response()?;
        Ok(version)
    }

@@ -649,6 +697,7 @@ impl WarmBackend for MockWarmBackend {
            object: object.to_string(),
        })
        .await;
+        self.maybe_lose_put_response()?;
        Ok(version)
    }

@@ -725,6 +774,26 @@ impl WarmBackend for MockWarmBackend {
        self.remove(object, rv).await
    }

+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        self.precondition().await?;
+        self.record(MockWarmOp::Probe {
+            object: object.to_string(),
+        })
+        .await;
+        if let Some(probe) = self.inner.transition_candidate_probe_override.lock().await.clone() {
+            return Ok(probe);
+        }
+        let objects = self.inner.objects.lock().await;
+        let Some(stored) = objects.get(object) else {
+            return Ok(TransitionCandidateProbe::Missing);
+        };
+        if stored.remote_version_id.is_empty() {
+            Ok(TransitionCandidateProbe::UnversionedPresent)
+        } else {
+            Ok(TransitionCandidateProbe::VersionedPresent(stored.remote_version_id.clone()))
+        }
+    }
+
    async fn in_use(&self) -> Result<bool, std::io::Error> {
        self.precondition().await?;
        self.record(MockWarmOp::InUse).await;
@@ -909,3 +978,73 @@ pub async fn wait_for_free_version_absence(disk_path: &Path, bucket: &str, objec
        tokio::time::sleep(POLL_INTERVAL).await;
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use bytes::Bytes;
+
+    #[tokio::test]
+    async fn mock_probe_distinguishes_missing_unversioned_and_versioned_candidates() {
+        let backend = MockWarmBackend::new();
+
+        assert_eq!(
+            backend
+                .probe_transition_candidate_state("missing")
+                .await
+                .expect("probe missing candidate"),
+            TransitionCandidateProbe::Missing
+        );
+
+        backend.set_put_remote_version(Some(String::new())).await;
+        backend
+            .put("unversioned", ReaderImpl::Body(Bytes::new()), 0)
+            .await
+            .expect("put unversioned candidate");
+        assert_eq!(
+            backend
+                .probe_transition_candidate_state("unversioned")
+                .await
+                .expect("probe unversioned candidate"),
+            TransitionCandidateProbe::UnversionedPresent
+        );
+
+        let remote_version = Uuid::new_v4().to_string();
+        backend.set_put_remote_version(Some(remote_version.clone())).await;
+        backend
+            .put("versioned", ReaderImpl::Body(Bytes::new()), 0)
+            .await
+            .expect("put versioned candidate");
+        assert_eq!(
+            backend
+                .probe_transition_candidate_state("versioned")
+                .await
+                .expect("probe versioned candidate"),
+            TransitionCandidateProbe::VersionedPresent(remote_version)
+        );
+
+        assert_eq!(
+            backend
+                .op_log()
+                .await
+                .into_iter()
+                .filter(|op| matches!(op, MockWarmOp::Probe { .. }))
+                .count(),
+            3
+        );
+    }
+
+    #[tokio::test]
+    async fn mock_probe_preserves_fault_fail_closed_behavior() {
+        let backend = MockWarmBackend::new();
+        backend.set_reject_credentials(true).await;
+
+        let err = backend
+            .probe_transition_candidate("remote-object")
+            .await
+            .expect_err("credential rejection must fail the authoritative probe");
+
+        assert_eq!(err.kind(), std::io::ErrorKind::PermissionDenied);
+        assert!(backend.op_log().await.is_empty());
+    }
+}
@@ -21,14 +21,17 @@ use uuid::Uuid;
 use crate::config::com;
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result as EcstoreResult};
-use crate::object_api::ObjectOptions;
+use crate::object_api::{ObjectInfo, ObjectOptions};
 use crate::services::tier::tier::TierDestinationId;
-use crate::storage_api_contracts::{list::ListOperations as _, object::HTTPPreconditions};
-use crate::store::ECStore;
+use crate::storage_api_contracts::{
+    list::{ListOperations, StorageListObjectsV2Info},
+    object::{EcstoreObjectIO, EcstoreObjectOperations, HTTPPreconditions},
+};

 pub(crate) const TIER_MUTATION_INTENT_SCHEMA: &str = "rustfs-tier-mutation-intent-v1";
 pub(crate) const MAX_TIER_MUTATION_INTENT_SIZE: usize = rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE;
 pub(crate) const TIER_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/records";
+pub(crate) const TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/coordinators";
 pub(crate) type TierMutationDigest = [u8; 32];

 pub(crate) type Result<T> = std::result::Result<T, TierMutationIntentError>;
@@ -293,21 +296,23 @@ impl TierMutationIntent {
 }

 pub(crate) fn tier_mutation_intent_record_object_name(mutation_id: Uuid) -> Result<String> {
+    tier_mutation_intent_record_object_name_with_prefix(TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id)
+}
+
+fn tier_mutation_intent_record_object_name_with_prefix(prefix: &str, mutation_id: Uuid) -> Result<String> {
    if mutation_id.is_nil() {
        return Err(TierMutationIntentError::Corrupt("mutation_id is nil"));
    }
    let mutation_key = mutation_id.simple().to_string();
-    Ok(format!(
-        "{}/{}/{}/{}.json",
-        TIER_MUTATION_INTENT_RECORD_PREFIX,
-        &mutation_key[..2],
-        &mutation_key[2..4],
-        mutation_key
-    ))
+    Ok(format!("{}/{}/{}/{}.json", prefix, &mutation_key[..2], &mutation_key[2..4], mutation_key))
 }

 pub(crate) fn tier_mutation_intent_id_from_record_object_name(object: &str) -> Result<Uuid> {
-    let prefix = format!("{TIER_MUTATION_INTENT_RECORD_PREFIX}/");
+    tier_mutation_intent_id_from_record_object_name_with_prefix(TIER_MUTATION_INTENT_RECORD_PREFIX, object)
+}
+
+fn tier_mutation_intent_id_from_record_object_name_with_prefix(prefix: &str, object: &str) -> Result<Uuid> {
+    let prefix = format!("{prefix}/");
    let suffix = object
        .strip_prefix(&prefix)
        .ok_or(TierMutationIntentError::Corrupt("intent record path has wrong prefix"))?;
@@ -340,17 +345,43 @@ pub(crate) fn tier_mutation_intent_id_from_record_object_name(object: &str) -> R
    Uuid::parse_str(mutation_key).map_err(|_| TierMutationIntentError::Corrupt("intent record path has invalid uuid"))
 }

-pub(crate) async fn save_tier_mutation_intent_record(api: Arc<ECStore>, intent: &TierMutationIntent) -> EcstoreResult<()> {
-    let object = tier_mutation_intent_record_object_name(intent.mutation_id).map_err(tier_mutation_intent_store_error)?;
+pub(crate) async fn save_tier_mutation_intent_record<S>(api: Arc<S>, intent: &TierMutationIntent) -> EcstoreResult<()>
+where
+    S: EcstoreObjectIO,
+{
+    let object = tier_mutation_intent_record_object_name_with_prefix(TIER_MUTATION_INTENT_RECORD_PREFIX, intent.mutation_id)
+        .map_err(tier_mutation_intent_store_error)?;
    let data = intent.encode().map_err(tier_mutation_intent_store_error)?;
    com::save_config(api, &object, data).await
 }

-pub(crate) async fn save_tier_mutation_intent_record_if_absent(
-    api: Arc<ECStore>,
+pub(crate) async fn save_tier_coordinator_mutation_intent_record_if_absent<S>(
+    api: Arc<S>,
    intent: &TierMutationIntent,
-) -> EcstoreResult<()> {
-    let object = tier_mutation_intent_record_object_name(intent.mutation_id).map_err(tier_mutation_intent_store_error)?;
+) -> EcstoreResult<()>
+where
+    S: EcstoreObjectIO,
+{
+    save_tier_mutation_intent_record_if_absent_with_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, intent).await
+}
+
+pub(crate) async fn save_tier_mutation_intent_record_if_absent<S>(api: Arc<S>, intent: &TierMutationIntent) -> EcstoreResult<()>
+where
+    S: EcstoreObjectIO,
+{
+    save_tier_mutation_intent_record_if_absent_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, intent).await
+}
+
+async fn save_tier_mutation_intent_record_if_absent_with_prefix<S>(
+    api: Arc<S>,
+    prefix: &str,
+    intent: &TierMutationIntent,
+) -> EcstoreResult<()>
+where
+    S: EcstoreObjectIO,
+{
+    let object = tier_mutation_intent_record_object_name_with_prefix(prefix, intent.mutation_id)
+        .map_err(tier_mutation_intent_store_error)?;
    let data = intent.encode().map_err(tier_mutation_intent_store_error)?;
    com::save_config_with_opts(
        api,
@@ -368,16 +399,34 @@ pub(crate) async fn save_tier_mutation_intent_record_if_absent(
    .await
 }

-pub(crate) async fn load_tier_mutation_intent_record(api: Arc<ECStore>, mutation_id: Uuid) -> EcstoreResult<TierMutationIntent> {
+pub(crate) async fn load_tier_mutation_intent_record<S>(api: Arc<S>, mutation_id: Uuid) -> EcstoreResult<TierMutationIntent>
+where
+    S: EcstoreObjectIO,
+{
    let (intent, _) = load_tier_mutation_intent_record_with_etag(api, mutation_id).await?;
    Ok(intent)
 }

-pub(crate) async fn load_tier_mutation_intent_record_with_etag(
-    api: Arc<ECStore>,
+pub(crate) async fn load_tier_mutation_intent_record_with_etag<S>(
+    api: Arc<S>,
    mutation_id: Uuid,
-) -> EcstoreResult<(TierMutationIntent, String)> {
-    let object = tier_mutation_intent_record_object_name(mutation_id).map_err(tier_mutation_intent_store_error)?;
+) -> EcstoreResult<(TierMutationIntent, String)>
+where
+    S: EcstoreObjectIO,
+{
+    load_tier_mutation_intent_record_with_etag_at_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
+}
+
+async fn load_tier_mutation_intent_record_with_etag_at_prefix<S>(
+    api: Arc<S>,
+    prefix: &str,
+    mutation_id: Uuid,
+) -> EcstoreResult<(TierMutationIntent, String)>
+where
+    S: EcstoreObjectIO,
+{
+    let object =
+        tier_mutation_intent_record_object_name_with_prefix(prefix, mutation_id).map_err(tier_mutation_intent_store_error)?;
    let (data, object_info) = com::read_config_with_metadata(api, &object, &ObjectOptions::default()).await?;
    let etag = object_info
        .etag
@@ -387,15 +436,34 @@ pub(crate) async fn load_tier_mutation_intent_record_with_etag(
    Ok((intent, etag))
 }

-pub(crate) async fn save_tier_mutation_intent_record_if_current(
-    api: Arc<ECStore>,
+pub(crate) async fn save_tier_mutation_intent_record_if_current<S>(
+    api: Arc<S>,
    intent: &TierMutationIntent,
    current_etag: &str,
-) -> EcstoreResult<()> {
+) -> EcstoreResult<()>
+where
+    S: EcstoreObjectIO,
+{
    if current_etag.trim().is_empty() {
        return Err(Error::other("tier mutation intent current ETag is empty"));
    }
-    let object = tier_mutation_intent_record_object_name(intent.mutation_id).map_err(tier_mutation_intent_store_error)?;
+    save_tier_mutation_intent_record_if_current_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, intent, current_etag).await
+}
+
+async fn save_tier_mutation_intent_record_if_current_with_prefix<S>(
+    api: Arc<S>,
+    prefix: &str,
+    intent: &TierMutationIntent,
+    current_etag: &str,
+) -> EcstoreResult<()>
+where
+    S: EcstoreObjectIO,
+{
+    if current_etag.trim().is_empty() {
+        return Err(Error::other("tier mutation intent current ETag is empty"));
+    }
+    let object = tier_mutation_intent_record_object_name_with_prefix(prefix, intent.mutation_id)
+        .map_err(tier_mutation_intent_store_error)?;
    let data = intent.encode().map_err(tier_mutation_intent_store_error)?;
    com::save_config_with_opts(
        api,
@@ -413,35 +481,122 @@ pub(crate) async fn save_tier_mutation_intent_record_if_current(
    .await
 }

-pub(crate) async fn delete_tier_mutation_intent_record(api: Arc<ECStore>, mutation_id: Uuid) -> EcstoreResult<()> {
-    let object = tier_mutation_intent_record_object_name(mutation_id).map_err(tier_mutation_intent_store_error)?;
+pub(crate) async fn delete_tier_mutation_intent_record<S>(api: Arc<S>, mutation_id: Uuid) -> EcstoreResult<()>
+where
+    S: EcstoreObjectOperations,
+{
+    delete_tier_mutation_intent_record_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
+}
+
+pub(crate) async fn delete_tier_coordinator_mutation_intent_record<S>(api: Arc<S>, mutation_id: Uuid) -> EcstoreResult<()>
+where
+    S: EcstoreObjectOperations,
+{
+    delete_tier_mutation_intent_record_with_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
+}
+
+async fn delete_tier_mutation_intent_record_with_prefix<S>(api: Arc<S>, prefix: &str, mutation_id: Uuid) -> EcstoreResult<()>
+where
+    S: EcstoreObjectOperations,
+{
+    let object =
+        tier_mutation_intent_record_object_name_with_prefix(prefix, mutation_id).map_err(tier_mutation_intent_store_error)?;
    match com::delete_config(api, &object).await {
        Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
        Err(err) => Err(err),
    }
 }

-pub(crate) async fn advance_tier_mutation_intent_record_idempotent(
-    api: Arc<ECStore>,
+pub(crate) async fn advance_tier_mutation_intent_record_idempotent<S>(
+    api: Arc<S>,
    mutation_id: Uuid,
    next: TierMutationIntentState,
    committed_config_etag: Option<String>,
-) -> EcstoreResult<(TierMutationIntent, bool)> {
-    let (mut intent, current_etag) = load_tier_mutation_intent_record_with_etag(api.clone(), mutation_id).await?;
+) -> EcstoreResult<(TierMutationIntent, bool)>
+where
+    S: EcstoreObjectIO,
+{
+    advance_tier_mutation_intent_record_idempotent_at_prefix(
+        api,
+        TIER_MUTATION_INTENT_RECORD_PREFIX,
+        mutation_id,
+        next,
+        committed_config_etag,
+    )
+    .await
+}
+
+pub(crate) async fn advance_tier_coordinator_mutation_intent_record_idempotent<S>(
+    api: Arc<S>,
+    mutation_id: Uuid,
+    next: TierMutationIntentState,
+    committed_config_etag: Option<String>,
+) -> EcstoreResult<(TierMutationIntent, bool)>
+where
+    S: EcstoreObjectIO,
+{
+    advance_tier_mutation_intent_record_idempotent_at_prefix(
+        api,
+        TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX,
+        mutation_id,
+        next,
+        committed_config_etag,
+    )
+    .await
+}
+
+async fn advance_tier_mutation_intent_record_idempotent_at_prefix<S>(
+    api: Arc<S>,
+    prefix: &str,
+    mutation_id: Uuid,
+    next: TierMutationIntentState,
+    committed_config_etag: Option<String>,
+) -> EcstoreResult<(TierMutationIntent, bool)>
+where
+    S: EcstoreObjectIO,
+{
+    let (mut intent, current_etag) =
+        load_tier_mutation_intent_record_with_etag_at_prefix(api.clone(), prefix, mutation_id).await?;
    let advanced = intent
        .advance_idempotent(next, committed_config_etag)
        .map_err(tier_mutation_intent_store_error)?;
    if advanced {
-        save_tier_mutation_intent_record_if_current(api, &intent, &current_etag).await?;
+        save_tier_mutation_intent_record_if_current_with_prefix(api, prefix, &intent, &current_etag).await?;
    }
    Ok((intent, advanced))
 }

-pub(crate) async fn list_tier_mutation_intent_records(
-    api: Arc<ECStore>,
+pub(crate) async fn list_tier_mutation_intent_records<S>(
+    api: Arc<S>,
    limit: usize,
    marker: Option<String>,
-) -> EcstoreResult<TierMutationIntentRecordScan> {
+) -> EcstoreResult<TierMutationIntentRecordScan>
+where
+    S: EcstoreObjectIO + ListOperations<Error = Error, ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>>,
+{
+    list_tier_mutation_intent_records_at_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, limit, marker).await
+}
+
+pub(crate) async fn list_tier_coordinator_mutation_intent_records<S>(
+    api: Arc<S>,
+    limit: usize,
+    marker: Option<String>,
+) -> EcstoreResult<TierMutationIntentRecordScan>
+where
+    S: EcstoreObjectIO + ListOperations<Error = Error, ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>>,
+{
+    list_tier_mutation_intent_records_at_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, limit, marker).await
+}
+
+async fn list_tier_mutation_intent_records_at_prefix<S>(
+    api: Arc<S>,
+    prefix: &str,
+    limit: usize,
+    marker: Option<String>,
+) -> EcstoreResult<TierMutationIntentRecordScan>
+where
+    S: EcstoreObjectIO + ListOperations<Error = Error, ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>>,
+{
    if limit == 0 {
        return Err(Error::other("tier mutation intent scan limit must be greater than zero"));
    }
@@ -450,7 +605,7 @@ pub(crate) async fn list_tier_mutation_intent_records(
        .clone()
        .list_objects_v2(
            RUSTFS_META_BUCKET,
-            TIER_MUTATION_INTENT_RECORD_PREFIX,
+            prefix,
            marker,
            None,
            i32::try_from(limit).map_or(i32::MAX, |value| value),
@@ -469,15 +624,15 @@ pub(crate) async fn list_tier_mutation_intent_records(

    for object in list.objects {
        scan.scanned += 1;
-        let mutation_id = match tier_mutation_intent_id_from_record_object_name(&object.name) {
+        let mutation_id = match tier_mutation_intent_id_from_record_object_name_with_prefix(prefix, &object.name) {
            Ok(mutation_id) => mutation_id,
            Err(_) => {
                scan.failed += 1;
                continue;
            }
        };
-        match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
-            Ok(intent) => scan.intents.push(intent),
+        match load_tier_mutation_intent_record_with_etag_at_prefix(api.clone(), prefix, mutation_id).await {
+            Ok((intent, _)) => scan.intents.push(intent),
            Err(Error::ConfigNotFound) => {}
            Err(_) => scan.failed += 1,
        }
@@ -17,7 +17,7 @@ use std::sync::Arc;
 use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase};
 use uuid::Uuid;

-use super::tier::TierConfigMgr;
+use super::tier::{TierConfigMgr, tier_config_etag_matches};
 use super::tier_mutation_intent::{
    MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, advance_tier_mutation_intent_record_idempotent,
    load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
@@ -140,13 +140,27 @@ async fn handle_commit(
 ) -> TierMutationPeerResult<TierMutationPeerOutcome> {
    let committed_config_etag = parse_commit_etag(canonical_payload)?;
    let tier_config_mgr = api.tier_config_mgr();
-    let (intent, applied) = advance_tier_mutation_intent_record_idempotent(
-        api,
+    let (intent, applied) = match advance_tier_mutation_intent_record_idempotent(
+        api.clone(),
        mutation_id,
        TierMutationIntentState::Committed,
-        Some(committed_config_etag),
+        Some(committed_config_etag.clone()),
    )
-    .await?;
+    .await
+    {
+        Ok(result) => result,
+        Err(Error::ConfigNotFound)
+            if tier_config_etag_matches(api, &committed_config_etag)
+                .await
+                .map_err(Error::other)? =>
+        {
+            return Ok(TierMutationPeerOutcome {
+                state: TierMutationPeerState::Committed,
+                applied: false,
+            });
+        }
+        Err(err) => return Err(err.into()),
+    };
    if intent.state == TierMutationIntentState::Committed {
        TierConfigMgr::clear_prepared_mutation_intent_block(&tier_config_mgr, mutation_id).await;
    }
@@ -64,6 +64,15 @@ pub struct WarmBackendGetOpts {
    pub length: i64,
 }

+#[derive(Clone, Debug, Eq, PartialEq)]
+pub enum TransitionCandidateProbe {
+    Missing,
+    UnversionedPresent,
+    VersionedPresent(String),
+    Ambiguous,
+    Unsupported,
+}
+
 #[async_trait::async_trait]
 pub trait WarmBackend {
    async fn validate(&self) -> Result<(), std::io::Error> {
@@ -101,6 +110,9 @@ pub trait WarmBackend {
        }
        self.remove(object, rv).await
    }
+    async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        Ok(TransitionCandidateProbe::Unsupported)
+    }
    async fn in_use(&self) -> Result<bool, std::io::Error>;
 }

@@ -615,6 +627,22 @@ mod tests {
        assert_eq!(removes.load(Ordering::SeqCst), 0);
    }

+    #[tokio::test]
+    async fn default_transition_candidate_probe_is_unsupported() {
+        let backend = RejectingValidationBackend {
+            validations: Arc::new(AtomicUsize::new(0)),
+            puts: Arc::new(AtomicUsize::new(0)),
+            removes: Arc::new(AtomicUsize::new(0)),
+        };
+
+        let probe = backend
+            .probe_transition_candidate("remote-object")
+            .await
+            .expect("default candidate probe should be a safe capability response");
+
+        assert_eq!(probe, TransitionCandidateProbe::Unsupported);
+    }
+
    #[tokio::test]
    async fn check_warm_backend_removes_exact_probe_when_versioning_drifts() {
        let gets = Arc::new(AtomicUsize::new(0));
@@ -29,7 +29,7 @@ use crate::client::{
 };
 use crate::services::tier::{
    tier_config::TierMinIO,
-    warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
+    warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
    warm_backend_s3::WarmBackendS3,
 };
 use tracing::warn;
@@ -126,6 +126,10 @@ impl WarmBackend for WarmBackendMinIO {
        self.0.remove(object, rv).await
    }

+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        self.0.probe_transition_candidate(object).await
+    }
+
    async fn in_use(&self) -> Result<bool, std::io::Error> {
        self.0.in_use().await
    }
@@ -29,7 +29,7 @@ use crate::client::{
 };
 use crate::services::tier::{
    tier_config::TierR2,
-    warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
+    warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
    warm_backend_s3::WarmBackendS3,
 };
 use tracing::warn;
@@ -126,6 +126,10 @@ impl WarmBackend for WarmBackendR2 {
        self.0.remove(object, rv).await
    }

+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        self.0.probe_transition_candidate(object).await
+    }
+
    async fn in_use(&self) -> Result<bool, std::io::Error> {
        self.0.in_use().await
    }
@@ -29,7 +29,7 @@ use crate::client::{
 };
 use crate::services::tier::{
    tier_config::TierRustFS,
-    warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
+    warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
    warm_backend_s3::WarmBackendS3,
 };

@@ -123,6 +123,10 @@ impl WarmBackend for WarmBackendRustFS {
        self.0.remove(object, rv).await
    }

+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        self.0.probe_transition_candidate(object).await
+    }
+
    async fn in_use(&self) -> Result<bool, std::io::Error> {
        self.0.in_use().await
    }
@@ -24,8 +24,10 @@ use url::Url;

 use crate::client::{
    api_get_options::GetObjectOptions,
+    api_list::ListObjectsOptions,
    api_put_object::PutObjectOptions,
    api_remove::{RemoveObjectOptions, RemoveObjectResult},
+    api_s3_datatypes::ListVersionsResult,
    credentials::{Credentials, SignatureType, Static, Value},
    transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore},
    transition_api::{ReadCloser, ReaderImpl},
@@ -34,11 +36,12 @@ use crate::error::ErrorResponse;
 use crate::error::error_resp_to_object_err;
 use crate::services::tier::{
    tier_config::TierS3,
-    warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
+    warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
 };
 use http::HeaderMap;
 use rustfs_utils::egress::validate_outbound_url;
 use rustfs_utils::path::SLASH_SEPARATOR;
+use s3s::dto::BucketVersioningStatus;

 pub struct WarmBackendS3 {
    pub client: Arc<TransitionClient>,
@@ -48,6 +51,27 @@ pub struct WarmBackendS3 {
    pub storage_class: String,
 }

+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum RemoteBucketVersioning {
+    Disabled,
+    Suspended,
+    Enabled,
+}
+
+fn remote_bucket_versioning_from_status(status: Option<&str>) -> Result<RemoteBucketVersioning, std::io::Error> {
+    Ok(match status {
+        Some(BucketVersioningStatus::ENABLED) => RemoteBucketVersioning::Enabled,
+        Some(BucketVersioningStatus::SUSPENDED) => RemoteBucketVersioning::Suspended,
+        Some(status) => {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::InvalidData,
+                format!("remote tier bucket returned unsupported versioning status {status}"),
+            ));
+        }
+        None => RemoteBucketVersioning::Disabled,
+    })
+}
+
 impl WarmBackendS3 {
    pub async fn new(conf: &TierS3, _tier: &str) -> Result<Self, std::io::Error> {
        Self::new_with_bucket_lookup(conf, BucketLookupType::BucketLookupAuto, "s3").await
@@ -159,11 +183,57 @@ impl WarmBackendS3 {
        let (_, headers, reader) = self.core.get_object(&self.bucket, &self.get_dest(object), &gopts).await?;
        Ok((headers, reader))
    }
+
+    async fn remote_bucket_versioning(&self) -> Result<RemoteBucketVersioning, std::io::Error> {
+        let config = self.client.get_bucket_versioning(&self.bucket).await?;
+        remote_bucket_versioning_from_status(config.status.as_ref().map(|status| status.as_str()))
+    }
+
+    async fn list_transition_candidate_versions(&self, object: &str) -> Result<ListVersionsResult, std::io::Error> {
+        let mut opts = ListObjectsOptions::default();
+        opts.set("prefix", &self.get_dest(object));
+        opts.set("max-keys", "2");
+        self.client.list_object_versions_query(&self.bucket, &opts, "", "", "").await
+    }
+}
+
+fn classify_transition_candidate_versions(
+    remote_object: &str,
+    bucket_versioning: RemoteBucketVersioning,
+    versions: &ListVersionsResult,
+) -> TransitionCandidateProbe {
+    if versions.is_truncated {
+        return TransitionCandidateProbe::Ambiguous;
+    }
+
+    if versions.delete_markers.iter().any(|marker| marker.key == remote_object) {
+        return TransitionCandidateProbe::Ambiguous;
+    }
+
+    let mut exact_versions = versions.versions.iter().filter(|version| version.key == remote_object);
+    let Some(version) = exact_versions.next() else {
+        return TransitionCandidateProbe::Missing;
+    };
+    if exact_versions.next().is_some() {
+        return TransitionCandidateProbe::Ambiguous;
+    }
+
+    match bucket_versioning {
+        RemoteBucketVersioning::Disabled => TransitionCandidateProbe::UnversionedPresent,
+        RemoteBucketVersioning::Suspended if version.version_id == "null" => {
+            TransitionCandidateProbe::VersionedPresent(version.version_id.clone())
+        }
+        RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled if !version.version_id.is_empty() => {
+            TransitionCandidateProbe::VersionedPresent(version.version_id.clone())
+        }
+        RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled => TransitionCandidateProbe::Ambiguous,
+    }
 }

 #[cfg(test)]
 mod tests {
    use super::*;
+    use crate::client::api_s3_datatypes::{ListVersionsResult, Version};

    #[tokio::test]
    async fn new_rejects_loopback_endpoint_before_network_setup() {
@@ -181,6 +251,116 @@ mod tests {
            Err(err) => assert!(err.to_string().contains("not allowed")),
        }
    }
+
+    fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult {
+        ListVersionsResult {
+            versions: versions
+                .iter()
+                .map(|(key, version_id)| Version {
+                    key: (*key).to_string(),
+                    version_id: (*version_id).to_string(),
+                    ..Default::default()
+                })
+                .collect(),
+            delete_markers: delete_markers
+                .iter()
+                .map(|(key, version_id)| Version {
+                    key: (*key).to_string(),
+                    version_id: (*version_id).to_string(),
+                    ..Default::default()
+                })
+                .collect(),
+            is_truncated,
+            ..Default::default()
+        }
+    }
+
+    #[test]
+    fn transition_candidate_probe_classifier_is_fail_closed() {
+        assert_eq!(
+            classify_transition_candidate_versions(
+                "archive/object",
+                RemoteBucketVersioning::Disabled,
+                &list_versions(&[], &[], false),
+            ),
+            TransitionCandidateProbe::Missing
+        );
+        assert_eq!(
+            classify_transition_candidate_versions(
+                "archive/object",
+                RemoteBucketVersioning::Disabled,
+                &list_versions(&[("archive/object", "")], &[], false),
+            ),
+            TransitionCandidateProbe::UnversionedPresent
+        );
+        assert_eq!(
+            classify_transition_candidate_versions(
+                "archive/object",
+                RemoteBucketVersioning::Enabled,
+                &list_versions(&[("archive/object", "version-a")], &[], false),
+            ),
+            TransitionCandidateProbe::VersionedPresent("version-a".to_string())
+        );
+        assert_eq!(
+            classify_transition_candidate_versions(
+                "archive/object",
+                RemoteBucketVersioning::Suspended,
+                &list_versions(&[("archive/object", "null")], &[], false),
+            ),
+            TransitionCandidateProbe::VersionedPresent("null".to_string())
+        );
+        assert_eq!(
+            classify_transition_candidate_versions(
+                "archive/object",
+                RemoteBucketVersioning::Enabled,
+                &list_versions(&[("archive/object", "")], &[], false),
+            ),
+            TransitionCandidateProbe::Ambiguous
+        );
+        assert_eq!(
+            classify_transition_candidate_versions(
+                "archive/object",
+                RemoteBucketVersioning::Enabled,
+                &list_versions(&[("archive/object", "version-a"), ("archive/object", "version-b")], &[], false),
+            ),
+            TransitionCandidateProbe::Ambiguous
+        );
+        assert_eq!(
+            classify_transition_candidate_versions(
+                "archive/object",
+                RemoteBucketVersioning::Enabled,
+                &list_versions(&[("archive/object", "version-a")], &[("archive/object", "marker-a")], false),
+            ),
+            TransitionCandidateProbe::Ambiguous
+        );
+        assert_eq!(
+            classify_transition_candidate_versions(
+                "archive/object",
+                RemoteBucketVersioning::Enabled,
+                &list_versions(&[("archive/object", "version-a")], &[], true),
+            ),
+            TransitionCandidateProbe::Ambiguous
+        );
+    }
+
+    #[test]
+    fn remote_bucket_versioning_status_parser_fails_closed() {
+        assert_eq!(
+            remote_bucket_versioning_from_status(None).expect("absent status means disabled"),
+            RemoteBucketVersioning::Disabled
+        );
+        assert_eq!(
+            remote_bucket_versioning_from_status(Some(BucketVersioningStatus::ENABLED)).expect("enabled status should parse"),
+            RemoteBucketVersioning::Enabled
+        );
+        assert_eq!(
+            remote_bucket_versioning_from_status(Some(BucketVersioningStatus::SUSPENDED)).expect("suspended status should parse"),
+            RemoteBucketVersioning::Suspended
+        );
+        let err = remote_bucket_versioning_from_status(Some("UnexpectedStatus"))
+            .expect_err("unknown versioning status must fail closed");
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
+    }
 }

 #[async_trait::async_trait]
@@ -215,6 +395,16 @@ impl WarmBackend for WarmBackendS3 {
        self.remove_with_result(object, rv).await.map(|_| ())
    }

+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        let bucket_versioning = self.remote_bucket_versioning().await?;
+        let versions = self.list_transition_candidate_versions(object).await?;
+        Ok(classify_transition_candidate_versions(
+            &self.get_dest(object),
+            bucket_versioning,
+            &versions,
+        ))
+    }
+
    async fn in_use(&self) -> Result<bool, std::io::Error> {
        let result = self
            .core
@@ -206,7 +206,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
        //     });
        // }

-        if object_info.size == 0 {
+        if object_info.size == 0 && !object_info.is_encrypted() {
            record_get_object_reader_path_observation(GET_OBJECT_PATH_EMPTY, object_class, size_bucket);
            // if let Some(rs) = range {
            //     let _ = rs.get_offset_length(object_info.size)?;
@@ -216,6 +216,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                stream: Box::new(Cursor::new(Vec::new())),
                object_info,
                buffered_body: Some(Bytes::new()),
+                resolved_sse: None,
                body_source: GetObjectBodySource::Unprobed,
            };
            return Ok(reader);
@@ -303,6 +304,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                        stream: Box::new(Cursor::new(body.clone())),
                        object_info,
                        buffered_body: Some(body),
+                        resolved_sse: None,
                        body_source: GetObjectBodySource::Unprobed,
                    };
                    return Ok(reader);
@@ -381,6 +383,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                    stream: Box::new(Cursor::new(body.clone())),
                    object_info,
                    buffered_body: Some(body),
+                    resolved_sse: None,
                    body_source: GetObjectBodySource::Unprobed,
                };
                return Ok(reader);
@@ -462,6 +465,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                        stream: Box::new(Cursor::new(body.clone())),
                        object_info,
                        buffered_body: Some(body),
+                        resolved_sse: None,
                        body_source: GetObjectBodySource::HookServed,
                    };
                    if lock_optimization_enabled {
@@ -505,6 +509,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                    stream: Box::new(Cursor::new(body.clone())),
                    object_info,
                    buffered_body: Some(body),
+                    resolved_sse: None,
                    body_source,
                };
                if lock_optimization_enabled {
@@ -547,6 +552,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                stream: Box::new(Cursor::new(body.clone())),
                object_info,
                buffered_body: Some(body),
+                resolved_sse: None,
                body_source,
            };
            if lock_optimization_enabled {
@@ -3311,6 +3317,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
            stream: Box::new(TransitionUploadReader::new(pr, Arc::clone(&consumed))),
            object_info: oi,
            buffered_body: None,
+            resolved_sse: None,
            body_source: GetObjectBodySource::Unprobed,
        });

@@ -3346,6 +3353,13 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
        };

        let mut upload_cleanup = TransitionUploadCleanup::new(tgt_client, &dest_obj, self.ctx.clone());
+        advance_and_save_transition_transaction(
+            transaction_api.as_ref(),
+            &mut transaction,
+            TransitionTransactionState::UploadOutcomeUnknown,
+            None,
+        )
+        .await?;
        let remote_upload = {
            let lease = &upload_cleanup.lease;
            let recorded_candidate = &mut upload_cleanup.candidate;
@@ -4040,6 +4054,54 @@ pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
    }
 }

+#[cfg(test)]
+mod zero_length_encrypted_read_tests {
+    use super::hermetic_set_disks_support::hermetic_set_disks;
+    use super::*;
+    use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
+
+    #[tokio::test]
+    async fn encrypted_zero_length_object_does_not_use_plain_empty_fast_path() {
+        let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "zero-length-encrypted-read";
+        let object = "empty";
+        set_disks
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket should be created");
+        let mut reader = PutObjReader::from_vec(Vec::new());
+        set_disks
+            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
+            .await
+            .expect("empty object should be written");
+
+        let metadata = HashMap::from([(rustfs_utils::http::SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]);
+        set_disks
+            .put_object_metadata(
+                bucket,
+                object,
+                &ObjectOptions {
+                    eval_metadata: Some(metadata),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("SSE-C metadata should be persisted");
+
+        let result = set_disks
+            .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+            .await;
+        let error = match result {
+            Ok(_) => panic!("encrypted empty object must validate SSE-C headers"),
+            Err(error) => error,
+        };
+        assert!(
+            error.to_string().contains("SSE-C") || error.to_string().contains("customer"),
+            "the encrypted read must fail at SSE-C validation, got: {error}"
+        );
+    }
+}
+
 #[cfg(test)]
 mod metadata_mutation_generation_tests {
    use super::hermetic_set_disks_support::hermetic_set_disks;
@@ -5906,7 +5968,6 @@ mod transition_upload_integrity_tests {
        let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
        let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
        let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
-        backend.set_reject_non_empty_remote_versions(true);

        for position in [
            ShardCorruptionPosition::First,
@@ -6078,7 +6139,7 @@ mod transition_upload_integrity_tests {
        let remote_version = Uuid::new_v4().to_string();
        let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
        backend.set_put_remote_version(Some(remote_version.clone())).await;
-        backend.set_reject_non_empty_remote_versions(true);
+        backend.reject_next_non_empty_remote_version_validation();

        set_disks
            .transition_object(bucket, object, &transition_options(&original, tier_name))
@@ -6106,7 +6167,7 @@ mod transition_upload_integrity_tests {
        let remote_version = Uuid::nil().to_string();
        let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
        backend.set_put_remote_version(Some(remote_version.clone())).await;
-        backend.set_reject_non_empty_remote_versions(true);
+        backend.reject_next_non_empty_remote_version_validation();

        set_disks
            .transition_object(bucket, object, &transition_options(&original, tier_name))
@@ -0,0 +1,950 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use aes_gcm::{
+    Aes256Gcm, Nonce,
+    aead::{Aead, KeyInit, Payload},
+};
+use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
+use chacha20poly1305::ChaCha20Poly1305;
+use hmac::{Hmac, Mac};
+use rustfs_utils::http::{AMZ_SERVER_SIDE_ENCRYPTION, get_consistent_metadata_value};
+use serde::Deserialize;
+use sha2::Sha256;
+use std::collections::HashMap;
+#[cfg(not(any(test, debug_assertions)))]
+use std::sync::OnceLock;
+use thiserror::Error;
+use zeroize::Zeroizing;
+
+pub(crate) const DEFAULT_SSE_ALGORITHM: &str = "AES256";
+const SSE_KMS_ALGORITHM: &str = "aws:kms";
+pub(crate) const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id";
+pub(crate) const INTERNAL_ENCRYPTION_KEY_HEADER: &str = "x-rustfs-encryption-key";
+pub(crate) const INTERNAL_ENCRYPTION_IV_HEADER: &str = "x-rustfs-encryption-iv";
+pub(crate) const AMZ_SERVER_SIDE_ENCRYPTION_KMS_KEY_ID: &str = "x-amz-server-side-encryption-aws-kms-key-id";
+pub(crate) const MINIO_INTERNAL_ENCRYPTION_IV_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Iv";
+pub(crate) const MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm";
+pub(crate) const MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key";
+pub(crate) const MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key";
+pub(crate) const MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER: &str =
+    "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key";
+pub(crate) const MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id";
+pub(crate) const MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM: &str = "DAREv2-HMAC-SHA256";
+const MINIO_STATIC_KMS_KEY_ENV: &str = "RUSTFS_MINIO_STATIC_KMS_KEY";
+const MINIO_STATIC_KMS_RANDOM_SIZE: usize = 28;
+const MINIO_STATIC_KMS_IV_SIZE: usize = 16;
+const MINIO_STATIC_KMS_NONCE_SIZE: usize = 12;
+
+type HmacSha256 = Hmac<Sha256>;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ManagedSseScheme {
+    SseS3,
+    SseKms,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ManagedDekProvider {
+    LocalSseS3,
+    MinioKeyValue,
+    Kms,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PersistedManagedEncryption {
+    LegacySseS3Local,
+    LegacySseS3Kms,
+    LegacySseKms,
+    MinioSseS3KeyValue,
+    MinioSseS3Kms,
+    MinioSseKmsKms,
+}
+
+impl PersistedManagedEncryption {
+    pub fn scheme(self) -> ManagedSseScheme {
+        match self {
+            Self::LegacySseS3Local | Self::LegacySseS3Kms | Self::MinioSseS3KeyValue | Self::MinioSseS3Kms => {
+                ManagedSseScheme::SseS3
+            }
+            Self::LegacySseKms | Self::MinioSseKmsKms => ManagedSseScheme::SseKms,
+        }
+    }
+
+    pub fn provider(self) -> ManagedDekProvider {
+        match self {
+            Self::LegacySseS3Local => ManagedDekProvider::LocalSseS3,
+            Self::MinioSseS3KeyValue => ManagedDekProvider::MinioKeyValue,
+            Self::LegacySseS3Kms | Self::LegacySseKms | Self::MinioSseS3Kms | Self::MinioSseKmsKms => ManagedDekProvider::Kms,
+        }
+    }
+
+    pub fn uses_object_key(self) -> bool {
+        matches!(self, Self::MinioSseS3KeyValue | Self::MinioSseS3Kms | Self::MinioSseKmsKms)
+    }
+}
+
+#[derive(Debug, Clone, Error, PartialEq, Eq)]
+pub enum PersistedEncryptionError {
+    #[error("conflicting values for encryption metadata {field}")]
+    ConflictingValue { field: &'static str },
+    #[error("conflicting MinIO managed SSE format markers")]
+    ConflictingMinioMarkers,
+    #[error("incomplete managed SSE metadata: missing {field}")]
+    MissingField { field: &'static str },
+    #[error("unsupported stored server-side encryption {algorithm}")]
+    UnsupportedSseAlgorithm { algorithm: String },
+    #[error("unsupported MinIO seal algorithm {algorithm}")]
+    UnsupportedSealAlgorithm { algorithm: String },
+    #[error("stored SSE algorithm conflicts with the persisted encryption format")]
+    SchemeConflict,
+    #[error("conflicting encrypted data-key metadata")]
+    ConflictingEncryptedDataKey,
+    #[error("conflicting KMS key-id metadata")]
+    ConflictingKmsKeyId,
+    #[error("invalid MinIO static KMS configuration: {reason}")]
+    InvalidMinioStaticKmsConfiguration { reason: String },
+    #[error("invalid MinIO static KMS ciphertext: {reason}")]
+    InvalidMinioStaticKmsCiphertext { reason: String },
+    #[error("unrecognized legacy managed SSE encrypted data-key format")]
+    UnknownLegacyEnvelope,
+}
+
+pub fn classify_persisted_managed_encryption(
+    metadata: &HashMap<String, String>,
+    encrypted_dek: &[u8],
+) -> Result<PersistedManagedEncryption, PersistedEncryptionError> {
+    let public_algorithm = consistent(metadata, AMZ_SERVER_SIDE_ENCRYPTION)?;
+    if let Some(algorithm) = public_algorithm
+        && !matches!(algorithm, DEFAULT_SSE_ALGORITHM | SSE_KMS_ALGORITHM)
+    {
+        return Err(PersistedEncryptionError::UnsupportedSseAlgorithm {
+            algorithm: algorithm.to_string(),
+        });
+    }
+
+    let s3_sealed_key = consistent(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)?;
+    let kms_sealed_key = consistent(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)?;
+    if s3_sealed_key.is_some() && kms_sealed_key.is_some() {
+        return Err(PersistedEncryptionError::ConflictingMinioMarkers);
+    }
+
+    if s3_sealed_key.is_some() || kms_sealed_key.is_some() {
+        if s3_sealed_key.is_some() {
+            require_non_empty(s3_sealed_key, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)?;
+        }
+        if kms_sealed_key.is_some() {
+            require_non_empty(kms_sealed_key, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)?;
+        }
+        require_non_empty(
+            consistent(metadata, MINIO_INTERNAL_ENCRYPTION_IV_HEADER)?,
+            MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
+        )?;
+        let seal_algorithm = require_non_empty(
+            consistent(metadata, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER)?,
+            MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
+        )?;
+        if seal_algorithm != MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM {
+            return Err(PersistedEncryptionError::UnsupportedSealAlgorithm {
+                algorithm: seal_algorithm.to_string(),
+            });
+        }
+        let kms_key_id = consistent(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER)?;
+        let encrypted_data_key = consistent(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER)?;
+        let has_kms_pair = match (kms_key_id, encrypted_data_key) {
+            (Some(key_id), Some(data_key)) => {
+                require_non_empty(Some(key_id), MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER)?;
+                require_non_empty(Some(data_key), MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER)?;
+                true
+            }
+            (None, None) => false,
+            (None, Some(_)) => {
+                return Err(PersistedEncryptionError::MissingField {
+                    field: MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
+                });
+            }
+            (Some(_), None) => {
+                return Err(PersistedEncryptionError::MissingField {
+                    field: MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER,
+                });
+            }
+        };
+        validate_encrypted_key_alias(metadata)?;
+        validate_kms_key_id_alias(metadata)?;
+
+        return match (s3_sealed_key, kms_sealed_key, public_algorithm) {
+            (Some(_), None, Some(SSE_KMS_ALGORITHM)) | (None, Some(_), Some(DEFAULT_SSE_ALGORITHM)) => {
+                Err(PersistedEncryptionError::SchemeConflict)
+            }
+            (Some(_), None, _) if !has_kms_pair => Ok(PersistedManagedEncryption::MinioSseS3KeyValue),
+            (Some(_), None, _) => Ok(PersistedManagedEncryption::MinioSseS3Kms),
+            (None, Some(_), _) if !has_kms_pair => Err(PersistedEncryptionError::MissingField {
+                field: MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
+            }),
+            (None, Some(_), _) => Ok(PersistedManagedEncryption::MinioSseKmsKms),
+            _ => Err(PersistedEncryptionError::ConflictingMinioMarkers),
+        };
+    }
+
+    if consistent(metadata, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER)?.is_some()
+        || consistent(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER)?.is_some()
+    {
+        return Err(PersistedEncryptionError::MissingField {
+            field: MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
+        });
+    }
+
+    require_non_empty(consistent(metadata, INTERNAL_ENCRYPTION_KEY_HEADER)?, INTERNAL_ENCRYPTION_KEY_HEADER)?;
+    require_non_empty(consistent(metadata, INTERNAL_ENCRYPTION_IV_HEADER)?, INTERNAL_ENCRYPTION_IV_HEADER)?;
+
+    match public_algorithm {
+        Some(SSE_KMS_ALGORITHM) if is_local_sse_s3_envelope(encrypted_dek) => Err(PersistedEncryptionError::SchemeConflict),
+        Some(SSE_KMS_ALGORITHM) => Ok(PersistedManagedEncryption::LegacySseKms),
+        Some(DEFAULT_SSE_ALGORITHM) | None if rustfs_kms::is_data_key_envelope(encrypted_dek) => {
+            // RUSTFS_COMPAT_TODO(rustfs-5063): older SSE-S3 objects may contain KMS-wrapped DEKs. Remove after every referenced legacy DEK has been rewrapped with the local SSE-S3 provider.
+            Ok(PersistedManagedEncryption::LegacySseS3Kms)
+        }
+        Some(DEFAULT_SSE_ALGORITHM) | None if is_local_sse_s3_envelope(encrypted_dek) => {
+            Ok(PersistedManagedEncryption::LegacySseS3Local)
+        }
+        Some(DEFAULT_SSE_ALGORITHM) | None => Err(PersistedEncryptionError::UnknownLegacyEnvelope),
+        Some(_) => unreachable!("unsupported algorithms return before format classification"),
+    }
+}
+
+fn consistent<'a>(
+    metadata: &'a HashMap<String, String>,
+    field: &'static str,
+) -> Result<Option<&'a str>, PersistedEncryptionError> {
+    get_consistent_metadata_value(metadata, field).map_err(|_| PersistedEncryptionError::ConflictingValue { field })
+}
+
+fn require_non_empty<'a>(value: Option<&'a str>, field: &'static str) -> Result<&'a str, PersistedEncryptionError> {
+    value
+        .filter(|value| !value.is_empty())
+        .ok_or(PersistedEncryptionError::MissingField { field })
+}
+
+fn validate_encrypted_key_alias(metadata: &HashMap<String, String>) -> Result<(), PersistedEncryptionError> {
+    let Some(rustfs_key) = consistent(metadata, INTERNAL_ENCRYPTION_KEY_HEADER)? else {
+        return Ok(());
+    };
+    let minio_key = require_non_empty(
+        consistent(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER)?,
+        MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER,
+    )?;
+    if rustfs_key != minio_key {
+        return Err(PersistedEncryptionError::ConflictingEncryptedDataKey);
+    }
+    Ok(())
+}
+
+fn validate_kms_key_id_alias(metadata: &HashMap<String, String>) -> Result<(), PersistedEncryptionError> {
+    let values = [
+        consistent(metadata, INTERNAL_ENCRYPTION_KEY_ID_HEADER)?,
+        consistent(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER)?,
+        consistent(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_KEY_ID)?,
+    ];
+    let mut present = values.into_iter().flatten().filter(|value| !value.is_empty());
+    let Some(first) = present.next() else {
+        return Ok(());
+    };
+    if present.any(|value| value != first) {
+        return Err(PersistedEncryptionError::ConflictingKmsKeyId);
+    }
+    Ok(())
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct MinioStaticKmsJsonCiphertext {
+    #[serde(rename = "aead")]
+    algorithm: String,
+    id: Option<String>,
+    iv: String,
+    nonce: String,
+    bytes: String,
+}
+
+#[derive(Clone, Copy)]
+enum MinioStaticKmsAlgorithm {
+    Aes256,
+    ChaCha20,
+}
+
+#[derive(Clone)]
+struct ConfiguredMinioStaticKmsKey {
+    id: String,
+    material: Zeroizing<[u8; 32]>,
+}
+
+struct ParsedMinioStaticKmsCiphertext {
+    ciphertext: Vec<u8>,
+    iv: [u8; 16],
+    nonce: [u8; 12],
+    algorithm: MinioStaticKmsAlgorithm,
+}
+
+pub fn decrypt_minio_static_kms_dek(
+    kms_key_id: &str,
+    encrypted_dek: &[u8],
+    context: &HashMap<String, String>,
+) -> Result<Option<[u8; 32]>, PersistedEncryptionError> {
+    let Some(configured_key) = minio_static_kms_key()? else {
+        return Ok(None);
+    };
+    if configured_key.id != kms_key_id {
+        return Ok(None);
+    }
+
+    let parsed = parse_minio_static_kms_ciphertext(encrypted_dek)?;
+    let ParsedMinioStaticKmsCiphertext {
+        ciphertext,
+        iv,
+        nonce,
+        algorithm,
+    } = parsed;
+    let master_key = configured_key.material;
+    let sealing_key: Zeroizing<[u8; 32]> = Zeroizing::new(match algorithm {
+        MinioStaticKmsAlgorithm::Aes256 => {
+            let mut mac = HmacSha256::new_from_slice(master_key.as_slice()).map_err(|err| {
+                PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
+                    reason: format!("invalid HMAC key: {err}"),
+                }
+            })?;
+            mac.update(&iv);
+            mac.finalize().into_bytes().into()
+        }
+        MinioStaticKmsAlgorithm::ChaCha20 => chacha20::hchacha::<chacha20::R20>((&*master_key).into(), (&iv).into()).into(),
+    });
+    let associated_data = marshal_minio_kms_context(context);
+    let plaintext = Zeroizing::new(
+        match algorithm {
+            MinioStaticKmsAlgorithm::Aes256 => Aes256Gcm::new_from_slice(sealing_key.as_slice())
+                .map_err(|err| PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
+                    reason: format!("invalid AES sealing key: {err}"),
+                })?
+                .decrypt(
+                    &Nonce::from(nonce),
+                    Payload {
+                        msg: &ciphertext,
+                        aad: &associated_data,
+                    },
+                ),
+            MinioStaticKmsAlgorithm::ChaCha20 => ChaCha20Poly1305::new_from_slice(sealing_key.as_slice())
+                .map_err(|err| PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
+                    reason: format!("invalid ChaCha20 sealing key: {err}"),
+                })?
+                .decrypt(
+                    &chacha20poly1305::Nonce::from(nonce),
+                    Payload {
+                        msg: &ciphertext,
+                        aad: &associated_data,
+                    },
+                ),
+        }
+        .map_err(|_| PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
+            reason: "AEAD authentication failed".to_string(),
+        })?,
+    );
+    plaintext
+        .as_slice()
+        .try_into()
+        .map(Some)
+        .map_err(|_| PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
+            reason: "plaintext data key must be 32 bytes".to_string(),
+        })
+}
+
+fn minio_static_kms_key() -> Result<Option<ConfiguredMinioStaticKmsKey>, PersistedEncryptionError> {
+    #[cfg(not(any(test, debug_assertions)))]
+    {
+        static CONFIG: OnceLock<Result<Option<ConfiguredMinioStaticKmsKey>, PersistedEncryptionError>> = OnceLock::new();
+        return CONFIG.get_or_init(parse_minio_static_kms_key).clone();
+    }
+
+    #[cfg(any(test, debug_assertions))]
+    parse_minio_static_kms_key()
+}
+
+fn parse_minio_static_kms_key() -> Result<Option<ConfiguredMinioStaticKmsKey>, PersistedEncryptionError> {
+    let Some(value) = std::env::var_os(MINIO_STATIC_KMS_KEY_ENV) else {
+        return Ok(None);
+    };
+    let value =
+        Zeroizing::new(
+            value
+                .into_string()
+                .map_err(|_| PersistedEncryptionError::InvalidMinioStaticKmsConfiguration {
+                    reason: format!("{MINIO_STATIC_KMS_KEY_ENV} must be valid UTF-8"),
+                })?,
+        );
+    let (key_id, encoded_key) =
+        value
+            .split_once(':')
+            .ok_or_else(|| PersistedEncryptionError::InvalidMinioStaticKmsConfiguration {
+                reason: format!("{MINIO_STATIC_KMS_KEY_ENV} must use <key-id>:<base64-key>"),
+            })?;
+    if key_id.is_empty() {
+        return Err(PersistedEncryptionError::InvalidMinioStaticKmsConfiguration {
+            reason: "key ID must not be empty".to_string(),
+        });
+    }
+    let decoded_key = Zeroizing::new(BASE64_STANDARD.decode(encoded_key).map_err(|_| {
+        PersistedEncryptionError::InvalidMinioStaticKmsConfiguration {
+            reason: "key material must be valid Base64".to_string(),
+        }
+    })?);
+    let key: [u8; 32] =
+        decoded_key
+            .as_slice()
+            .try_into()
+            .map_err(|_| PersistedEncryptionError::InvalidMinioStaticKmsConfiguration {
+                reason: "key material must decode to exactly 32 bytes".to_string(),
+            })?;
+    if key == [0u8; 32] {
+        return Err(PersistedEncryptionError::InvalidMinioStaticKmsConfiguration {
+            reason: "key material must not be all zero".to_string(),
+        });
+    }
+    Ok(Some(ConfiguredMinioStaticKmsKey {
+        id: key_id.to_string(),
+        material: Zeroizing::new(key),
+    }))
+}
+
+fn parse_minio_static_kms_ciphertext(encrypted_dek: &[u8]) -> Result<ParsedMinioStaticKmsCiphertext, PersistedEncryptionError> {
+    if encrypted_dek.first() == Some(&b'{') && encrypted_dek.last() == Some(&b'}') {
+        let envelope: MinioStaticKmsJsonCiphertext =
+            serde_json::from_slice(encrypted_dek).map_err(|err| PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
+                reason: format!("invalid legacy JSON: {err}"),
+            })?;
+        let _ = envelope.id;
+        let algorithm = match envelope.algorithm.as_str() {
+            "AES-256-GCM-HMAC-SHA-256" => MinioStaticKmsAlgorithm::Aes256,
+            "ChaCha20Poly1305" => MinioStaticKmsAlgorithm::ChaCha20,
+            algorithm => {
+                return Err(PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
+                    reason: format!("unsupported algorithm {algorithm}"),
+                });
+            }
+        };
+        return Ok(ParsedMinioStaticKmsCiphertext {
+            ciphertext: decode_minio_static_kms_field("bytes", &envelope.bytes)?,
+            iv: decode_minio_static_kms_array("iv", &envelope.iv)?,
+            nonce: decode_minio_static_kms_array("nonce", &envelope.nonce)?,
+            algorithm,
+        });
+    }
+
+    if encrypted_dek.len() <= MINIO_STATIC_KMS_RANDOM_SIZE {
+        return Err(PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
+            reason: "binary ciphertext is too short".to_string(),
+        });
+    }
+    let split_at = encrypted_dek.len() - MINIO_STATIC_KMS_RANDOM_SIZE;
+    let (ciphertext, random) = encrypted_dek.split_at(split_at);
+    Ok(ParsedMinioStaticKmsCiphertext {
+        ciphertext: ciphertext.to_vec(),
+        iv: random[..MINIO_STATIC_KMS_IV_SIZE].try_into().map_err(|_| {
+            PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
+                reason: "invalid binary IV length".to_string(),
+            }
+        })?,
+        nonce: random[MINIO_STATIC_KMS_IV_SIZE..MINIO_STATIC_KMS_IV_SIZE + MINIO_STATIC_KMS_NONCE_SIZE]
+            .try_into()
+            .map_err(|_| PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
+                reason: "invalid binary nonce length".to_string(),
+            })?,
+        algorithm: MinioStaticKmsAlgorithm::Aes256,
+    })
+}
+
+fn decode_minio_static_kms_field(field: &'static str, value: &str) -> Result<Vec<u8>, PersistedEncryptionError> {
+    BASE64_STANDARD
+        .decode(value)
+        .map_err(|_| PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
+            reason: format!("{field} must be valid Base64"),
+        })
+}
+
+fn decode_minio_static_kms_array<const N: usize>(field: &'static str, value: &str) -> Result<[u8; N], PersistedEncryptionError> {
+    decode_minio_static_kms_field(field, value)?.try_into().map_err(|_| {
+        PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
+            reason: format!("{field} must decode to exactly {N} bytes"),
+        }
+    })
+}
+
+fn marshal_minio_kms_context(context: &HashMap<String, String>) -> Vec<u8> {
+    let mut entries: Vec<_> = context.iter().collect();
+    entries.sort_by_key(|(key, _)| *key);
+    let mut json = String::from("{");
+    for (index, (key, value)) in entries.into_iter().enumerate() {
+        if index > 0 {
+            json.push(',');
+        }
+        push_minio_json_string(&mut json, key);
+        json.push(':');
+        push_minio_json_string(&mut json, value);
+    }
+    json.push('}');
+    json.into_bytes()
+}
+
+fn push_minio_json_string(output: &mut String, value: &str) {
+    output.push('"');
+    for character in value.chars() {
+        match character {
+            '"' => output.push_str("\\\""),
+            '\\' => output.push_str("\\\\"),
+            '\n' => output.push_str("\\n"),
+            '\r' => output.push_str("\\r"),
+            '\t' => output.push_str("\\t"),
+            '<' => output.push_str("\\u003c"),
+            '>' => output.push_str("\\u003e"),
+            '&' => output.push_str("\\u0026"),
+            '\u{2028}' => output.push_str("\\u2028"),
+            '\u{2029}' => output.push_str("\\u2029"),
+            character if character <= '\u{1f}' => {
+                const HEX: &[u8; 16] = b"0123456789abcdef";
+                let byte = character as u8;
+                output.push_str("\\u00");
+                output.push(HEX[(byte >> 4) as usize] as char);
+                output.push(HEX[(byte & 0x0f) as usize] as char);
+            }
+            character => output.push(character),
+        }
+    }
+    output.push('"');
+}
+
+fn is_local_sse_s3_envelope(encrypted_dek: &[u8]) -> bool {
+    let Ok(encoded) = std::str::from_utf8(encrypted_dek) else {
+        return false;
+    };
+    let Some((nonce, ciphertext)) = encoded.split_once(':') else {
+        return false;
+    };
+    use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
+    BASE64_STANDARD.decode(nonce).is_ok_and(|nonce| nonce.len() == 12)
+        && BASE64_STANDARD
+            .decode(ciphertext)
+            .is_ok_and(|ciphertext| ciphertext.len() == 48)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
+
+    fn local_envelope() -> Vec<u8> {
+        format!("{}:{}", BASE64_STANDARD.encode([1u8; 12]), BASE64_STANDARD.encode([2u8; 48])).into_bytes()
+    }
+
+    fn kms_envelope() -> Vec<u8> {
+        serde_json::to_vec(&serde_json::json!({
+            "key_id": "data-key",
+            "master_key_id": "master-key",
+            "key_spec": "AES_256",
+            "encrypted_key": [1, 2, 3, 4],
+            "nonce": [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
+            "encryption_context": {},
+            "created_at": "2024-01-01T00:00:00+00:00"
+        }))
+        .expect("serialize KMS envelope fixture")
+    }
+
+    fn legacy_metadata(algorithm: Option<&str>, encrypted_dek: &[u8]) -> HashMap<String, String> {
+        let mut metadata = HashMap::from([
+            (INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode(encrypted_dek)),
+            (INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([3u8; 12])),
+        ]);
+        if let Some(algorithm) = algorithm {
+            metadata.insert(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), algorithm.to_string());
+        }
+        metadata
+    }
+
+    fn minio_metadata(kms: bool, encrypted_dek: &[u8]) -> HashMap<String, String> {
+        let object_key_header = if kms {
+            MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER
+        } else {
+            MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER
+        };
+        let mut metadata = HashMap::from([
+            (object_key_header.to_string(), BASE64_STANDARD.encode([4u8; 64])),
+            (
+                MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(),
+                BASE64_STANDARD.encode(encrypted_dek),
+            ),
+            (MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()),
+            (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([5u8; 32])),
+            (
+                MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
+                MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
+            ),
+        ]);
+        metadata.insert(
+            AMZ_SERVER_SIDE_ENCRYPTION.to_string(),
+            if kms { SSE_KMS_ALGORITHM } else { DEFAULT_SSE_ALGORITHM }.to_string(),
+        );
+        metadata
+    }
+
+    #[test]
+    fn classifies_released_legacy_formats() {
+        let local = local_envelope();
+        let kms = kms_envelope();
+
+        assert_eq!(
+            classify_persisted_managed_encryption(&legacy_metadata(Some(DEFAULT_SSE_ALGORITHM), &local), &local)
+                .expect("classify local Direct format"),
+            PersistedManagedEncryption::LegacySseS3Local
+        );
+        assert_eq!(
+            classify_persisted_managed_encryption(&legacy_metadata(Some(DEFAULT_SSE_ALGORITHM), &kms), &kms)
+                .expect("classify legacy SSE-S3 KMS envelope"),
+            PersistedManagedEncryption::LegacySseS3Kms
+        );
+        assert_eq!(
+            classify_persisted_managed_encryption(
+                &legacy_metadata(Some(SSE_KMS_ALGORITHM), b"opaque-kms-key"),
+                b"opaque-kms-key"
+            )
+            .expect("classify opaque SSE-KMS envelope"),
+            PersistedManagedEncryption::LegacySseKms
+        );
+    }
+
+    #[test]
+    fn minio_markers_override_envelope_shape() {
+        let local = local_envelope();
+        let kms = kms_envelope();
+
+        assert_eq!(
+            classify_persisted_managed_encryption(&minio_metadata(false, &kms), &kms)
+                .expect("SSE-S3 marker selects MinIO SSE-S3"),
+            PersistedManagedEncryption::MinioSseS3Kms
+        );
+        assert_eq!(
+            classify_persisted_managed_encryption(&minio_metadata(true, &local), &local)
+                .expect("SSE-KMS marker selects MinIO SSE-KMS"),
+            PersistedManagedEncryption::MinioSseKmsKms
+        );
+    }
+
+    #[test]
+    fn accepts_minio_sse_s3_key_value_only_when_kms_pair_is_absent() {
+        let local = local_envelope();
+        let mut metadata = minio_metadata(false, &local);
+        metadata.remove(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER);
+        metadata.remove(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER);
+
+        assert_eq!(
+            classify_persisted_managed_encryption(&metadata, &[]).expect("classify MinIO SSE-S3 K/V metadata"),
+            PersistedManagedEncryption::MinioSseS3KeyValue
+        );
+
+        metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), String::new());
+        metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(), String::new());
+        assert_eq!(
+            classify_persisted_managed_encryption(&metadata, &[]),
+            Err(PersistedEncryptionError::MissingField {
+                field: MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER
+            })
+        );
+    }
+
+    #[test]
+    fn rejects_minio_sse_kms_without_kms_pair() {
+        let local = local_envelope();
+        let mut metadata = minio_metadata(true, &local);
+        metadata.remove(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER);
+        metadata.remove(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER);
+
+        assert_eq!(
+            classify_persisted_managed_encryption(&metadata, &[]),
+            Err(PersistedEncryptionError::MissingField {
+                field: MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER
+            })
+        );
+    }
+
+    #[test]
+    fn persisted_minio_provider_does_not_depend_on_runtime_kms_availability() {
+        let static_kms = br#"{"aead":"AES-256-GCM-HMAC-SHA-256","iv":"AQEBAQEBAQEBAQEBAQEBAQ==","nonce":"AgICAgICAgICAgIC","bytes":"AwMDAw=="}"#;
+        let opaque_kms = b"opaque-external-kms-ciphertext";
+
+        assert_eq!(
+            classify_persisted_managed_encryption(&minio_metadata(false, static_kms), static_kms)
+                .expect("classify MinIO static KMS envelope")
+                .provider(),
+            ManagedDekProvider::Kms
+        );
+        assert_eq!(
+            classify_persisted_managed_encryption(&minio_metadata(true, opaque_kms), opaque_kms)
+                .expect("classify external KMS ciphertext")
+                .provider(),
+            ManagedDekProvider::Kms
+        );
+
+        let mut key_value = minio_metadata(false, opaque_kms);
+        key_value.remove(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER);
+        key_value.remove(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER);
+        assert_eq!(
+            classify_persisted_managed_encryption(&key_value, &[])
+                .expect("classify historical K/V provider")
+                .provider(),
+            ManagedDekProvider::MinioKeyValue
+        );
+    }
+
+    #[test]
+    fn decrypts_minio_static_kms_binary_aes_ciphertext() {
+        let master_key = [0x31; 32];
+        let plaintext_key = [0x52; 32];
+        let iv = [0x63; 16];
+        let nonce = [0x74; 12];
+        let context = HashMap::from([("bucket".to_string(), "bucket/object".to_string())]);
+        let mut mac = HmacSha256::new_from_slice(&master_key).expect("valid master key");
+        mac.update(&iv);
+        let sealing_key = mac.finalize().into_bytes();
+        let ciphertext = Aes256Gcm::new_from_slice(sealing_key.as_slice())
+            .expect("valid sealing key")
+            .encrypt(
+                &Nonce::from(nonce),
+                Payload {
+                    msg: &plaintext_key,
+                    aad: &marshal_minio_kms_context(&context),
+                },
+            )
+            .expect("encrypt fixture");
+        let mut envelope = ciphertext;
+        envelope.extend_from_slice(&iv);
+        envelope.extend_from_slice(&nonce);
+        let configured_key = format!("minio-key:{}", BASE64_STANDARD.encode(master_key));
+
+        temp_env::with_var(MINIO_STATIC_KMS_KEY_ENV, Some(configured_key), || {
+            assert_eq!(
+                decrypt_minio_static_kms_dek("minio-key", &envelope, &context).expect("decrypt binary ciphertext"),
+                Some(plaintext_key)
+            );
+            assert_eq!(
+                decrypt_minio_static_kms_dek("external-key", &envelope, &context)
+                    .expect("different key ID must remain external KMS"),
+                None
+            );
+        });
+    }
+
+    #[test]
+    fn decrypts_official_minio_legacy_chacha20_ciphertext() {
+        let configured_key = "my-key:eEm+JI9/q4JhH8QwKvf3LKo4DEBl6QbfvAl1CAbMIv8=";
+        let ciphertext = br#"{"aead":"ChaCha20Poly1305","iv":"JbI+vwvYww1lCb5VpkAFuQ==","nonce":"ARjIjJxBSD541Gz8","bytes":"KCbEc2sA0TLvA7aWTWa23AdccVfJMpOxwgG8hm+4PaNrxYfy1xFWZg2gEenVrOgv"}"#;
+        let expected: [u8; 32] = BASE64_STANDARD
+            .decode("zmS7NrG765UZ0ZN85oPjybelxqVvpz01vxsSpOISy2M=")
+            .expect("decode official plaintext")
+            .try_into()
+            .expect("official plaintext is 32 bytes");
+
+        temp_env::with_var(MINIO_STATIC_KMS_KEY_ENV, Some(configured_key), || {
+            assert_eq!(
+                decrypt_minio_static_kms_dek("my-key", ciphertext, &HashMap::new()).expect("decrypt official MinIO ciphertext"),
+                Some(expected)
+            );
+        });
+    }
+
+    #[test]
+    fn rejects_malformed_config_and_unknown_legacy_fields() {
+        temp_env::with_var(MINIO_STATIC_KMS_KEY_ENV, Some("missing-separator"), || {
+            assert!(matches!(
+                decrypt_minio_static_kms_dek("my-key", b"ciphertext", &HashMap::new()),
+                Err(PersistedEncryptionError::InvalidMinioStaticKmsConfiguration { .. })
+            ));
+        });
+
+        let configured_key = format!("my-key:{}", BASE64_STANDARD.encode([0x31; 32]));
+        let ciphertext =
+            br#"{"aead":"AES-256-GCM-HMAC-SHA-256","iv":"Y2NjY2NjY2NjY2NjY2NjYw==","nonce":"dHR0dHR0dHR0dHR0","bytes":"AA==","extra":true}"#;
+        temp_env::with_var(MINIO_STATIC_KMS_KEY_ENV, Some(configured_key), || {
+            assert!(matches!(
+                decrypt_minio_static_kms_dek("my-key", ciphertext, &HashMap::new()),
+                Err(PersistedEncryptionError::InvalidMinioStaticKmsCiphertext { .. })
+            ));
+        });
+    }
+
+    #[test]
+    fn rejects_all_zero_minio_static_kms_key() {
+        let configured_key = format!("my-key:{}", BASE64_STANDARD.encode([0u8; 32]));
+        temp_env::with_var(MINIO_STATIC_KMS_KEY_ENV, Some(configured_key), || {
+            assert!(matches!(
+                decrypt_minio_static_kms_dek("my-key", b"ciphertext", &HashMap::new()),
+                Err(PersistedEncryptionError::InvalidMinioStaticKmsConfiguration { .. })
+            ));
+        });
+    }
+
+    #[test]
+    fn marshals_minio_kms_context_with_go_json_escaping() {
+        let context = HashMap::from([
+            ("z".to_string(), "line\n".to_string()),
+            ("<\u{8}".to_string(), "&\u{2028}".to_string()),
+        ]);
+
+        assert_eq!(marshal_minio_kms_context(&context), br#"{"\u003c\u0008":"\u0026\u2028","z":"line\n"}"#);
+    }
+
+    #[test]
+    fn rejects_conflicting_or_partial_minio_metadata() {
+        let local = local_envelope();
+        let mut both = minio_metadata(false, &local);
+        both.insert(
+            MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(),
+            BASE64_STANDARD.encode([6u8; 64]),
+        );
+        assert_eq!(
+            classify_persisted_managed_encryption(&both, &local),
+            Err(PersistedEncryptionError::ConflictingMinioMarkers)
+        );
+
+        let mut wrong_scheme = minio_metadata(true, &local);
+        wrong_scheme.insert(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), DEFAULT_SSE_ALGORITHM.to_string());
+        assert_eq!(
+            classify_persisted_managed_encryption(&wrong_scheme, &local),
+            Err(PersistedEncryptionError::SchemeConflict)
+        );
+
+        let mut partial = minio_metadata(false, &local);
+        partial.remove(MINIO_INTERNAL_ENCRYPTION_IV_HEADER);
+        assert_eq!(
+            classify_persisted_managed_encryption(&partial, &local),
+            Err(PersistedEncryptionError::MissingField {
+                field: MINIO_INTERNAL_ENCRYPTION_IV_HEADER
+            })
+        );
+
+        let mut missing_key_id = minio_metadata(false, &local);
+        missing_key_id.remove(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER);
+        assert_eq!(
+            classify_persisted_managed_encryption(&missing_key_id, &local),
+            Err(PersistedEncryptionError::MissingField {
+                field: MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER
+            })
+        );
+
+        let mut missing_data_key = minio_metadata(false, &local);
+        missing_data_key.remove(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER);
+        assert_eq!(
+            classify_persisted_managed_encryption(&missing_data_key, &local),
+            Err(PersistedEncryptionError::MissingField {
+                field: MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER
+            })
+        );
+
+        let mut unknown_algorithm = minio_metadata(false, &local);
+        unknown_algorithm.insert(
+            MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
+            "future-seal-algorithm".to_string(),
+        );
+        assert_eq!(
+            classify_persisted_managed_encryption(&unknown_algorithm, &local),
+            Err(PersistedEncryptionError::UnsupportedSealAlgorithm {
+                algorithm: "future-seal-algorithm".to_string()
+            })
+        );
+
+        for field in [
+            MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
+            MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER,
+        ] {
+            let mut empty = minio_metadata(false, &local);
+            empty.insert(field.to_string(), String::new());
+            assert_eq!(
+                classify_persisted_managed_encryption(&empty, &local),
+                Err(PersistedEncryptionError::MissingField { field })
+            );
+        }
+
+        for field in [
+            MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
+            MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER,
+        ] {
+            let metadata = HashMap::from([(field.to_string(), "orphaned".to_string())]);
+            assert_eq!(
+                classify_persisted_managed_encryption(&metadata, &local),
+                Err(PersistedEncryptionError::MissingField {
+                    field: MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER
+                })
+            );
+        }
+    }
+
+    #[test]
+    fn rejects_unknown_legacy_envelope_and_conflicting_alias() {
+        let unknown = b"not-an-envelope";
+        assert_eq!(
+            classify_persisted_managed_encryption(&legacy_metadata(Some(DEFAULT_SSE_ALGORITHM), unknown), unknown),
+            Err(PersistedEncryptionError::UnknownLegacyEnvelope)
+        );
+
+        let local = local_envelope();
+        let mut conflicting = minio_metadata(false, &local);
+        conflicting.insert(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode(b"different"));
+        assert_eq!(
+            classify_persisted_managed_encryption(&conflicting, &local),
+            Err(PersistedEncryptionError::ConflictingEncryptedDataKey)
+        );
+
+        let mut conflicting_key_id = minio_metadata(false, &local);
+        conflicting_key_id.insert(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "different-key".to_string());
+        assert_eq!(
+            classify_persisted_managed_encryption(&conflicting_key_id, &local),
+            Err(PersistedEncryptionError::ConflictingKmsKeyId)
+        );
+
+        assert_eq!(
+            classify_persisted_managed_encryption(&legacy_metadata(Some(SSE_KMS_ALGORITHM), &local), &local),
+            Err(PersistedEncryptionError::SchemeConflict)
+        );
+    }
+
+    #[test]
+    fn accepts_case_insensitive_minio_markers_and_rejects_conflicting_duplicates() {
+        let local = local_envelope();
+        let mut metadata = minio_metadata(false, &local);
+        let sealed_key = metadata
+            .remove(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)
+            .expect("MinIO fixture contains the sealed-key marker");
+        metadata.insert(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_ascii_lowercase(), sealed_key);
+
+        assert_eq!(
+            classify_persisted_managed_encryption(&metadata, &local).expect("classify mixed-case MinIO metadata"),
+            PersistedManagedEncryption::MinioSseS3Kms
+        );
+
+        metadata.insert(
+            MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(),
+            BASE64_STANDARD.encode([0x77u8; 64]),
+        );
+        assert_eq!(
+            classify_persisted_managed_encryption(&metadata, &local),
+            Err(PersistedEncryptionError::ConflictingValue {
+                field: MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER
+            })
+        );
+    }
+}
@@ -537,7 +537,8 @@ mod tests {
            transition_transaction::{
                TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionRemoteVersion,
                TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit,
-                TransitionTransactionState, recover_transition_transaction_records, save_transition_transaction_record,
+                TransitionTransactionState, load_transition_transaction_record, recover_transition_transaction_records,
+                save_transition_transaction_record,
            },
        },
        client::transition_api::ReaderImpl,
@@ -545,8 +546,8 @@ mod tests {
        disk::RUSTFS_META_BUCKET,
        runtime::{global::set_object_store_resolver, sources as runtime_sources},
        services::tier::{
-            test_util::{MockWarmBackend, TransitionCleanupStoreBarrier, register_mock_tier},
-            tier::TierConfigMgr,
+            test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
+            tier::{TIER_CONFIG_FILE, TierConfigMgr},
            tier_mutation_intent::{
                TIER_MUTATION_INTENT_RECORD_PREFIX, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState,
                TierMutationIntentTarget, advance_tier_mutation_intent_record_idempotent, delete_tier_mutation_intent_record,
@@ -554,7 +555,7 @@ mod tests {
                save_tier_mutation_intent_record, save_tier_mutation_intent_record_if_current,
            },
            tier_mutation_peer::{TierMutationPeerError, TierMutationPeerState, handle_tier_mutation_peer_request},
-            warm_backend::WarmBackend,
+            warm_backend::{TransitionCandidateProbe, WarmBackend},
        },
        storage_api_contracts::{
            bucket::{BucketOperations as _, MakeBucketOptions},
@@ -641,6 +642,7 @@ mod tests {
                stream: Box::new(Cursor::new(self.read_payload.clone())),
                object_info: self.object_info(bucket, object, self.read_payload.len()),
                buffered_body: None,
+                resolved_sse: None,
                body_source: Default::default(),
            })
        }
@@ -1589,6 +1591,47 @@ mod tests {
        assert_eq!(loaded.state, TierMutationIntentState::Committed);
        assert_eq!(loaded.committed_config_etag.as_deref(), Some("new-etag"));

+        store
+            .tier_config_mgr()
+            .read()
+            .await
+            .save_tiering_config(store.clone())
+            .await
+            .expect("tier config should persist for cleaned intent commit proof");
+        let tier_config_info = store
+            .get_object_info(
+                RUSTFS_META_BUCKET,
+                &format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
+                &ObjectOptions::default(),
+            )
+            .await
+            .expect("tier config object info should load");
+        let tier_config_etag = tier_config_info.etag.expect("tier config should carry an ETag");
+        delete_tier_mutation_intent_record(store.clone(), mutation_id)
+            .await
+            .expect("committed peer intent cleanup should persist");
+        let cleaned_commit_retry = handle_tier_mutation_peer_request(
+            store.clone(),
+            TIER_MUTATION_RPC_PROTOCOL_VERSION,
+            TierMutationRpcPhase::Commit,
+            mutation_id,
+            tier_config_etag.as_bytes(),
+        )
+        .await
+        .expect("commit retry after durable cleanup should be idempotently terminal");
+        assert!(!cleaned_commit_retry.applied);
+        assert_eq!(cleaned_commit_retry.state, TierMutationPeerState::Committed);
+        let mismatched_cleaned_commit = handle_tier_mutation_peer_request(
+            store.clone(),
+            TIER_MUTATION_RPC_PROTOCOL_VERSION,
+            TierMutationRpcPhase::Commit,
+            mutation_id,
+            b"not-the-current-etag",
+        )
+        .await
+        .expect_err("missing intent without a matching committed config ETag must fail closed");
+        assert!(matches!(mismatched_cleaned_commit, TierMutationPeerError::Store(Error::ConfigNotFound)));
+
        let abort_id = uuid::Uuid::new_v4();
        let abort_intent = tier_mutation_peer_test_intent(abort_id, "COLD-B", [4; 32]);
        let abort_prepare_payload = abort_intent.encode().expect("abort prepare intent should encode");
@@ -1808,7 +1851,7 @@ mod tests {
        let tier_name = "CROSSCTXA";
        let backend = register_mock_tier(&ctx_a.tier_config_mgr(), tier_name).await;
        backend.set_put_remote_version(Some(uuid::Uuid::new_v4().to_string())).await;
-        backend.set_reject_non_empty_remote_versions(true);
+        backend.reject_next_non_empty_remote_version_validation();
        let remove_barrier = backend.arm_failing_remove_barrier().await;

        let bucket = "transition-cleanup-context-a";
@@ -2452,4 +2495,307 @@ mod tests {
        );
        assert_eq!(backend.remove_count().await, 0);
    }
+
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn transition_transaction_recovery_retains_unproven_remote_candidates() {
+        let temp_dir = tempfile::tempdir().expect("create temp store dir");
+        let (ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-unproven", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+
+        let tier_name = "TXUNPROVEN";
+        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
+        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
+            .await
+            .expect("tier lease should resolve")
+            .backend_identity();
+        let remote_version = uuid::Uuid::new_v4().to_string();
+        let new_transaction = || {
+            TransitionTransaction::new(TransitionTransactionInit {
+                deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
+                transaction_id: uuid::Uuid::new_v4(),
+                owner_epoch: uuid::Uuid::new_v4(),
+                write_id: uuid::Uuid::new_v4(),
+                source: TransitionSourceIdentity {
+                    bucket: "absent-source-bucket".to_string(),
+                    object: "source-object".to_string(),
+                    version_id: None,
+                    data_dir: uuid::Uuid::new_v4(),
+                    mod_time_unix_nanos: 1_770_000_000_000_000_000,
+                    size: 42,
+                    etag: "source-etag".to_string(),
+                    version_mode: TransitionSourceVersionMode::Unversioned,
+                },
+                tier_name: tier_name.to_string(),
+                backend_fingerprint: backend_identity,
+                not_after_unix_nanos: 1_780_000_000_000_000_000,
+            })
+            .expect("transaction should build")
+        };
+
+        let upload_started = new_transaction();
+        let mut local_commit_started = new_transaction();
+        local_commit_started
+            .advance(
+                local_commit_started.fence(),
+                TransitionTransactionState::Uploaded,
+                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
+            )
+            .expect("transaction should enter uploaded state");
+        local_commit_started
+            .advance(local_commit_started.fence(), TransitionTransactionState::LocalCommitStarted, None)
+            .expect("transaction should enter local commit state");
+
+        backend.set_put_remote_version(Some(remote_version)).await;
+        for transaction in [&upload_started, &local_commit_started] {
+            let candidate = bytes::Bytes::from_static(b"unproven transition remote candidate");
+            backend
+                .put(
+                    &transaction.remote_object,
+                    ReaderImpl::Body(candidate.clone()),
+                    i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
+                )
+                .await
+                .expect("mock backend should accept candidate");
+            save_transition_transaction_record(store.clone(), transaction)
+                .await
+                .expect("transaction record should persist");
+        }
+
+        let stats = recover_transition_transaction_records(store.clone(), 100, None)
+            .await
+            .expect("transition transaction recovery should run");
+
+        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (2, 0, 2, 0));
+        assert_eq!(
+            transition_transaction_record_count(store.clone()).await,
+            2,
+            "an upload without completion proof or unproven local commit must remain for authoritative reconcile"
+        );
+        assert_eq!(backend.object_count().await, 2, "recovery must not delete an unproven remote candidate");
+        assert_eq!(backend.remove_count().await, 0);
+        assert_eq!(backend.exact_remove_count(), 0);
+    }
+
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn transition_transaction_recovery_deletes_provider_recovered_unknown_upload() {
+        let versioned_remote = uuid::Uuid::new_v4().to_string();
+        for (case, tier_name, remote_version) in [
+            ("missing", "TXPROBEMISSING", None),
+            ("unversioned", "TXPROBEUNVERSIONED", Some(String::new())),
+            ("versioned", "TXPROBEVERSIONED", Some(versioned_remote)),
+        ] {
+            let temp_dir = tempfile::tempdir().expect("create temp store dir");
+            let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
+                temp_dir.path(),
+                &format!("transition-transaction-probe-{case}"),
+                &[4],
+            ))
+            .await;
+            crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+
+            let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
+            let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
+                .await
+                .expect("tier lease should resolve")
+                .backend_identity();
+            let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
+                deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
+                transaction_id: uuid::Uuid::new_v4(),
+                owner_epoch: uuid::Uuid::new_v4(),
+                write_id: uuid::Uuid::new_v4(),
+                source: TransitionSourceIdentity {
+                    bucket: "source-bucket".to_string(),
+                    object: "source-object".to_string(),
+                    version_id: None,
+                    data_dir: uuid::Uuid::new_v4(),
+                    mod_time_unix_nanos: 1_770_000_000_000_000_000,
+                    size: 42,
+                    etag: "source-etag".to_string(),
+                    version_mode: TransitionSourceVersionMode::Unversioned,
+                },
+                tier_name: tier_name.to_string(),
+                backend_fingerprint: backend_identity,
+                not_after_unix_nanos: 1_780_000_000_000_000_000,
+            })
+            .expect("transaction should build");
+            transaction
+                .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
+                .expect("transaction should enter unknown upload outcome state");
+
+            if let Some(version) = &remote_version {
+                backend.set_put_remote_version(Some(version.clone())).await;
+                let candidate = bytes::Bytes::from_static(b"provider-recovered transition remote candidate");
+                backend
+                    .put(
+                        &transaction.remote_object,
+                        ReaderImpl::Body(candidate.clone()),
+                        i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
+                    )
+                    .await
+                    .expect("mock backend should accept candidate");
+            }
+            save_transition_transaction_record(store.clone(), &transaction)
+                .await
+                .expect("transaction record should persist");
+
+            let stats = recover_transition_transaction_records(store.clone(), 100, None)
+                .await
+                .expect("transition transaction recovery should run");
+
+            assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
+            assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
+            assert_eq!(
+                backend.object_count().await,
+                0,
+                "case {case}: recovered unknown upload candidate must be absent"
+            );
+            let removed = remote_version
+                .map(|version| vec![(transaction.remote_object.clone(), version)])
+                .unwrap_or_default();
+            assert_eq!(
+                backend.remove_versions().await,
+                removed,
+                "case {case}: recovery must delete only provider-recovered candidates"
+            );
+            assert_eq!(
+                backend.exact_remove_count(),
+                usize::from(removed.first().is_some_and(|(_, version)| !version.is_empty()))
+            );
+        }
+    }
+
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn transition_response_loss_persists_unknown_outcome_for_provider_recovery() {
+        let temp_dir = tempfile::tempdir().expect("create temp store dir");
+        let (ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-response-loss", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+
+        let tier_name = "TXRESPONSELOSS";
+        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
+        let bucket = "transition-response-loss-bucket";
+        let object = "source.bin";
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("source bucket should be created");
+        let payload = b"a response-lost tier PUT must remain recoverable".repeat(1024);
+        let mut reader = PutObjReader::from_vec(payload.clone());
+        let source = store
+            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
+            .await
+            .expect("source object should be written");
+        backend.lose_next_put_response();
+
+        let error = store
+            .transition_object(
+                bucket,
+                object,
+                &ObjectOptions {
+                    no_lock: true,
+                    transition: TransitionOptions {
+                        status: TRANSITION_PENDING.to_string(),
+                        tier: tier_name.to_string(),
+                        etag: source.etag.clone().expect("source object should have an ETag"),
+                        ..Default::default()
+                    },
+                    version_id: source.version_id.map(|version| version.to_string()),
+                    mod_time: source.mod_time,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect_err("a lost tier PUT response must fail the transition request");
+        assert!(
+            matches!(error, StorageError::Io(ref err) if err.kind() == std::io::ErrorKind::ConnectionReset),
+            "the response-loss error must remain visible to the caller: {error:?}"
+        );
+
+        let records = store
+            .clone()
+            .list_objects_v2(
+                RUSTFS_META_BUCKET,
+                TRANSITION_TRANSACTION_RECORD_PREFIX,
+                None,
+                None,
+                10,
+                false,
+                None,
+                false,
+            )
+            .await
+            .expect("transition transaction records should be listable");
+        assert_eq!(records.objects.len(), 1, "response loss must leave one durable transaction record");
+        let transaction_id = records.objects[0]
+            .name
+            .rsplit('/')
+            .next()
+            .and_then(|name| name.strip_suffix(".json"))
+            .and_then(|name| uuid::Uuid::parse_str(name).ok())
+            .expect("transaction record name should contain a UUID");
+        let transaction = load_transition_transaction_record(store.clone(), transaction_id)
+            .await
+            .expect("response loss transaction record should load");
+        assert_eq!(
+            transaction.state,
+            TransitionTransactionState::UploadOutcomeUnknown,
+            "a response-lost PUT must not remain in UploadStarted"
+        );
+        assert!(
+            backend.contains(&transaction.remote_object).await,
+            "the test backend must retain the remote candidate"
+        );
+
+        backend
+            .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
+            .await;
+        let unsupported_stats = recover_transition_transaction_records(store.clone(), 100, None)
+            .await
+            .expect("unsupported provider recovery should fail closed");
+        assert_eq!(
+            (
+                unsupported_stats.scanned,
+                unsupported_stats.recovered,
+                unsupported_stats.retained,
+                unsupported_stats.failed
+            ),
+            (1, 0, 1, 0),
+            "an unsupported provider probe must retain the unknown upload"
+        );
+        assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
+        assert!(
+            backend.contains(&transaction.remote_object).await,
+            "unsupported recovery must not delete the candidate"
+        );
+        assert_eq!(backend.remove_count().await, 0, "unsupported recovery must not attempt cleanup");
+
+        backend.set_transition_candidate_probe_override(None).await;
+        let stats = recover_transition_transaction_records(store.clone(), 100, None)
+            .await
+            .expect("provider-authoritative recovery should run");
+        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
+        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
+        assert_eq!(backend.object_count().await, 0, "recovery must delete the provider-confirmed candidate");
+        let op_log = backend.op_log().await;
+        assert!(
+            op_log.iter().any(|operation| matches!(operation, MockWarmOp::Probe { .. })),
+            "response-loss recovery must enter the provider probe branch"
+        );
+        assert!(
+            op_log.iter().any(|operation| matches!(operation, MockWarmOp::Put { .. })),
+            "response-loss fixture must record that the remote PUT reached the backend"
+        );
+        let source_after = store
+            .get_object_info(bucket, object, &ObjectOptions::default())
+            .await
+            .expect("recovery must preserve the local source object");
+        assert_eq!(source_after.size, i64::try_from(payload.len()).expect("payload length should fit i64"));
+    }
 }
@@ -872,6 +872,8 @@ impl ECStore {
        let idx = if opts.data_movement && opts.version_id.is_some() {
            self.select_data_movement_pool_idx(bucket, &object, data.size(), opts, false)
                .await?
+        } else if opts.no_lock {
+            self.get_pool_idx_no_lock(bucket, &object, data.size()).await?
        } else {
            self.get_pool_idx(bucket, &object, data.size()).await?
        };
@@ -2644,6 +2646,7 @@ mod tests {
                stream: Box::new(Cursor::new(Vec::<u8>::new())),
                object_info: ObjectInfo::default(),
                buffered_body: None,
+                resolved_sse: None,
                body_source: Default::default(),
            };

@@ -2684,6 +2687,7 @@ mod tests {
                stream: Box::new(Cursor::new(vec![1, 2, 3])),
                object_info: ObjectInfo::default(),
                buffered_body: None,
+                resolved_sse: None,
                body_source: Default::default(),
            };

@@ -2721,6 +2725,7 @@ mod tests {
                stream: Box::new(Cursor::new(vec![1, 2, 3])),
                object_info: ObjectInfo::default(),
                buffered_body: Some(Bytes::from_static(b"123")),
+                resolved_sse: None,
                body_source: Default::default(),
            };

@@ -2758,6 +2763,7 @@ mod tests {
                stream: Box::new(Cursor::new(vec![1, 2, 3])),
                object_info: ObjectInfo::default(),
                buffered_body: None,
+                resolved_sse: None,
                body_source: Default::default(),
            };

@@ -7,10 +7,12 @@ use std::path::{Path, PathBuf};
 mod storage_api;

 use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
+use rustfs_utils::HashAlgorithm;
 use serde::Deserialize;
 use sha2::{Digest, Sha256};
 use storage_api::minio_generated_read::{
-    DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk,
+    DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, create_bitrot_reader,
+    new_disk,
 };
 use temp_env::async_with_vars;
 use tokio::io::{AsyncReadExt, AsyncWrite};
@@ -22,6 +24,11 @@ struct ManifestRecord {
    backend_files: Vec<String>,
 }

+#[derive(Debug, Deserialize)]
+struct RequestRecord {
+    headers: std::collections::HashMap<String, String>,
+}
+
 #[derive(Default)]
 struct VecAsyncWriter {
    bytes: Vec<u8>,
@@ -56,21 +63,17 @@ fn case_dir(case_id: &str) -> PathBuf {
    fixture_root().join("cases").join(case_id)
 }

+fn beta5_fixture_root() -> PathBuf {
+    std::env::var_os("RUSTFS_BETA5_FIXTURE_ROOT")
+        .map(PathBuf::from)
+        .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../rio-v2/tests/fixtures/rustfs-beta5-generated"))
+}
+
 fn read_json<T: for<'de> Deserialize<'de>>(path: &Path) -> T {
    let text = fs::read_to_string(path).unwrap_or_else(|err| panic!("read {}: {err}", path.display()));
    serde_json::from_str(&text).unwrap_or_else(|err| panic!("parse {}: {err}", path.display()))
 }

-fn require_fixture_case(case_id: &str) -> PathBuf {
-    let path = case_dir(case_id);
-    assert!(
-        path.is_dir(),
-        "fixture case missing: {}. Run scripts/minio_fixture_lab/lab.py capture-matrix first.",
-        path.display()
-    );
-    path
-}
-
 fn read_plaintext_sha256(case_dir: &Path) -> String {
    fs::read_to_string(case_dir.join("plaintext.sha256"))
        .unwrap_or_else(|err| panic!("read plaintext.sha256 under {}: {err}", case_dir.display()))
@@ -78,9 +81,9 @@ fn read_plaintext_sha256(case_dir: &Path) -> String {
        .to_string()
 }

-fn minio_static_kms_key_b64() -> String {
-    std::env::var("RUSTFS_MINIO_STATIC_KMS_KEY_B64")
-        .unwrap_or_else(|_| panic!("RUSTFS_MINIO_STATIC_KMS_KEY_B64 must point to the 32-byte static MinIO KMS key"))
+fn minio_static_kms_key() -> String {
+    std::env::var("RUSTFS_MINIO_STATIC_KMS_KEY")
+        .unwrap_or_else(|_| panic!("RUSTFS_MINIO_STATIC_KMS_KEY must use <key-id>:<base64-32-byte-key>"))
 }

 fn object_xl_meta_path(case_dir: &Path, manifest: &ManifestRecord) -> PathBuf {
@@ -117,37 +120,53 @@ fn sha256_hex(bytes: &[u8]) -> String {
    hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower)
 }

-async fn load_fixture_reader_input(case_id: &str) -> (ObjectInfo, Vec<u8>, String) {
-    let case_dir = require_fixture_case(case_id);
+async fn load_fixture_reader_input(case_id: &str) -> (ObjectInfo, Vec<u8>, String, http::HeaderMap) {
+    load_fixture_reader_input_from(case_dir(case_id)).await
+}
+
+async fn load_fixture_reader_input_from(case_dir: PathBuf) -> (ObjectInfo, Vec<u8>, String, http::HeaderMap) {
+    assert!(case_dir.is_dir(), "fixture case missing: {}", case_dir.display());
    let manifest: ManifestRecord = read_json(&case_dir.join("manifest.json"));
+    let request: RequestRecord = read_json(&case_dir.join("request.json"));
    let expected_sha256 = read_plaintext_sha256(&case_dir);
    let file_info = load_file_info(&case_dir, &manifest);
    let encrypted = encrypted_fixture_bytes(&case_dir, &manifest, &file_info).await;
    let object_info = load_object_info(&file_info, &manifest);

-    (object_info, encrypted, expected_sha256)
+    let mut headers = http::HeaderMap::new();
+    for (name, value) in request.headers {
+        let name = http::header::HeaderName::from_bytes(name.as_bytes())
+            .unwrap_or_else(|err| panic!("invalid fixture request header {name}: {err}"));
+        let value =
+            http::HeaderValue::try_from(value).unwrap_or_else(|err| panic!("invalid fixture request header value: {err}"));
+        headers.insert(name, value);
+    }
+
+    (object_info, encrypted, expected_sha256, headers)
 }

-async fn read_fixture_plaintext(encrypted: Vec<u8>, object_info: ObjectInfo, kms_key_b64: String) -> Result<Vec<u8>, String> {
+async fn read_fixture_plaintext(
+    encrypted: Vec<u8>,
+    object_info: ObjectInfo,
+    headers: http::HeaderMap,
+    static_kms_key: Option<String>,
+    range: Option<HTTPRangeSpec>,
+) -> Result<Vec<u8>, String> {
    let object_size = object_info.size;
+    let full_object = range.is_none();

    async_with_vars(
        [
-            ("__RUSTFS_SSE_SIMPLE_CMK", Some(kms_key_b64)),
-            ("RUSTFS_SSE_S3_MASTER_KEY", None::<String>),
+            ("RUSTFS_MINIO_STATIC_KMS_KEY", static_kms_key),
+            ("__RUSTFS_SSE_SIMPLE_CMK", None::<String>),
        ],
        async move {
-            let (mut reader, offset, length) = GetObjectReader::new(
-                Box::new(Cursor::new(encrypted)),
-                None,
-                &object_info,
-                &ObjectOptions::default(),
-                &http::HeaderMap::new(),
-            )
-            .await
-            .map_err(|err| format!("construct GetObjectReader from MinIO raw fixture: {err:?}"))?;
+            let (mut reader, offset, length) =
+                GetObjectReader::new(Box::new(Cursor::new(encrypted)), range, &object_info, &ObjectOptions::default(), &headers)
+                    .await
+                    .map_err(|err| format!("construct GetObjectReader from MinIO raw fixture: {err:?}"))?;

-            if offset != 0 || length != object_size {
+            if full_object && (offset != 0 || length != object_size) {
                return Err(format!("unexpected fixture range offset={offset} length={length} size={object_size}"));
            }

@@ -164,6 +183,12 @@ async fn read_fixture_plaintext(encrypted: Vec<u8>, object_info: ObjectInfo, kms
 }

 async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, file_info: &FileInfo) -> Vec<u8> {
+    let erasure = Erasure::new_with_options(
+        file_info.erasure.data_blocks,
+        file_info.erasure.parity_blocks,
+        file_info.erasure.block_size,
+        file_info.uses_legacy_checksum,
+    );
    let mut disks = Vec::with_capacity(file_info.erasure.distribution.len());
    for disk_number in 1..=file_info.erasure.distribution.len() {
        let disk_root = case_dir.join("backend").join(format!("disk{disk_number}"));
@@ -198,8 +223,13 @@ async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, fil
    let mut encrypted = Vec::new();
    for part in &file_info.parts {
        let checksum_info = file_info.erasure.get_checksum_info(part.number);
+        let checksum_algorithm = if file_info.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
+            HashAlgorithm::HighwayHash256SLegacy
+        } else {
+            checksum_info.algorithm.clone()
+        };
        let path = format!("{}/{}/part.{}", manifest.object, data_dir, part.number);
-        let shard_read_len = file_info.erasure.shard_file_size(part.size as i64);
+        let shard_read_len = erasure.shard_file_size(part.size as i64);
        let mut readers = Vec::with_capacity(disks.len());
        for (idx, disk) in disk_order.iter().enumerate() {
            let reader = create_bitrot_reader(
@@ -210,7 +240,7 @@ async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, fil
                0,
                shard_read_len as usize,
                file_info.erasure.shard_size(),
-                checksum_info.algorithm.clone(),
+                checksum_algorithm.clone(),
                false,
                false,
            )
@@ -219,11 +249,6 @@ async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, fil
            readers.push(reader);
        }

-        let erasure = Erasure::new(
-            file_info.erasure.data_blocks,
-            file_info.erasure.parity_blocks,
-            file_info.erasure.block_size,
-        );
        let mut writer = VecAsyncWriter::default();
        let (written, err) = erasure.decode(&mut writer, readers, 0, part.size, part.size).await;
        if let Some(err) = err {
@@ -244,6 +269,12 @@ async fn reads_minio_generated_sse_s3_multipart_fixture() {
    assert_fixture_round_trip("sse-s3-multipart-8m", 8 * 1024 * 1024).await;
 }

+#[tokio::test]
+#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
+async fn reads_minio_generated_sse_s3_singlepart_fixture() {
+    assert_fixture_round_trip("sse-s3-singlepart-64k", 64 * 1024).await;
+}
+
 #[tokio::test]
 #[ignore = "requires generated MinIO fixture data and a local static KMS key"]
 async fn reads_minio_generated_sse_kms_multipart_fixture() {
@@ -252,11 +283,47 @@ async fn reads_minio_generated_sse_kms_multipart_fixture() {

 #[tokio::test]
 #[ignore = "requires generated MinIO fixture data and a local static KMS key"]
-async fn rejects_minio_generated_sse_s3_fixture_with_wrong_kms_key() {
-    let (object_info, encrypted, _) = load_fixture_reader_input("sse-s3-multipart-8m").await;
-    let wrong_key_b64 = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=".to_string();
+async fn reads_minio_generated_sse_kms_singlepart_fixture() {
+    assert_fixture_round_trip("sse-kms-singlepart-64k", 64 * 1024).await;
+}

-    let result = read_fixture_plaintext(encrypted, object_info, wrong_key_b64).await;
+#[tokio::test]
+#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
+async fn reads_minio_generated_sse_c_multipart_fixture() {
+    assert_fixture_round_trip("sse-c-multipart-8m", 8 * 1024 * 1024).await;
+}
+
+#[tokio::test]
+#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
+async fn reads_minio_generated_sse_c_singlepart_fixture() {
+    assert_fixture_round_trip("sse-c-singlepart-64k", 64 * 1024).await;
+}
+
+#[tokio::test]
+#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
+async fn reads_minio_generated_sse_s3_range_fixture() {
+    assert_fixture_range_round_trip("sse-s3-multipart-8m").await;
+}
+
+#[tokio::test]
+#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
+async fn reads_minio_generated_sse_kms_range_fixture() {
+    assert_fixture_range_round_trip("sse-kms-multipart-8m").await;
+}
+
+#[tokio::test]
+#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
+async fn reads_minio_generated_sse_c_range_fixture() {
+    assert_fixture_range_round_trip("sse-c-multipart-8m").await;
+}
+
+#[tokio::test]
+#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
+async fn rejects_minio_generated_sse_s3_fixture_with_wrong_kms_key() {
+    let (object_info, encrypted, _, headers) = load_fixture_reader_input("sse-s3-multipart-8m").await;
+    let wrong_key_b64 = "minio-default-key:AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=".to_string();
+
+    let result = read_fixture_plaintext(encrypted, object_info, headers, Some(wrong_key_b64), None).await;

    assert!(result.is_err(), "wrong KMS key must fail closed");
 }
@@ -264,22 +331,16 @@ async fn rejects_minio_generated_sse_s3_fixture_with_wrong_kms_key() {
 #[tokio::test]
 #[ignore = "requires generated MinIO fixture data and a local static KMS key"]
 async fn rejects_minio_generated_sse_s3_fixture_with_truncated_ciphertext() {
-    let (object_info, mut encrypted, expected_sha256) = load_fixture_reader_input("sse-s3-multipart-8m").await;
+    let (object_info, mut encrypted, _, headers) = load_fixture_reader_input("sse-s3-multipart-8m").await;
    encrypted.truncate(encrypted.len() / 2);

-    let result = read_fixture_plaintext(encrypted, object_info, minio_static_kms_key_b64()).await;
+    let result = read_fixture_plaintext(encrypted, object_info, headers, Some(minio_static_kms_key()), None).await;

-    if let Ok(plaintext) = result {
-        assert_ne!(
-            sha256_hex(&plaintext),
-            expected_sha256,
-            "truncated ciphertext must not restore the original plaintext"
-        );
-    }
+    assert!(result.is_err(), "truncated ciphertext must return an explicit read error");
 }

 async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) {
-    let (object_info, encrypted, expected_sha256) = load_fixture_reader_input(case_id).await;
+    let (object_info, encrypted, expected_sha256, headers) = load_fixture_reader_input(case_id).await;
    // `ObjectInfo.size` is the on-disk size. For SSE objects that is the
    // DARE-encrypted size (plaintext + 32 bytes per 64 KiB block), which is
    // deliberately larger than the logical object size. The size a client sees
@@ -287,9 +348,9 @@ async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) {
    // `decrypted_size()`/`get_actual_size()`, so assert against that — the raw
    // `size` field would never equal the plaintext length for encrypted objects.
    let decrypted_size = object_info.decrypted_size().expect("decrypted size from MinIO metadata");
-    let kms_key_b64 = minio_static_kms_key_b64();
+    let kms_key = minio_static_kms_key();

-    let plaintext = read_fixture_plaintext(encrypted, object_info, kms_key_b64)
+    let plaintext = read_fixture_plaintext(encrypted, object_info, headers, Some(kms_key), None)
        .await
        .expect("fixture must restore with the configured KMS key");

@@ -297,3 +358,59 @@ async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) {
    assert_eq!(plaintext.len(), expected_size as usize);
    assert_eq!(sha256_hex(&plaintext), expected_sha256);
 }
+
+async fn assert_fixture_range_round_trip(case_id: &str) {
+    const START: usize = 65_520;
+    const END: usize = 65_680;
+
+    let (object_info, encrypted, _, headers) = load_fixture_reader_input(case_id).await;
+    let kms_key = minio_static_kms_key();
+    let plaintext = read_fixture_plaintext(encrypted.clone(), object_info.clone(), headers.clone(), Some(kms_key.clone()), None)
+        .await
+        .expect("fixture full read must restore");
+    let ranged = read_fixture_plaintext(
+        encrypted,
+        object_info,
+        headers,
+        Some(kms_key),
+        Some(HTTPRangeSpec {
+            is_suffix_length: false,
+            start: START as i64,
+            end: END as i64,
+        }),
+    )
+    .await
+    .expect("fixture range read must restore");
+
+    assert_eq!(ranged, plaintext[START..=END]);
+}
+
+#[tokio::test]
+#[ignore = "requires a fixture generated by the pinned RustFS beta.5 release"]
+async fn reads_real_rustfs_beta5_sse_kms_fixture_through_production_reader() {
+    const CASE_ID: &str = "rustfs-beta5-sse-kms-singlepart-64k";
+    const KMS_KEY_ID: &str = "beta5-test-key";
+
+    let root = beta5_fixture_root();
+    let manager = rustfs_kms::init_global_kms_service_manager();
+    manager
+        .configure(
+            rustfs_kms::KmsConfig::local(root.join("kms"))
+                .with_default_key(KMS_KEY_ID.to_string())
+                .with_insecure_development_defaults(),
+        )
+        .await
+        .expect("configure production local KMS for beta.5 fixture");
+    manager.start().await.expect("start production local KMS for beta.5 fixture");
+
+    let (object_info, encrypted, expected_sha256, headers) =
+        load_fixture_reader_input_from(root.join("cases").join(CASE_ID)).await;
+    let expected_size = object_info.decrypted_size().expect("beta.5 encrypted object size");
+    let plaintext = read_fixture_plaintext(encrypted, object_info, headers, None, None)
+        .await
+        .expect("current production GET reader must decrypt the real beta.5 KMS fixture");
+
+    assert_eq!(expected_size, 64 * 1024);
+    assert_eq!(plaintext.len() as i64, expected_size);
+    assert_eq!(sha256_hex(&plaintext), expected_sha256);
+}
@@ -36,6 +36,7 @@ pub(crate) mod legacy_bitrot_read {
 }

 pub(crate) mod minio_generated_read {
+    pub(crate) use super::storage_contracts::HTTPRangeSpec;
    pub(crate) use super::{
        DiskAPI, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk,
    };
@@ -56,7 +56,7 @@ moka = { workspace = true, features = ["future"] }
 # Additional dependencies
 md5 = { workspace = true }
 arc-swap = { workspace = true }
-rustfs-utils = { workspace = true }
+rustfs-utils = { workspace = true, features = ["http"] }
 rustfs-security-governance = { workspace = true }

 # HTTP client for Vault
@@ -44,6 +44,23 @@ pub struct DataKeyEnvelope {
    pub created_at: Zoned,
 }

+/// Return whether bytes contain a complete RustFS KMS data-key envelope.
+pub fn is_data_key_envelope(ciphertext: &[u8]) -> bool {
+    const MAX_ENVELOPE_SIZE: usize = 64 * 1024;
+
+    if ciphertext.is_empty() || ciphertext.len() > MAX_ENVELOPE_SIZE {
+        return false;
+    }
+
+    serde_json::from_slice::<DataKeyEnvelope>(ciphertext).is_ok_and(|envelope| {
+        !envelope.key_id.trim().is_empty()
+            && !envelope.master_key_id.trim().is_empty()
+            && envelope.key_spec == "AES_256"
+            && !envelope.encrypted_key.is_empty()
+            && (envelope.nonce.is_empty() || envelope.nonce.len() == 12)
+    })
+}
+
 /// Trait for encrypting and decrypting data encryption keys (DEK)
 ///
 /// This trait abstracts the encryption operations used to protect
@@ -293,6 +310,24 @@ mod tests {
        assert_eq!(deserialized.key_id, envelope.key_id);
        assert_eq!(deserialized.master_key_id, envelope.master_key_id);
        assert_eq!(deserialized.encrypted_key, envelope.encrypted_key);
+        assert!(is_data_key_envelope(&serialized));
+        assert!(!is_data_key_envelope(b"not-a-kms-envelope"));
+        let mut boundary_envelope = serialized;
+        boundary_envelope.resize(64 * 1024, b' ');
+        assert!(is_data_key_envelope(&boundary_envelope));
+        boundary_envelope.push(b' ');
+        assert!(!is_data_key_envelope(&boundary_envelope));
+
+        let mut invalid = serde_json::to_value(&envelope).expect("Envelope should convert to JSON");
+        invalid["key_spec"] = serde_json::Value::String("AES_128".to_string());
+        assert!(!is_data_key_envelope(
+            &serde_json::to_vec(&invalid).expect("Invalid envelope should serialize")
+        ));
+        invalid["key_spec"] = serde_json::Value::String("AES_256".to_string());
+        invalid["unknown"] = serde_json::Value::Bool(true);
+        assert!(is_data_key_envelope(
+            &serde_json::to_vec(&invalid).expect("Envelope with unknown field should serialize")
+        ));
    }

    #[tokio::test]
@@ -17,4 +17,4 @@
 pub mod ciphers;
 pub mod dek;

-pub use dek::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
+pub use dek::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material, is_data_key_envelope};
@@ -70,6 +70,7 @@ mod cache;
 pub mod config;
 mod encryption;
 mod error;
+mod managed_context;
 pub mod manager;
 pub mod service;
 pub mod service_manager;
@@ -83,7 +84,11 @@ pub use api_types::{
    TagKeyRequest, TagKeyResponse, UntagKeyRequest, UntagKeyResponse, UpdateKeyDescriptionRequest, UpdateKeyDescriptionResponse,
 };
 pub use config::*;
+pub use encryption::is_data_key_envelope;
 pub use error::{KmsError, Result};
+pub use managed_context::{
+    MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, RUSTFS_ENCRYPTION_CONTEXT_HEADER, decode_managed_kms_context,
+};
 pub use manager::KmsManager;
 pub use service::{DataKey, ObjectEncryptionService};
 pub use service_manager::{
@@ -0,0 +1,110 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{KmsError, Result};
+use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
+use rustfs_utils::http::get_consistent_metadata_value;
+use std::collections::HashMap;
+
+pub const RUSTFS_ENCRYPTION_CONTEXT_HEADER: &str = "x-rustfs-encryption-context";
+pub const MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Context";
+
+pub fn decode_managed_kms_context(metadata: &HashMap<String, String>) -> Result<Option<HashMap<String, String>>> {
+    let minio_context = consistent_value(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)?
+        .map(|context| {
+            let decoded = BASE64_STANDARD
+                .decode(context)
+                .map_err(|err| KmsError::serialization_error(format!("Failed to decode MinIO KMS context: {err}")))?;
+            serde_json::from_slice(&decoded)
+                .map_err(|err| KmsError::serialization_error(format!("Failed to parse MinIO KMS context: {err}")))
+        })
+        .transpose()?;
+    let rustfs_context = consistent_value(metadata, RUSTFS_ENCRYPTION_CONTEXT_HEADER)?
+        .map(|context| {
+            serde_json::from_str(context)
+                .map_err(|err| KmsError::serialization_error(format!("Failed to parse RustFS KMS context: {err}")))
+        })
+        .transpose()?;
+
+    match (minio_context, rustfs_context) {
+        (Some(minio), Some(rustfs)) if minio != rustfs => {
+            Err(KmsError::context_mismatch("Conflicting RustFS and MinIO KMS contexts"))
+        }
+        (Some(context), _) | (_, Some(context)) => Ok(Some(context)),
+        (None, None) => Ok(None),
+    }
+}
+
+fn consistent_value<'a>(metadata: &'a HashMap<String, String>, name: &str) -> Result<Option<&'a str>> {
+    get_consistent_metadata_value(metadata, name)
+        .map_err(|_| KmsError::validation_error(format!("Conflicting managed encryption metadata for {name}")))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn decode_context_accepts_compatible_headers_and_rejects_conflicts() {
+        let expected = HashMap::from([("tenant".to_string(), "alpha".to_string())]);
+        let metadata = HashMap::from([
+            (
+                RUSTFS_ENCRYPTION_CONTEXT_HEADER.to_string(),
+                serde_json::to_string(&expected).expect("RustFS KMS context should serialize"),
+            ),
+            (
+                MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(),
+                BASE64_STANDARD.encode(serde_json::to_vec(&expected).expect("MinIO KMS context should serialize")),
+            ),
+        ]);
+        assert_eq!(
+            decode_managed_kms_context(&metadata).expect("matching KMS contexts should parse"),
+            Some(expected.clone())
+        );
+        assert_eq!(
+            decode_managed_kms_context(&HashMap::from([(
+                RUSTFS_ENCRYPTION_CONTEXT_HEADER.to_string(),
+                serde_json::to_string(&expected).expect("legacy RustFS KMS context should serialize"),
+            )]))
+            .expect("legacy RustFS KMS context should parse"),
+            Some(expected)
+        );
+
+        let conflicting = HashMap::from([
+            (
+                RUSTFS_ENCRYPTION_CONTEXT_HEADER.to_string(),
+                serde_json::to_string(&HashMap::from([("tenant", "alpha")])).expect("RustFS KMS context should serialize"),
+            ),
+            (
+                MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(),
+                BASE64_STANDARD.encode(
+                    serde_json::to_vec(&HashMap::from([("tenant", "beta")])).expect("MinIO KMS context should serialize"),
+                ),
+            ),
+        ]);
+        assert!(decode_managed_kms_context(&conflicting).is_err());
+
+        assert!(
+            decode_managed_kms_context(&HashMap::from([(
+                MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(),
+                "not-base64".to_string(),
+            )]))
+            .is_err()
+        );
+        assert!(
+            decode_managed_kms_context(&HashMap::from([(RUSTFS_ENCRYPTION_CONTEXT_HEADER.to_string(), "not-json".to_string(),)]))
+                .is_err()
+        );
+    }
+}
@@ -44,6 +44,7 @@ serde = { workspace = true, features = ["derive"] }
 starshard = { workspace = true, features = ["rayon", "async", "serde"] }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time", "fs"] }
+tokio-util = { workspace = true }
 tracing = { workspace = true }
 url = { workspace = true }
 metrics = { workspace = true }
@@ -62,6 +63,7 @@ axum = { workspace = true }
 rustfs-utils = { workspace = true, features = ["path"] }
 serde_json = { workspace = true, features = ["raw_value"] }
 criterion = { workspace = true, features = ["html_reports"] }
+tempfile = { workspace = true }

 [lints]
 workspace = true
@@ -13,89 +13,83 @@
 // limitations under the License.

 use crate::{
-    Event, NotificationError, registry::TargetRegistry, resolve_notify_object_store_handle, rule_engine::NotifyRuleEngine,
+    NotificationError,
+    lifecycle::{NotificationRuntimeState, NotifyLifecycleCoordinator},
+    registry::TargetRegistry,
+    resolve_notify_object_store_handle,
+    rule_engine::NotifyRuleEngine,
    runtime_facade::NotifyRuntimeFacade,
+    with_notify_server_config_read_lock, with_notify_server_config_write_lock,
 };
 use rustfs_config::notify::{
    NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS,
    NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
 };
 use rustfs_config::server_config::{Config, KVS};
-use rustfs_targets::{Target, arn::TargetID};
-use std::sync::{Arc, LazyLock};
-use tokio::sync::{Mutex, RwLock};
+use rustfs_targets::arn::TargetID;
+use std::sync::Arc;
+use tokio::sync::RwLock;
 use tracing::{debug, info};

-/// Serializes the read-modify-write sequence over the persisted notify server
-/// config. The persisted config is a single process-global resource (there is
-/// only one backing object store), so without this guard two concurrent updates
-/// can both read the same base config, apply disjoint changes, and race their
-/// full-config writes — the later write silently overwrites the earlier one,
-/// losing updates. Holding this mutex across the whole read→modify→write makes
-/// concurrent updates apply serially so every change is preserved.
-///
-/// The lock is only ever acquired inside `update_server_config`; it never nests
-/// with the per-manager `config` RwLock (the in-memory reload runs after this
-/// guard is released), so it introduces no lock-ordering risk.
-static NOTIFY_CONFIG_RMW_LOCK: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(()));
-
 const LOG_COMPONENT_NOTIFY: &str = "notify";
 const LOG_SUBSYSTEM_CONFIG: &str = "config";
-const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
 const EVENT_NOTIFY_CONFIG_UPDATE: &str = "notify_config_update";

 #[derive(Debug)]
 enum NotifyConfigStoreError {
+    Lock(String),
    StorageNotAvailable,
    Read(String),
    Save(String),
 }

-async fn update_server_config<F>(modifier: F) -> Result<Option<Config>, NotifyConfigStoreError>
+async fn update_server_config<F>(
+    modifier: F,
+    lifecycle: NotifyLifecycleCoordinator,
+) -> Result<Option<crate::lifecycle::NotificationLifecycleTransition>, NotifyConfigStoreError>
 where
-    F: FnMut(&mut Config) -> bool,
+    F: FnMut(&mut Config) -> bool + Send + 'static,
 {
    let Some(store) = resolve_notify_object_store_handle() else {
        return Err(NotifyConfigStoreError::StorageNotAvailable);
    };

+    let store_for_read = store.clone();
    let store_for_save = store.clone();
-    serialized_read_modify_write(
-        modifier,
-        move || async move {
-            crate::read_notify_server_config_without_migrate(store)
-                .await
-                .map_err(NotifyConfigStoreError::Read)
-        },
-        move |config| async move {
-            crate::save_notify_server_config(store_for_save, &config)
-                .await
-                .map_err(NotifyConfigStoreError::Save)
-        },
-    )
+    with_notify_server_config_write_lock(store, move || {
+        read_modify_write(
+            modifier,
+            move || async move {
+                crate::read_notify_server_config_without_migrate_no_lock(store_for_read)
+                    .await
+                    .map_err(NotifyConfigStoreError::Read)
+            },
+            move |config| async move {
+                crate::save_notify_server_config_no_lock(store_for_save, &config)
+                    .await
+                    .map_err(NotifyConfigStoreError::Save)
+            },
+            move |config| lifecycle.update_config(config),
+        )
+    })
    .await
+    .map_err(NotifyConfigStoreError::Lock)?
 }

-/// Runs a `read → modify → write` over the persisted notify config while holding
-/// [`NOTIFY_CONFIG_RMW_LOCK`], so concurrent updates serialize and cannot clobber
-/// each other's changes (backlog#968). `read`/`save` are injected so the exact
-/// production serialization path can be exercised in tests without a live store.
-async fn serialized_read_modify_write<F, R, RFut, S, SFut>(
+async fn read_modify_write<F, R, RFut, S, SFut, P, T>(
    mut modifier: F,
    read: R,
    save: S,
-) -> Result<Option<Config>, NotifyConfigStoreError>
+    publish: P,
+) -> Result<Option<T>, NotifyConfigStoreError>
 where
    F: FnMut(&mut Config) -> bool,
    R: FnOnce() -> RFut,
    RFut: std::future::Future<Output = Result<Config, NotifyConfigStoreError>>,
    S: FnOnce(Config) -> SFut,
    SFut: std::future::Future<Output = Result<(), NotifyConfigStoreError>>,
+    P: FnOnce(Config) -> T,
 {
-    // Hold the RMW lock across the entire read→modify→write so concurrent
-    // updates serialize and cannot clobber each other's changes (backlog#968).
-    let _rmw_guard = NOTIFY_CONFIG_RMW_LOCK.lock().await;
-
    let mut new_config = read().await?;

    if !modifier(&mut new_config) {
@@ -104,7 +98,7 @@ where

    save(new_config.clone()).await?;

-    Ok(Some(new_config))
+    Ok(Some(publish(new_config)))
 }

 pub(crate) fn notify_configuration_hint() -> String {
@@ -142,9 +136,8 @@ pub fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) ->
 #[derive(Clone)]
 pub struct NotifyConfigManager {
    config: Arc<RwLock<Config>>,
-    registry: Arc<TargetRegistry>,
+    lifecycle: NotifyLifecycleCoordinator,
    rule_engine: NotifyRuleEngine,
-    runtime_facade: NotifyRuntimeFacade,
 }

 impl NotifyConfigManager {
@@ -154,64 +147,21 @@ impl NotifyConfigManager {
        rule_engine: NotifyRuleEngine,
        runtime_facade: NotifyRuntimeFacade,
    ) -> Self {
+        let lifecycle = NotifyLifecycleCoordinator::new(config.clone(), registry, runtime_facade);
        Self {
            config,
-            registry,
+            lifecycle,
            rule_engine,
-            runtime_facade,
        }
    }

+    pub(crate) fn lifecycle(&self) -> NotifyLifecycleCoordinator {
+        self.lifecycle.clone()
+    }
+
    pub async fn init(&self) -> Result<(), NotificationError> {
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "initializing",
-            "notify runtime lifecycle"
-        );
-
-        let config = {
-            let guard = self.config.read().await;
-            debug!(
-                subsystem_count = guard.0.len(),
-                "Initializing notification system with configuration summary"
-            );
-            guard.clone()
-        };
-
-        let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self.registry.create_targets_from_config(&config).await?;
-
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "targets_created",
-            target_count = targets.len(),
-            "notify runtime lifecycle"
-        );
-        if targets.is_empty() {
-            debug!(
-                event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-                component = LOG_COMPONENT_NOTIFY,
-                subsystem = LOG_SUBSYSTEM_CONFIG,
-                state = "idle",
-                reason = "no_targets_configured",
-                hint = %notify_configuration_hint(),
-                "notify runtime lifecycle"
-            );
-        }
-
-        let activation = self.runtime_facade.activate_targets_with_replay(targets).await;
-        self.runtime_facade.replace_targets(activation).await?;
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "initialized",
-            "notify runtime lifecycle"
-        );
-        Ok(())
+        let config = self.config.read().await.clone();
+        self.lifecycle.set_mode(true, Some(config)).wait().await
    }

    pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> {
@@ -227,6 +177,7 @@ impl NotifyConfigManager {

        let ttype = target_type.to_lowercase();
        let tname = target_id.id.to_lowercase();
+        let log_target_id = target_id.clone();

        // Guard against orphaning bucket notification rules (backlog#979). Removing a
        // target while a bucket rule still references it would leave a dangling
@@ -240,7 +191,7 @@ impl NotifyConfigManager {
            )));
        }

-        self.update_config_and_reload(|config| {
+        self.update_config_and_reload(move |config| {
            let mut changed = false;
            if let Some(targets_of_type) = config.0.get_mut(&ttype) {
                if targets_of_type.remove(&tname).is_some() {
@@ -249,7 +200,7 @@ impl NotifyConfigManager {
                            component = LOG_COMPONENT_NOTIFY,
                        subsystem = LOG_SUBSYSTEM_CONFIG,
                        action = "remove_target",
-                        target_id = %target_id,
+                        target_id = %log_target_id,
                        result = "removed",
                        "notify config update"
                    );
@@ -265,7 +216,7 @@ impl NotifyConfigManager {
                    component = LOG_COMPONENT_NOTIFY,
                    subsystem = LOG_SUBSYSTEM_CONFIG,
                    action = "remove_target",
-                    target_id = %target_id,
+                    target_id = %log_target_id,
                    result = "not_found",
                    "notify config update"
                );
@@ -287,7 +238,7 @@ impl NotifyConfigManager {
        );
        let ttype = target_type.to_lowercase();
        let tname = target_name.to_lowercase();
-        self.update_config_and_reload(|config| {
+        self.update_config_and_reload(move |config| {
            config.0.entry(ttype.clone()).or_default().insert(tname.clone(), kvs.clone());
            true
        })
@@ -316,7 +267,7 @@ impl NotifyConfigManager {
            )));
        }

-        self.update_config_and_reload(|config| {
+        self.update_config_and_reload(move |config| {
            let mut changed = false;
            if let Some(targets) = config.0.get_mut(&ttype) {
                if targets.remove(&tname).is_some() {
@@ -332,8 +283,8 @@ impl NotifyConfigManager {
                    component = LOG_COMPONENT_NOTIFY,
                    subsystem = LOG_SUBSYSTEM_CONFIG,
                    action = "remove_target_config",
-                    target_type = %target_type,
-                    target_name = %target_name,
+                    target_type = %ttype,
+                    target_name = %tname,
                    result = "not_found",
                    "notify config update"
                );
@@ -348,81 +299,62 @@ impl NotifyConfigManager {
    }

    pub async fn reload_config(&self, new_config: Config) -> Result<(), NotificationError> {
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "reloading",
-            "notify runtime lifecycle"
-        );
+        self.lifecycle.set_mode(true, Some(new_config)).wait().await
+    }

-        self.update_config(new_config.clone()).await;
+    pub async fn reload_persisted_config(&self) -> Result<(), NotificationError> {
+        let Some(store) = resolve_notify_object_store_handle() else {
+            return Err(NotificationError::StorageNotAvailable(
+                "Failed to load target configuration: server storage not initialized".to_string(),
+            ));
+        };
+        self.reload_persisted_config_from_store(store).await
+    }

-        // Stop the currently running replay workers *before* activating the new ones
-        // (backlog#970). Each replay worker drains a per-target persisted store; if the
-        // new workers start while the old ones are still running against the same
-        // stores, both drain the same queues and re-deliver events. `replace_targets`
-        // below also stops workers, but only after `activate_targets_with_replay` has
-        // already spawned the new ones — so without this explicit stop-before-start
-        // there is a window where old and new workers overlap. (The full "signal +
-        // join" shutdown lives in the targets crate and is tracked under the same
-        // issue; this reorders the notify-side lifecycle.)
-        self.runtime_facade.stop_replay_workers().await;
-
-        let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self
-            .registry
-            .create_targets_from_config(&new_config)
-            .await
-            .map_err(NotificationError::Target)?;
-
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "targets_created",
-            target_count = targets.len(),
-            "notify runtime lifecycle"
-        );
-        if targets.is_empty() {
-            debug!(
-                event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-                component = LOG_COMPONENT_NOTIFY,
-                subsystem = LOG_SUBSYSTEM_CONFIG,
-                state = "idle",
-                reason = "no_targets_configured",
-                hint = %notify_configuration_hint(),
-                "notify runtime lifecycle"
-            );
+    pub async fn reload_persisted_config_from_store(&self, store: Arc<crate::NotifyStore>) -> Result<(), NotificationError> {
+        if self.lifecycle.state() == NotificationRuntimeState::Terminated {
+            return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
        }

-        let activation = self.runtime_facade.activate_targets_with_replay(targets).await;
-        self.runtime_facade.replace_targets(activation).await?;
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "reloaded",
-            "notify runtime lifecycle"
-        );
+        let read_store = store.clone();
+        let config_cache = self.config.clone();
+        let lifecycle = self.lifecycle.clone();
+        let transition = with_notify_server_config_read_lock(store, move || async move {
+            let config = crate::read_existing_notify_server_config_no_lock(read_store)
+                .await
+                .map_err(NotificationError::ReadConfig)?;
+            Ok::<_, NotificationError>(if *config_cache.read().await == config && lifecycle.is_converged() {
+                None
+            } else {
+                Some(lifecycle.update_config(config))
+            })
+        })
+        .await
+        .map_err(NotificationError::StorageNotAvailable)??;
+
+        if let Some(transition) = transition {
+            transition.wait().await?;
+        }
        Ok(())
    }

-    async fn update_config(&self, new_config: Config) {
-        let mut config = self.config.write().await;
-        *config = new_config;
-    }
-
-    async fn update_config_and_reload<F>(&self, mut modifier: F) -> Result<(), NotificationError>
+    async fn update_config_and_reload<F>(&self, modifier: F) -> Result<(), NotificationError>
    where
-        F: FnMut(&mut Config) -> bool,
+        F: FnMut(&mut Config) -> bool + Send + 'static,
    {
-        let Some(new_config) = update_server_config(&mut modifier).await.map_err(|err| match err {
-            NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable(
-                "Failed to save target configuration: server storage not initialized".to_string(),
-            ),
-            NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err),
-            NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err),
-        })?
+        if self.lifecycle.state() == NotificationRuntimeState::Terminated {
+            return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
+        }
+        let Some(transition) = update_server_config(modifier, self.lifecycle.clone())
+            .await
+            .map_err(|err| match err {
+                NotifyConfigStoreError::Lock(err) => NotificationError::StorageNotAvailable(err),
+                NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable(
+                    "Failed to save target configuration: server storage not initialized".to_string(),
+                ),
+                NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err),
+                NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err),
+            })?
        else {
            debug!(
                event = EVENT_NOTIFY_CONFIG_UPDATE,
@@ -443,15 +375,15 @@ impl NotifyConfigManager {
            result = "updated",
            "notify config update"
        );
-        self.reload_config(new_config).await
+        transition.wait().await
    }
 }

 #[cfg(test)]
 mod tests {
-    use super::{NotifyConfigManager, NotifyConfigStoreError, runtime_target_id_for_subsystem, serialized_read_modify_write};
-    use crate::NotificationError;
+    use super::{NotifyConfigManager, NotifyConfigStoreError, read_modify_write, runtime_target_id_for_subsystem};
    use crate::rules::RulesMap;
+    use crate::{NotificationError, NotificationRuntimeState};
    use crate::{
        integration::NotificationMetrics, notifier::EventNotifier, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
        runtime_facade::NotifyRuntimeFacade,
@@ -464,7 +396,10 @@ mod tests {
    use rustfs_s3_types::EventName;
    use rustfs_targets::ReplayWorkerManager;
    use rustfs_targets::arn::TargetID;
-    use std::sync::Arc;
+    use std::sync::{
+        Arc,
+        atomic::{AtomicBool, Ordering},
+    };
    use tokio::sync::{RwLock, Semaphore};

    fn build_manager() -> NotifyConfigManager {
@@ -474,9 +409,10 @@ mod tests {
        let rule_engine = NotifyRuleEngine::new();
        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
        let target_list = notifier.target_list();
-        let runtime_facade = NotifyRuntimeFacade::new(
+        let runtime_facade = NotifyRuntimeFacade::new_with_dispatch_gate(
            target_list,
            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
            Arc::new(Semaphore::new(4)),
            metrics,
        );
@@ -524,78 +460,39 @@ mod tests {
            .reload_config(Config::default())
            .await
            .expect("reload_config should succeed for empty targets");
+        assert!(matches!(manager.lifecycle().state(), NotificationRuntimeState::TargetsEnabled { .. }));
    }

-    // Regression test for backlog#968: the read-modify-write over the persisted
-    // notify config must be serialized. Many tasks concurrently add a distinct
-    // target through the same production RMW path (`serialized_read_modify_write`,
-    // which holds the global RMW lock across read→modify→write) against a shared
-    // in-memory backend. Every update must survive — no lost updates. Without the
-    // lock, concurrent tasks would read the same base config and clobber each
-    // other's writes, leaving only a subset of targets.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-    async fn concurrent_config_updates_preserve_all_targets() {
-        // Shared in-memory stand-in for the persisted config backend.
-        let backend = Arc::new(RwLock::new(Config::default()));
-        const TASKS: usize = 32;
+    #[tokio::test]
+    async fn read_modify_write_publishes_only_after_save() {
+        let saved = Arc::new(AtomicBool::new(false));
+        let saved_by_writer = saved.clone();
+        let observed_by_publisher = saved.clone();

-        let mut handles = Vec::with_capacity(TASKS);
-        for idx in 0..TASKS {
-            let backend = backend.clone();
-            handles.push(tokio::spawn(async move {
-                let ttype = NOTIFY_WEBHOOK_SUB_SYS.to_lowercase();
-                let tname = format!("target-{idx}");
-
-                let read_backend = backend.clone();
-                let save_backend = backend.clone();
-
-                let result = serialized_read_modify_write(
-                    |config: &mut Config| {
-                        config
-                            .0
-                            .entry(ttype.clone())
-                            .or_default()
-                            .insert(tname.clone(), KVS::default());
-                        true
-                    },
-                    move || async move {
-                        let snapshot = read_backend.read().await.clone();
-                        // Yield inside the critical section to widen the race window:
-                        // if the RMW were not serialized, other tasks would read this
-                        // same base snapshot and their writes would clobber ours.
-                        tokio::task::yield_now().await;
-                        Ok::<_, NotifyConfigStoreError>(snapshot)
-                    },
-                    move |config: Config| async move {
-                        *save_backend.write().await = config;
-                        Ok::<_, NotifyConfigStoreError>(())
-                    },
-                )
-                .await
-                .expect("serialized RMW should succeed");
-                assert!(result.is_some(), "modifier reported a change, expected Some(config)");
-            }));
-        }
-
-        for handle in handles {
-            handle.await.expect("update task should not panic");
-        }
-
-        let final_config = backend.read().await;
-        let webhook_targets = final_config
-            .0
-            .get(&NOTIFY_WEBHOOK_SUB_SYS.to_lowercase())
-            .expect("webhook subsystem should exist after updates");
-
-        assert_eq!(
-            webhook_targets.len(),
-            TASKS,
-            "all concurrent target additions must be preserved (no lost updates)"
-        );
-        for idx in 0..TASKS {
-            let tname = format!("target-{idx}");
-            assert!(webhook_targets.contains_key(&tname), "missing target {tname}: concurrent update was lost");
-        }
+        read_modify_write(
+            |config| {
+                config
+                    .0
+                    .entry(NOTIFY_WEBHOOK_SUB_SYS.to_string())
+                    .or_default()
+                    .insert("primary".to_string(), KVS::default());
+                true
+            },
+            || async { Ok::<_, NotifyConfigStoreError>(Config::default()) },
+            move |_config| async move {
+                saved_by_writer.store(true, Ordering::Release);
+                Ok::<_, NotifyConfigStoreError>(())
+            },
+            move |_config| {
+                assert!(
+                    observed_by_publisher.load(Ordering::Acquire),
+                    "publication must observe the completed save"
+                );
+            },
+        )
+        .await
+        .expect("read-modify-write should succeed")
+        .expect("changed config should publish");
    }

    #[test]
@@ -15,6 +15,7 @@
 use rustfs_targets::{TargetError, arn::TargetID};
 use std::io;
 use thiserror::Error;
+use tokio::task::JoinError;

 /// Errors related to the notification system's lifecycle.
 #[derive(Debug, Error)]
@@ -70,3 +71,24 @@ pub enum NotificationError {
    #[error("Storage not available: {0}")]
    StorageNotAvailable(String),
 }
+
+pub(crate) fn transition_join_error(error: JoinError) -> NotificationError {
+    let reason = if error.is_cancelled() { "cancelled" } else { "panicked" };
+    NotificationError::Initialization(format!("Notification lifecycle transition task {reason}"))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::transition_join_error;
+
+    #[tokio::test]
+    async fn transition_join_error_does_not_expose_panic_payload() {
+        let join_error = tokio::spawn(async { panic!("do-not-expose-payload") })
+            .await
+            .expect_err("test task should panic");
+
+        let rendered = transition_join_error(join_error).to_string();
+        assert!(rendered.contains("panicked"));
+        assert!(!rendered.contains("do-not-expose-payload"));
+    }
+}
@@ -14,38 +14,86 @@

 use crate::{
    BucketNotificationConfig, Event, EventArgs, LifecycleError, NotificationError, NotificationMetricSnapshot,
-    NotificationSystem, NotificationTargetMetricSnapshot,
+    NotificationSystem, NotificationTargetMetricSnapshot, error::transition_join_error,
 };
 use rustfs_config::server_config::Config;
 use rustfs_s3_types::EventName;
 use rustfs_targets::arn::TargetID;
-use std::sync::{Arc, OnceLock};
+use std::sync::{Arc, LazyLock, Mutex, OnceLock, Weak};
 use tracing::error;

 static NOTIFICATION_SYSTEM: OnceLock<Arc<NotificationSystem>> = OnceLock::new();
+static LEGACY_INITIALIZATION: LazyLock<Mutex<Option<LegacyInitialization>>> = LazyLock::new(|| Mutex::new(None));
+
+enum LegacyInitialization {
+    Initializing(Weak<NotificationSystem>),
+    Retryable(Weak<NotificationSystem>),
+    Initialized,
+}
 const LOG_COMPONENT_NOTIFY: &str = "notify";
 const LOG_SUBSYSTEM_GLOBAL: &str = "global";
 const EVENT_NOTIFY_GLOBAL_STATE: &str = "notify_global_state";

-/// Initialize the global notification system with the given configuration.
-/// This function should only be called once throughout the application life cycle.
-pub async fn initialize(config: Config) -> Result<(), NotificationError> {
-    // `new` is synchronous and responsible for creating instances
-    let system = NotificationSystem::new(config);
-    // `init` is asynchronous and responsible for performing I/O-intensive initialization
-    system.init().await?;
+fn notification_system_or_init(config: Config) -> Arc<NotificationSystem> {
+    NOTIFICATION_SYSTEM
+        .get_or_init(|| Arc::new(NotificationSystem::new(config)))
+        .clone()
+}

-    match NOTIFICATION_SYSTEM.set(Arc::new(system)) {
-        Ok(_) => Ok(()),
-        Err(losing_system) => {
-            // Another initializer won the race. `init()` above already started this
-            // system's targets and replay workers, so simply dropping it would leak
-            // those background tasks. Shut the losing instance down cleanly before
-            // reporting the conflict (backlog#984).
-            losing_system.shutdown().await;
-            Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized))
+/// Initialize the global notification system with the given configuration.
+///
+/// This preserves the historical one-shot API contract. Server lifecycle code
+/// that needs idempotent reconciliation should use [`reconcile`] instead.
+pub async fn initialize(config: Config) -> Result<(), NotificationError> {
+    let system = {
+        let mut legacy = LEGACY_INITIALIZATION.lock().unwrap_or_else(|err| err.into_inner());
+        match legacy.as_ref() {
+            Some(LegacyInitialization::Retryable(system)) => {
+                let Some(system) = system.upgrade() else {
+                    return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+                };
+                if !NOTIFICATION_SYSTEM.get().is_some_and(|global| Arc::ptr_eq(global, &system)) {
+                    return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+                }
+                *legacy = Some(LegacyInitialization::Initializing(Arc::downgrade(&system)));
+                system
+            }
+            Some(LegacyInitialization::Initializing(_)) | Some(LegacyInitialization::Initialized) => {
+                return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+            }
+            None => {
+                if NOTIFICATION_SYSTEM.get().is_some() {
+                    return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+                }
+                let system = Arc::new(NotificationSystem::new(config.clone()));
+                if NOTIFICATION_SYSTEM.set(system.clone()).is_err() {
+                    return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+                }
+                *legacy = Some(LegacyInitialization::Initializing(Arc::downgrade(&system)));
+                system
+            }
        }
-    }
+    };
+
+    let task_system = system.clone();
+    tokio::spawn(async move {
+        let result = task_system.set_targets_enabled(true, Some(config)).await;
+        let mut legacy = LEGACY_INITIALIZATION.lock().unwrap_or_else(|err| err.into_inner());
+        if matches!(
+            legacy.as_ref(),
+            Some(LegacyInitialization::Initializing(current))
+                if current.upgrade().is_some_and(|current| Arc::ptr_eq(&current, &task_system))
+        ) {
+            *legacy = Some(if result.is_ok() {
+                LegacyInitialization::Initialized
+            } else {
+                LegacyInitialization::Retryable(Arc::downgrade(&task_system))
+            });
+        }
+        result
+    })
+    .await
+    .map_err(transition_join_error)?
 }

 /// Initialize the global notification system only for live in-process consumers.
@@ -54,12 +102,24 @@ pub async fn initialize(config: Config) -> Result<(), NotificationError> {
 /// ListenBucketNotification clients can receive live events even when external
 /// notification targets are disabled.
 pub fn initialize_live_events() -> Result<(), NotificationError> {
-    let system = NotificationSystem::new(Config::new());
-
-    match NOTIFICATION_SYSTEM.set(Arc::new(system)) {
-        Ok(_) => Ok(()),
-        Err(_) => Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized)),
+    if NOTIFICATION_SYSTEM
+        .set(Arc::new(NotificationSystem::new(Config::new())))
+        .is_err()
+    {
+        return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
    }
+    Ok(())
+}
+
+/// Ensures the stable process-wide live-event container exists.
+pub fn ensure_live_events() -> Arc<NotificationSystem> {
+    notification_system_or_init(Config::new())
+}
+
+/// Ensures the stable singleton exists and reconciles its target runtime.
+pub async fn reconcile(config: Config) -> Result<(), NotificationError> {
+    let system = notification_system_or_init(config.clone());
+    system.set_targets_enabled(true, Some(config)).await
 }

 /// Returns a handle to the global NotificationSystem instance.
@@ -16,7 +16,12 @@ use crate::notification_system_subscriber::NotificationSystemSubscriberView;
 use crate::notifier::{EventNotifier, TargetList};
 use crate::services::NotifyServices;
 use crate::{
-    Event, error::NotificationError, pipeline::LiveEventHistory, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
+    Event,
+    error::NotificationError,
+    lifecycle::{NotificationLifecycleTransition, NotificationRuntimeState},
+    pipeline::LiveEventHistory,
+    registry::TargetRegistry,
+    rule_engine::NotifyRuleEngine,
    rules::BucketNotificationConfig,
 };
 use hashbrown::HashMap;
@@ -160,11 +165,12 @@ impl NotificationMetrics {

 /// The notification system that integrates all components
 pub struct NotificationSystem {
-    /// The event notifier
+    /// Event dispatcher. Runtime target mutation remains lifecycle-owned.
    pub notifier: Arc<EventNotifier>,
-    /// The target registry
+    /// Target factory registry. Creating a target does not publish it.
    pub registry: Arc<TargetRegistry>,
-    /// The current configuration
+    /// The current cached configuration. Runtime publication must still go
+    /// through the lifecycle methods on this type.
    pub config: Arc<RwLock<Config>>,
    services: NotifyServices,
 }
@@ -220,10 +226,12 @@ impl NotificationSystem {
        self.services.runtime_view.get_active_targets().await
    }

-    /// Gets the complete Target list, including both active and inactive Targets.
-    ///
-    /// # Return
-    /// An `Arc<RwLock<TargetList>>` containing all Targets.
+    pub async fn config_snapshot(&self) -> Config {
+        self.config.read().await.clone()
+    }
+
+    /// Gets a read-only runtime container handle. Public mutation methods on
+    /// `TargetList` are intentionally unavailable outside this crate.
    pub async fn get_all_targets(&self) -> Arc<RwLock<TargetList>> {
        self.services.runtime_view.get_all_targets()
    }
@@ -327,6 +335,43 @@ impl NotificationSystem {
        self.services.config_manager.reload_config(new_config).await
    }

+    /// Synchronously publishes a config generation without changing target mode.
+    pub fn publish_config(&self, new_config: Config) -> NotificationLifecycleTransition {
+        self.services.config_manager.lifecycle().update_config(new_config)
+    }
+
+    /// Reconciles the cached and active configuration with the persisted
+    /// server config without changing the target-runtime mode.
+    pub async fn reload_persisted_config(&self) -> Result<(), NotificationError> {
+        self.services.config_manager.reload_persisted_config().await
+    }
+
+    /// Reconciles from an explicitly selected storage context.
+    pub async fn reload_persisted_config_from_store(&self, store: Arc<crate::NotifyStore>) -> Result<(), NotificationError> {
+        self.services.config_manager.reload_persisted_config_from_store(store).await
+    }
+
+    /// Enables or suspends configured notification targets without replacing
+    /// the process-wide live-event container.
+    pub async fn set_targets_enabled(&self, enabled: bool, config: Option<Config>) -> Result<(), NotificationError> {
+        self.publish_targets_enabled(enabled, config).wait().await
+    }
+
+    /// Synchronously accepts a target-runtime mode transition. The returned
+    /// receipt can be awaited after the caller releases its persistence lock.
+    pub fn publish_targets_enabled(&self, enabled: bool, config: Option<Config>) -> NotificationLifecycleTransition {
+        self.services.config_manager.lifecycle().set_mode(enabled, config)
+    }
+
+    pub fn runtime_lifecycle_state(&self) -> NotificationRuntimeState {
+        self.services.config_manager.lifecycle().state()
+    }
+
+    /// Returns whether the latest accepted lifecycle generation is active.
+    pub fn runtime_lifecycle_is_converged(&self) -> bool {
+        self.services.config_manager.lifecycle().is_converged()
+    }
+
    /// Loads the bucket notification configuration
    pub async fn load_bucket_notification_config(
        &self,
@@ -365,9 +410,13 @@ impl NotificationSystem {
        self.services.runtime_view.runtime_status_snapshot().await
    }

-    // Add a method to shut down the system
    pub async fn shutdown(&self) {
-        self.services.runtime_facade.shutdown().await;
+        let _ = self.shutdown_checked().await;
+    }
+
+    /// Irreversibly terminates the notification target runtime for this process.
+    pub async fn shutdown_checked(&self) -> Result<(), NotificationError> {
+        self.services.config_manager.lifecycle().terminate().wait().await
    }
 }

@@ -25,6 +25,7 @@ mod event;
 pub mod factory;
 mod global;
 pub mod integration;
+mod lifecycle;
 mod notification_system_subscriber;
 pub mod notifier;
 mod pipeline;
@@ -42,10 +43,11 @@ pub use config_manager::{NotifyConfigManager, runtime_target_id_for_subsystem};
 pub use error::{LifecycleError, NotificationError};
 pub use event::{Event, EventArgs, EventArgsBuilder, NotifyObjectInfo};
 pub use global::{
-    initialize, initialize_live_events, is_notification_system_initialized, notification_metrics_snapshot, notification_system,
-    notification_target_metrics, notifier_global,
+    ensure_live_events, initialize, initialize_live_events, is_notification_system_initialized, notification_metrics_snapshot,
+    notification_system, notification_target_metrics, notifier_global, reconcile,
 };
 pub use integration::{NotificationMetricSnapshot, NotificationSystem, NotificationTargetMetricSnapshot};
+pub use lifecycle::{NotificationLifecycleTransition, NotificationRuntimeState};
 pub use pipeline::{LiveEventHistory, NotifyEventBridge, NotifyPipeline};
 pub use rule_engine::NotifyRuleEngine;
 pub use rules::BucketNotificationConfig;
@@ -53,6 +55,9 @@ pub use runtime_facade::NotifyRuntimeFacade;
 pub use runtime_view::NotifyRuntimeView;
 pub use services::NotifyServices;
 pub use status_view::NotifyStatusView;
+pub use storage_api::NotifyStore;
 pub(crate) use storage_api::crate_boundary::{
-    read_notify_server_config_without_migrate, resolve_notify_object_store_handle, save_notify_server_config,
+    read_existing_notify_server_config_no_lock, read_notify_server_config_without_migrate_no_lock,
+    resolve_notify_object_store_handle, save_notify_server_config_no_lock, with_notify_server_config_read_lock,
+    with_notify_server_config_write_lock,
 };
@@ -12,13 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use crate::{error::NotificationError, event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine};
+use crate::error::NotificationError;
+use crate::{event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine};
 use rustfs_config::notify::{DEFAULT_NOTIFY_SEND_CONCURRENCY, ENV_NOTIFY_SEND_CONCURRENCY};
+use rustfs_targets::Target;
 use rustfs_targets::arn::TargetID;
 use rustfs_targets::target::EntityTarget;
-use rustfs_targets::{SharedTarget, Target, TargetRuntimeManager};
-use std::sync::Arc;
-use tokio::sync::{RwLock, Semaphore};
+use rustfs_targets::{SharedTarget, TargetRuntimeManager};
+use std::collections::HashMap;
+use std::sync::{Arc, LazyLock, Mutex as StdMutex, Weak};
+use tokio::sync::{RwLock, Semaphore, watch};
+use tokio::task::JoinHandle;
+use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, instrument, warn};

 const LOG_COMPONENT_NOTIFY: &str = "notify";
@@ -29,8 +34,101 @@ const EVENT_NOTIFY_DISPATCH_STARTED: &str = "notify_dispatch_started";
 const EVENT_NOTIFY_DISPATCH_COMPLETED: &str = "notify_dispatch_completed";
 const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";

+async fn wait_for_dispatch_tasks(handles: Vec<JoinHandle<()>>) {
+    for handle in handles {
+        if let Err(e) = handle.await {
+            let reason = if e.is_cancelled() { "join_cancelled" } else { "join_panicked" };
+            error!(
+                event = EVENT_NOTIFY_DISPATCH_FAILED,
+                component = LOG_COMPONENT_NOTIFY,
+                subsystem = LOG_SUBSYSTEM_DISPATCH,
+                reason,
+                "Notify dispatch task failed"
+            );
+        }
+    }
+}
+
 pub type SharedNotifyTargetList = Arc<RwLock<TargetList>>;

+static TARGET_LIST_DISPATCH_GATES: LazyLock<StdMutex<HashMap<usize, Weak<RwLock<()>>>>> =
+    LazyLock::new(|| StdMutex::new(HashMap::new()));
+
+pub(crate) fn shared_dispatch_gate(target_list: &SharedNotifyTargetList, preferred: Option<Arc<RwLock<()>>>) -> Arc<RwLock<()>> {
+    let key = Arc::as_ptr(target_list) as usize;
+    let mut gates = TARGET_LIST_DISPATCH_GATES.lock().unwrap_or_else(|err| err.into_inner());
+    gates.retain(|_, gate| gate.strong_count() != 0);
+    if let Some(gate) = gates.get(&key).and_then(Weak::upgrade) {
+        return gate;
+    }
+    let gate = preferred.unwrap_or_else(|| Arc::new(RwLock::new(())));
+    gates.insert(key, Arc::downgrade(&gate));
+    gate
+}
+
+pub(crate) struct DirectDispatchTracker {
+    cancellation: CancellationToken,
+    inflight: watch::Sender<usize>,
+}
+
+impl DirectDispatchTracker {
+    fn new() -> Self {
+        let (inflight, _) = watch::channel(0);
+        Self {
+            cancellation: CancellationToken::new(),
+            inflight,
+        }
+    }
+
+    fn acquire(self: &Arc<Self>) -> DirectDispatchLease {
+        self.inflight.send_modify(|count| *count += 1);
+        DirectDispatchLease {
+            tracker: Arc::clone(self),
+        }
+    }
+
+    pub(crate) async fn wait_idle(&self) {
+        let mut inflight = self.inflight.subscribe();
+        while *inflight.borrow_and_update() != 0 {
+            if inflight.changed().await.is_err() {
+                return;
+            }
+        }
+    }
+
+    fn cancel_pending(&self) {
+        self.cancellation.cancel();
+    }
+}
+
+struct DirectDispatchLease {
+    tracker: Arc<DirectDispatchTracker>,
+}
+
+impl DirectDispatchLease {
+    fn cancellation(&self) -> CancellationToken {
+        self.tracker.cancellation.clone()
+    }
+}
+
+impl Drop for DirectDispatchLease {
+    fn drop(&mut self) {
+        self.tracker.inflight.send_modify(|count| {
+            if let Some(next) = count.checked_sub(1) {
+                *count = next;
+            } else {
+                error!(
+                    event = EVENT_NOTIFY_DISPATCH_FAILED,
+                    component = LOG_COMPONENT_NOTIFY,
+                    subsystem = LOG_SUBSYSTEM_DISPATCH,
+                    reason = "direct_lease_underflow",
+                    "Notify direct dispatch lease accounting underflowed"
+                );
+            }
+        });
+    }
+}
+
 /// Resolves the effective send concurrency (semaphore permit count).
 ///
 /// A value of `0` would build a zero-permit semaphore, so `acquire` never
@@ -63,6 +161,8 @@ fn coerce_send_concurrency(configured: usize) -> usize {

 /// Manages event notification to targets based on rules
 pub struct EventNotifier {
+    dispatch_gate: Arc<RwLock<()>>,
+    enqueue_limiter: Arc<Semaphore>,
    metrics: Arc<NotificationMetrics>,
    rule_engine: NotifyRuleEngine,
    target_list: SharedNotifyTargetList,
@@ -82,10 +182,14 @@ impl EventNotifier {
    /// Returns a new instance of EventNotifier.
    pub fn new(metrics: Arc<NotificationMetrics>, rule_engine: NotifyRuleEngine) -> Self {
        let max_inflight = resolve_send_concurrency();
+        let target_list = Arc::new(RwLock::new(TargetList::new()));
+        let dispatch_gate = shared_dispatch_gate(&target_list, None);
        EventNotifier {
+            dispatch_gate,
+            enqueue_limiter: Arc::new(Semaphore::new(max_inflight)),
            metrics,
            rule_engine,
-            target_list: Arc::new(RwLock::new(TargetList::new())),
+            target_list,
            send_limiter: Arc::new(Semaphore::new(max_inflight)),
        }
    }
@@ -99,6 +203,10 @@ impl EventNotifier {
        Arc::clone(&self.target_list)
    }

+    pub(crate) fn dispatch_gate(&self) -> Arc<RwLock<()>> {
+        self.dispatch_gate.clone()
+    }
+
    /// Returns a list of ARNs for the registered targets
    ///
    /// # Arguments
@@ -115,12 +223,9 @@ impl EventNotifier {
            .collect()
    }

-    /// Removes all targets
    pub async fn remove_all_bucket_targets(&self) {
        let mut target_list_guard = self.target_list.write().await;
-        // The logic for sending cancel signals via stream_cancel_senders would be removed.
-        // TargetList::clear_targets_only already handles calling target.close().
-        target_list_guard.clear_targets_only().await; // Modified clear to not re-cancel
+        target_list_guard.clear_targets_only().await;
        info!(
            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
            component = LOG_COMPONENT_NOTIFY,
@@ -155,7 +260,14 @@ impl EventNotifier {
            return;
        }
        let target_ids_len = target_ids.len();
-        let mut handles = vec![];
+        let mut deferred_handles = Vec::new();
+        let mut direct_handles = Vec::new();
+
+        // A lifecycle writer holds this gate only while handing queue-store
+        // ownership from one runtime generation to the next. Taking the read
+        // guard before cloning targets means the writer both blocks new sends
+        // and drains every save already using the old generation.
+        let dispatch_guard = Arc::new(self.dispatch_gate.clone().read_owned().await);

        // Use scope to limit the borrow scope of target_list
        let target_list_guard = self.target_list.read().await;
@@ -185,10 +297,17 @@ impl EventNotifier {
                    );
                    continue;
                }
-                let limiter = self.send_limiter.clone();
+                let is_deferred = target_for_task.store().is_some();
+                let direct_dispatch_lease = (!is_deferred).then(|| target_list_guard.direct_dispatch_lease());
+                let direct_cancellation = direct_dispatch_lease.as_ref().map(DirectDispatchLease::cancellation);
+                let deferred_dispatch_guard = is_deferred.then(|| Arc::clone(&dispatch_guard));
+                let limiter = if is_deferred {
+                    self.enqueue_limiter.clone()
+                } else {
+                    self.send_limiter.clone()
+                };
                let metrics = self.metrics.clone();
                let event_clone = event.clone();
-                let is_deferred = target_for_task.store().is_some();
                let target_name_for_task = target_for_task.name(); // Get the name before generating the task
                debug!(
                    event = EVENT_NOTIFY_DISPATCH_STARTED,
@@ -207,9 +326,32 @@ impl EventNotifier {
                    data: event_clone.as_ref().clone(),
                });
                let handle = tokio::spawn(async move {
+                    let _direct_dispatch_lease = direct_dispatch_lease;
+                    let _deferred_dispatch_guard = deferred_dispatch_guard;
                    metrics.increment_processing();
-                    let _permit = match limiter.acquire_owned().await {
-                        Ok(p) => p,
+                    let permit = if let Some(cancellation) = direct_cancellation {
+                        tokio::select! {
+                            biased;
+                            _ = cancellation.cancelled() => {
+                                metrics.decrement_processing();
+                                metrics.increment_skipped();
+                                debug!(
+                                    event = EVENT_NOTIFY_DISPATCH_SKIPPED,
+                                    component = LOG_COMPONENT_NOTIFY,
+                                    subsystem = LOG_SUBSYSTEM_DISPATCH,
+                                    target_id = %target_name_for_task,
+                                    reason = "runtime_generation_replaced",
+                                    "Skipped pending direct notify dispatch"
+                                );
+                                return;
+                            }
+                            permit = limiter.acquire_owned() => permit,
+                        }
+                    } else {
+                        limiter.acquire_owned().await
+                    };
+                    let _permit = match permit {
+                        Ok(permit) => permit,
                        Err(e) => {
                            error!(
                                event = EVENT_NOTIFY_DISPATCH_FAILED,
@@ -218,7 +360,7 @@ impl EventNotifier {
                                target_id = %target_name_for_task,
                                error = %e,
                                reason = "permit_acquire_failed",
-                                "Failed to acquire notify send permit"
+                                "Failed to acquire notify dispatch permit"
                            );
                            metrics.increment_failed();
                            return;
@@ -260,7 +402,11 @@ impl EventNotifier {
                        );
                    }
                });
-                handles.push(handle);
+                if is_deferred {
+                    deferred_handles.push(handle);
+                } else {
+                    direct_handles.push(handle);
+                }
            } else {
                warn!(
                    event = EVENT_NOTIFY_DISPATCH_FAILED,
@@ -276,19 +422,13 @@ impl EventNotifier {
        // target_list is automatically released here
        drop(target_list_guard);

-        // Wait for all tasks to be completed
-        for handle in handles {
-            if let Err(e) = handle.await {
-                error!(
-                    event = EVENT_NOTIFY_DISPATCH_FAILED,
-                    component = LOG_COMPONENT_NOTIFY,
-                    subsystem = LOG_SUBSYSTEM_DISPATCH,
-                    error = %e,
-                    reason = "join_failed",
-                    "Notify dispatch task failed"
-                );
-            }
-        }
+        // Every store-backed save owns a share of the generation guard, so
+        // caller cancellation cannot race lifecycle handoff with an enqueue.
+        // Direct targets own no queue store and may finish against the detached
+        // target while lifecycle progresses.
+        drop(dispatch_guard);
+        wait_for_dispatch_tasks(deferred_handles).await;
+        wait_for_dispatch_tasks(direct_handles).await;
        debug!(
            event = EVENT_NOTIFY_DISPATCH_COMPLETED,
            component = LOG_COMPONENT_NOTIFY,
@@ -320,7 +460,7 @@ impl EventNotifier {
            target_list_guard.add(target)?;
        }

-        info!(
+        tracing::info!(
            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
            component = LOG_COMPONENT_NOTIFY,
            subsystem = LOG_SUBSYSTEM_DISPATCH,
@@ -334,6 +474,7 @@ impl EventNotifier {

 /// A thread-safe list of targets
 pub struct TargetList {
+    direct_dispatches: Arc<DirectDispatchTracker>,
    /// Map of TargetID to Target
    runtime: TargetRuntimeManager<Event>,
 }
@@ -348,6 +489,7 @@ impl TargetList {
    /// Creates a new TargetList
    pub fn new() -> Self {
        TargetList {
+            direct_dispatches: Arc::new(DirectDispatchTracker::new()),
            runtime: TargetRuntimeManager::new(),
        }
    }
@@ -435,6 +577,20 @@ impl TargetList {
        self.runtime.status_snapshot(replay_workers)
    }

+    fn direct_dispatch_lease(&self) -> DirectDispatchLease {
+        self.direct_dispatches.acquire()
+    }
+
+    pub(crate) fn replace_runtime(
+        &mut self,
+        replacement: TargetRuntimeManager<Event>,
+    ) -> (TargetRuntimeManager<Event>, Arc<DirectDispatchTracker>) {
+        let runtime = std::mem::replace(&mut self.runtime, replacement);
+        let direct_dispatches = std::mem::replace(&mut self.direct_dispatches, Arc::new(DirectDispatchTracker::new()));
+        direct_dispatches.cancel_pending();
+        (runtime, direct_dispatches)
+    }
+
    pub fn runtime_mut(&mut self) -> &mut TargetRuntimeManager<Event> {
        &mut self.runtime
    }
@@ -458,7 +614,7 @@ mod tests {
    use rustfs_s3_types::EventName;
    use rustfs_targets::StoreError;
    use rustfs_targets::{
-        TargetError,
+        ReplayWorkerManager, TargetError,
        store::{Key, QueueStore, Store},
        target::{EntityTarget, QueuedPayload, QueuedPayloadMeta},
    };
@@ -466,6 +622,7 @@ mod tests {
        Arc,
        atomic::{AtomicUsize, Ordering},
    };
+    use tokio::sync::Notify;

    #[tokio::test]
    async fn encoded_event_key_matches_raw_prefix_suffix_filter() {
@@ -525,19 +682,44 @@ mod tests {

    #[derive(Clone)]
    struct TestTarget {
+        block_first_save: Option<(Arc<Notify>, Arc<Notify>)>,
+        close_calls: Arc<AtomicUsize>,
+        close_entered: Option<Arc<Notify>>,
        id: TargetID,
        enabled: bool,
        save_calls: Arc<AtomicUsize>,
+        selected_calls: Arc<AtomicUsize>,
+        store: Option<QueueStore<QueuedPayload>>,
    }

    impl TestTarget {
        fn new(id: &str, name: &str, enabled: bool) -> Self {
            Self {
+                block_first_save: None,
+                close_calls: Arc::new(AtomicUsize::new(0)),
+                close_entered: None,
                id: TargetID::new(id.to_string(), name.to_string()),
                enabled,
                save_calls: Arc::new(AtomicUsize::new(0)),
+                selected_calls: Arc::new(AtomicUsize::new(0)),
+                store: None,
            }
        }
+
+        fn with_blocked_first_save(mut self, entered: Arc<Notify>, release: Arc<Notify>) -> Self {
+            self.block_first_save = Some((entered, release));
+            self
+        }
+
+        fn with_store(mut self, store: QueueStore<QueuedPayload>) -> Self {
+            self.store = Some(store);
+            self
+        }
+
+        fn with_close_observer(mut self, close_entered: Arc<Notify>) -> Self {
+            self.close_entered = Some(close_entered);
+            self
+        }
    }

    #[async_trait]
@@ -554,7 +736,13 @@ mod tests {
        }

        async fn save(&self, _event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
-            self.save_calls.fetch_add(1, Ordering::SeqCst);
+            let call = self.save_calls.fetch_add(1, Ordering::SeqCst);
+            if call == 0
+                && let Some((entered, release)) = &self.block_first_save
+            {
+                entered.notify_one();
+                release.notified().await;
+            }
            Ok(())
        }

@@ -563,11 +751,17 @@ mod tests {
        }

        async fn close(&self) -> Result<(), TargetError> {
+            self.close_calls.fetch_add(1, Ordering::SeqCst);
+            if let Some(close_entered) = &self.close_entered {
+                close_entered.notify_one();
+            }
            Ok(())
        }

        fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
-            None
+            self.store
+                .as_ref()
+                .map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
        }

        fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
@@ -580,10 +774,425 @@ mod tests {
        }

        fn is_enabled(&self) -> bool {
+            self.selected_calls.fetch_add(1, Ordering::SeqCst);
            self.enabled
        }
    }

+    #[tokio::test]
+    async fn lifecycle_pause_drains_entered_deferred_dispatch_and_blocks_new_dispatch() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
+        let save_entered = Arc::new(Notify::new());
+        let save_release = Arc::new(Notify::new());
+        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
+        let target = TestTarget::new("gated-target", "webhook", true)
+            .with_blocked_first_save(save_entered.clone(), save_release.clone())
+            .with_store(QueueStore::new(queue_dir.path(), 16, ".event"));
+
+        let mut rules_map = RulesMap::new();
+        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+        notifier
+            .target_list()
+            .write()
+            .await
+            .add(Arc::new(target.clone()))
+            .expect("target install should succeed");
+
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics,
+        );
+        let first_dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "first", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        save_entered.notified().await;
+        assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
+
+        let mut pause = Box::pin(facade.pause_dispatch());
+        tokio::select! {
+            biased;
+            _ = &mut pause => panic!("lifecycle pause crossed an in-flight dispatch"),
+            _ = std::future::ready(()) => {}
+        }
+
+        save_release.notify_one();
+        first_dispatch.await.expect("first dispatch task should finish");
+        let pause_guard = pause.await;
+
+        let replacement = TestTarget::new("gated-target", "webhook", true);
+        {
+            let target_list = notifier.target_list();
+            let mut target_list = target_list.write().await;
+            target_list.clear();
+            target_list
+                .add(Arc::new(replacement.clone()))
+                .expect("replacement target install should succeed");
+        }
+
+        let mut second_dispatch =
+            Box::pin(notifier.send(Arc::new(Event::new_test_event("bucket", "second", EventName::ObjectCreatedPut))));
+        tokio::select! {
+            biased;
+            _ = &mut second_dispatch => panic!("dispatch crossed the lifecycle pause"),
+            _ = std::future::ready(()) => {}
+        }
+        assert_eq!(
+            replacement.selected_calls.load(Ordering::SeqCst),
+            0,
+            "a paused dispatch must not select a target from the replacement generation early"
+        );
+        assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(replacement.save_calls.load(Ordering::SeqCst), 0);
+
+        drop(pause_guard);
+        second_dispatch.await;
+        assert_eq!(target.selected_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(replacement.selected_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(replacement.save_calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn lifecycle_pause_does_not_wait_for_direct_network_dispatch() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
+        let save_entered = Arc::new(Notify::new());
+        let save_release = Arc::new(Notify::new());
+        let target =
+            TestTarget::new("direct-target", "webhook", true).with_blocked_first_save(save_entered.clone(), save_release.clone());
+
+        let mut rules_map = RulesMap::new();
+        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+        notifier
+            .target_list()
+            .write()
+            .await
+            .add(Arc::new(target))
+            .expect("target install should succeed");
+
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics,
+        );
+        let dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        save_entered.notified().await;
+
+        let mut pause = Box::pin(facade.pause_dispatch());
+        let pause_guard = tokio::select! {
+            biased;
+            guard = &mut pause => guard,
+            _ = std::future::ready(()) => panic!("a direct network send blocked lifecycle handoff"),
+        };
+        drop(pause_guard);
+        save_release.notify_one();
+        dispatch.await.expect("direct dispatch should finish after release");
+    }
+
+    #[tokio::test]
+    async fn replacement_cancels_permit_waiting_direct_dispatch_before_closing_target() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier {
+            send_limiter: Arc::new(Semaphore::new(1)),
+            ..EventNotifier::new(metrics.clone(), rule_engine.clone())
+        });
+        let first_entered = Arc::new(Notify::new());
+        let first_release = Arc::new(Notify::new());
+        let close_entered = Arc::new(Notify::new());
+        let target = TestTarget::new("direct-target", "webhook", true)
+            .with_blocked_first_save(first_entered.clone(), first_release.clone())
+            .with_close_observer(close_entered);
+
+        let mut rules_map = RulesMap::new();
+        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+        notifier
+            .target_list()
+            .write()
+            .await
+            .add(Arc::new(target.clone()))
+            .expect("target should install");
+
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics.clone(),
+        );
+        let first = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "first", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        first_entered.notified().await;
+
+        // This task selects the old generation and acquires its lease before
+        // waiting for the saturated direct-send permit.
+        let second = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "second", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        tokio::time::timeout(std::time::Duration::from_secs(1), async {
+            while target.selected_calls.load(Ordering::SeqCst) != 2 {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("both direct sends should select the old generation");
+
+        let activation = facade.activate_targets_with_replay(Vec::new()).await;
+        let mut replace = Box::pin(facade.replace_targets(activation));
+        tokio::select! {
+            biased;
+            result = &mut replace => panic!("replacement closed a generation with selected direct sends: {result:?}"),
+            _ = std::future::ready(()) => {}
+        }
+        assert_eq!(target.close_calls.load(Ordering::SeqCst), 0);
+
+        first_release.notify_one();
+        first.await.expect("first direct dispatch should finish");
+        second.await.expect("permit-waiting direct dispatch should be cancelled");
+        replace.await.expect("replacement should close after direct leases drain");
+        assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(target.close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(metrics.processing_count(), 0);
+        assert_eq!(metrics.processed_count(), 1);
+        assert_eq!(metrics.skipped_count(), 1);
+    }
+
+    #[tokio::test]
+    async fn caller_abort_does_not_release_deferred_generation_lease() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
+        let save_entered = Arc::new(Notify::new());
+        let save_release = Arc::new(Notify::new());
+        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
+        let target = TestTarget::new("deferred", "webhook", true)
+            .with_blocked_first_save(save_entered.clone(), save_release.clone())
+            .with_store(QueueStore::new(queue_dir.path(), 16, ".event"));
+
+        let mut rules_map = RulesMap::new();
+        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+        notifier
+            .target_list()
+            .write()
+            .await
+            .add(Arc::new(target))
+            .expect("target should install");
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics,
+        );
+
+        let dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        save_entered.notified().await;
+        dispatch.abort();
+        let _ = dispatch.await;
+
+        let mut pause = Box::pin(facade.pause_dispatch());
+        tokio::select! {
+            biased;
+            _ = &mut pause => panic!("caller abort released the deferred generation lease"),
+            _ = std::future::ready(()) => {}
+        }
+        save_release.notify_one();
+        let pause_guard = pause.await;
+        drop(pause_guard);
+    }
+
+    #[tokio::test]
+    async fn deferred_enqueue_concurrency_is_bounded() {
+        const LIMIT: usize = 2;
+        const TARGETS: usize = 3;
+
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier {
+            enqueue_limiter: Arc::new(Semaphore::new(LIMIT)),
+            ..EventNotifier::new(metrics, rule_engine.clone())
+        });
+        let entered = Arc::new(Notify::new());
+        let release = Arc::new(Notify::new());
+        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
+        let mut targets = Vec::new();
+        let mut rules_map = RulesMap::new();
+        for index in 0..TARGETS {
+            let target = TestTarget::new(&format!("deferred-{index}"), "webhook", true)
+                .with_blocked_first_save(entered.clone(), release.clone())
+                .with_store(QueueStore::new(queue_dir.path().join(index.to_string()), 16, ".event"));
+            rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+            notifier
+                .target_list()
+                .write()
+                .await
+                .add(Arc::new(target.clone()))
+                .expect("target should install");
+            targets.push(target);
+        }
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+
+        let dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        let total_calls = || {
+            targets
+                .iter()
+                .map(|target| target.save_calls.load(Ordering::SeqCst))
+                .sum::<usize>()
+        };
+        tokio::time::timeout(std::time::Duration::from_secs(1), async {
+            while total_calls() != LIMIT {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("the configured number of enqueues should enter");
+        for _ in 0..10 {
+            tokio::task::yield_now().await;
+        }
+        assert_eq!(total_calls(), LIMIT, "enqueue concurrency exceeded its semaphore capacity");
+
+        release.notify_waiters();
+        tokio::time::timeout(std::time::Duration::from_secs(1), async {
+            while total_calls() != TARGETS {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("the waiting enqueue should enter after a permit is released");
+        release.notify_waiters();
+        dispatch.await.expect("all bounded enqueues should finish");
+    }
+
+    #[tokio::test]
+    async fn deferred_enqueue_does_not_wait_for_a_blocked_direct_send_permit() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier {
+            send_limiter: Arc::new(Semaphore::new(1)),
+            ..EventNotifier::new(metrics.clone(), rule_engine.clone())
+        });
+        let direct_entered = Arc::new(Notify::new());
+        let direct_release = Arc::new(Notify::new());
+        let direct =
+            TestTarget::new("direct", "webhook", true).with_blocked_first_save(direct_entered.clone(), direct_release.clone());
+        let deferred_entered = Arc::new(Notify::new());
+        let deferred_release = Arc::new(Notify::new());
+        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
+        let deferred = TestTarget::new("deferred", "webhook", true)
+            .with_blocked_first_save(deferred_entered.clone(), deferred_release.clone())
+            .with_store(QueueStore::new(queue_dir.path(), 16, ".event"));
+
+        let mut direct_rules = RulesMap::new();
+        direct_rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), direct.id.clone());
+        rule_engine.set_bucket_rules("direct-bucket", direct_rules).await;
+        let mut deferred_rules = RulesMap::new();
+        deferred_rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), deferred.id.clone());
+        rule_engine.set_bucket_rules("deferred-bucket", deferred_rules).await;
+        {
+            let target_list = notifier.target_list();
+            let mut target_list = target_list.write().await;
+            target_list.add(Arc::new(direct)).expect("direct target should install");
+            target_list.add(Arc::new(deferred)).expect("deferred target should install");
+        }
+
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics,
+        );
+        let direct_dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("direct-bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        direct_entered.notified().await;
+        let deferred_dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("deferred-bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        tokio::time::timeout(std::time::Duration::from_secs(1), deferred_entered.notified())
+            .await
+            .expect("queue persistence must not wait behind a direct network send permit");
+
+        let mut pause = Box::pin(facade.pause_dispatch());
+        tokio::select! {
+            biased;
+            _ = &mut pause => panic!("lifecycle pause crossed the blocked deferred enqueue"),
+            _ = std::future::ready(()) => {}
+        }
+        deferred_release.notify_one();
+        deferred_dispatch
+            .await
+            .expect("deferred dispatch should finish after release");
+        let pause_guard = tokio::select! {
+            biased;
+            guard = &mut pause => guard,
+            _ = std::future::ready(()) => panic!("direct network delivery kept the lifecycle gate locked"),
+        };
+        drop(pause_guard);
+
+        direct_release.notify_one();
+        direct_dispatch.await.expect("direct dispatch should finish after release");
+    }
+
    #[tokio::test]
    async fn test_send_event_skips_disabled_target() {
        let rule_engine = NotifyRuleEngine::new();
@@ -38,6 +38,11 @@ impl TargetRegistry {
        TargetRegistry { plugins }
    }

+    #[cfg(test)]
+    pub(crate) fn with_plugins(plugins: TargetPluginRegistry<Event>) -> Self {
+        Self { plugins }
+    }
+
    pub fn supports_target_type(&self, target_type: &str) -> bool {
        self.plugins.supports_target_type(target_type)
    }
@@ -67,6 +72,15 @@ impl TargetRegistry {
    ) -> Result<Vec<Box<dyn Target<Event> + Send + Sync>>, TargetError> {
        self.plugins.create_targets_from_config(config, NOTIFY_ROUTE_PREFIX).await
    }
+
+    pub(crate) async fn create_dormant_targets_from_config(
+        &self,
+        config: &Config,
+    ) -> Result<(Vec<Box<dyn Target<Event> + Send + Sync>>, Vec<String>), TargetError> {
+        self.plugins
+            .create_dormant_targets_from_config(config, NOTIFY_ROUTE_PREFIX)
+            .await
+    }
 }

 #[cfg(test)]
@@ -12,13 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use crate::{Event, NotificationError, integration::NotificationMetrics, notifier::SharedNotifyTargetList};
-use rustfs_targets::{
-    BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter, ReplayEvent, ReplayWorkerManager, RuntimeActivation, Target,
+use crate::{
+    Event, NotificationError,
+    error::transition_join_error,
+    integration::NotificationMetrics,
+    notifier::{DirectDispatchTracker, SharedNotifyTargetList, shared_dispatch_gate},
+};
+use rustfs_targets::{
+    BuiltinPluginRuntimeAdapter, OpenedActivation, PluginRuntimeAdapter, PreparedActivation, ReplayEvent, ReplayWorkerManager,
+    RuntimeActivation, Target, TargetRuntimeManager,
+};
+use std::sync::{
+    Arc,
+    atomic::{AtomicBool, Ordering},
 };
-use std::sync::Arc;
 use std::time::Duration;
-use tokio::sync::{RwLock, Semaphore};
+use tokio::sync::{OwnedRwLockWriteGuard, RwLock, Semaphore};
+use tokio_util::sync::CancellationToken;
 use tracing::{debug, info, warn};

 const LOG_COMPONENT_NOTIFY: &str = "notify";
@@ -26,12 +36,25 @@ const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
 const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
 const EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED: &str = "notify_runtime_shutdown_failed";
 const EVENT_NOTIFY_REPLAY_RETRY_EXHAUSTED: &str = "notify_replay_retry_exhausted";
+const TARGET_CLOSE_TIMEOUT: Duration = Duration::from_secs(10);

+pub(crate) struct DetachedNotifyRuntime {
+    direct_dispatches: Arc<DirectDispatchTracker>,
+    runtime: TargetRuntimeManager<Event>,
+    replay_workers: ReplayWorkerManager,
+}
+
+// Multi-lock publication order: replay_workers -> target_list ->
+// publication_gate. The lifecycle may already hold dispatch_gate while
+// entering this facade; no path that needs these three locks may acquire
+// publication_gate before either runtime lock.
 #[derive(Clone)]
 pub struct NotifyRuntimeFacade {
+    dispatch_gate: Arc<RwLock<()>>,
+    legacy_terminated: Arc<AtomicBool>,
    target_list: SharedNotifyTargetList,
    replay_workers: Arc<RwLock<ReplayWorkerManager>>,
-    runtime_adapter: Arc<dyn PluginRuntimeAdapter<Event>>,
+    runtime_adapter: Arc<BuiltinPluginRuntimeAdapter<Event>>,
 }

 impl NotifyRuntimeFacade {
@@ -41,6 +64,18 @@ impl NotifyRuntimeFacade {
        concurrency_limiter: Arc<Semaphore>,
        metrics: Arc<NotificationMetrics>,
    ) -> Self {
+        let dispatch_gate = shared_dispatch_gate(&target_list, None);
+        Self::new_with_dispatch_gate(target_list, replay_workers, dispatch_gate, concurrency_limiter, metrics)
+    }
+
+    pub(crate) fn new_with_dispatch_gate(
+        target_list: SharedNotifyTargetList,
+        replay_workers: Arc<RwLock<ReplayWorkerManager>>,
+        dispatch_gate: Arc<RwLock<()>>,
+        concurrency_limiter: Arc<Semaphore>,
+        metrics: Arc<NotificationMetrics>,
+    ) -> Self {
+        let dispatch_gate = shared_dispatch_gate(&target_list, Some(dispatch_gate));
        let replay_metrics = metrics;
        let runtime_adapter = BuiltinPluginRuntimeAdapter::new(
            Arc::new(move |event: ReplayEvent<Event>| {
@@ -97,36 +132,166 @@ impl NotifyRuntimeFacade {
        );

        Self {
+            dispatch_gate,
+            legacy_terminated: Arc::new(AtomicBool::new(false)),
            target_list,
            replay_workers,
            runtime_adapter: Arc::new(runtime_adapter),
        }
    }

+    pub(crate) async fn pause_dispatch(&self) -> OwnedRwLockWriteGuard<()> {
+        self.dispatch_gate.clone().write_owned().await
+    }
+
    pub async fn activate_targets_with_replay(
        &self,
        targets: Vec<Box<dyn Target<Event> + Send + Sync>>,
    ) -> RuntimeActivation<Event> {
-        self.runtime_adapter.activate_with_replay(targets).await
+        // The compatibility pair must not start replacement replay before
+        // replace_targets has stopped and joined the current generation.
+        self.runtime_adapter.prepare_dormant_compat_activation(targets).await
    }

-    pub async fn replace_targets(&self, activation: RuntimeActivation<Event>) -> Result<(), NotificationError> {
-        // Lock order: replay_workers -> target_list (matches notify AGENTS.md).
+    pub(crate) async fn prepare_targets(
+        &self,
+        targets: Vec<Box<dyn Target<Event> + Send + Sync>>,
+        cancellation: &CancellationToken,
+    ) -> PreparedActivation<Event> {
+        self.runtime_adapter.prepare_targets_cancellable(targets, cancellation).await
+    }
+
+    pub(crate) async fn open_prepared_stores(
+        &self,
+        prepared: PreparedActivation<Event>,
+    ) -> Result<(OpenedActivation<Event>, PreparedActivation<Event>), NotificationError> {
+        let runtime_adapter = self.runtime_adapter.clone();
+        tokio::task::spawn_blocking(move || runtime_adapter.open_prepared_stores(prepared))
+            .await
+            .map_err(transition_join_error)
+    }
+
+    pub(crate) async fn commit_prepared<Committed>(
+        &self,
+        opened: OpenedActivation<Event>,
+        on_committed: Committed,
+    ) -> (DetachedNotifyRuntime, PreparedActivation<Event>)
+    where
+        Committed: FnOnce(bool),
+    {
+        // The lifecycle coordinator validates the generation immediately before
+        // entering the non-cancellable handoff. Once old replay workers have
+        // been joined, this generation must publish before a later accepted
+        // intent can run; abandoning it here would leave the old runtime
+        // visible without replay workers.
        let mut replay_workers = self.replay_workers.write().await;
        let mut target_list = self.target_list.write().await;
-        self.runtime_adapter
-            .replace_runtime_targets(target_list.runtime_mut(), &mut replay_workers, activation)
-            .await
-            .map_err(NotificationError::Target)?;
+
+        let (activation, rejected) = self.runtime_adapter.try_activate_prepared(opened);
+        let fully_activated = rejected.failure_summary().is_none();
+        let (runtime, replay_workers, direct_dispatches) =
+            Self::swap_activation(&mut target_list, &mut replay_workers, activation);
+        on_committed(fully_activated);
+        (
+            DetachedNotifyRuntime {
+                direct_dispatches,
+                runtime,
+                replay_workers,
+            },
+            rejected,
+        )
+    }
+
+    pub(crate) async fn commit_disabled<Committed>(&self, on_committed: Committed) -> DetachedNotifyRuntime
+    where
+        Committed: FnOnce(),
+    {
+        // Lock order: replay_workers -> target_list. The lifecycle coordinator
+        // crosses its publication barrier before stopping the old workers, so
+        // this non-cancellable commit must publish even if a newer intent was
+        // accepted while the workers joined.
+        let mut replay_workers = self.replay_workers.write().await;
+        let mut target_list = self.target_list.write().await;
+
+        let (runtime, direct_dispatches) = target_list.replace_runtime(TargetRuntimeManager::new());
+        let detached = DetachedNotifyRuntime {
+            direct_dispatches,
+            runtime,
+            replay_workers: std::mem::take(&mut *replay_workers),
+        };
+        on_committed();
+        detached
+    }
+
+    pub async fn replace_targets(&self, mut activation: RuntimeActivation<Event>) -> Result<(), NotificationError> {
+        // A caller may supply an activation created outside the compatibility
+        // prepare method. Stop any already-running replacement workers before
+        // entering the ordered handoff; the supported activate→replace pair is
+        // dormant here and therefore never overlaps the old generation.
+        self.runtime_adapter.stop_replay_workers(&mut activation.replay_workers).await;
+        let dispatch_guard = self.pause_dispatch().await;
+        if self.legacy_terminated.load(Ordering::Acquire) {
+            drop(dispatch_guard);
+            self.runtime_adapter
+                .close_compat_activation(activation)
+                .await
+                .map_err(NotificationError::Target)?;
+            return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
+        }
+        self.stop_active_replay_workers().await;
+        let (activation, open_rejected, activation_rejected) = self.runtime_adapter.start_dormant_compat_activation(activation);
+        let activation_failures = [open_rejected.failure_summary(), activation_rejected.failure_summary()]
+            .into_iter()
+            .flatten()
+            .collect::<Vec<_>>();
+        let (old_runtime, old_replay_workers, old_direct_dispatches) = {
+            // Lock order: replay_workers -> target_list (matches notify AGENTS.md).
+            let mut replay_workers = self.replay_workers.write().await;
+            let mut target_list = self.target_list.write().await;
+            Self::swap_activation(&mut target_list, &mut replay_workers, activation)
+        };
+        drop(dispatch_guard);
+        let close_old = self.close_detached_targets(DetachedNotifyRuntime {
+            direct_dispatches: old_direct_dispatches,
+            runtime: old_runtime,
+            replay_workers: old_replay_workers,
+        });
+        let close_open_rejected = self.close_prepared(open_rejected);
+        let close_activation_rejected = self.close_prepared(activation_rejected);
+        let (close_old, close_open_rejected, close_activation_rejected) =
+            tokio::join!(close_old, close_open_rejected, close_activation_rejected);
+        close_old?;
+        close_open_rejected?;
+        close_activation_rejected?;
+        if !activation_failures.is_empty() {
+            return Err(NotificationError::Initialization(format!(
+                "one or more notification targets failed to activate: {}",
+                activation_failures.join("; ")
+            )));
+        }
        Ok(())
    }

    pub async fn stop_replay_workers(&self) {
-        let mut replay_workers = self.replay_workers.write().await;
-        self.runtime_adapter.stop_replay_workers(&mut replay_workers).await;
+        let _dispatch_guard = self.pause_dispatch().await;
+        self.stop_active_replay_workers().await;
+    }
+
+    pub(crate) async fn stop_active_replay_workers(&self) {
+        let mut detached = {
+            let mut replay_workers = self.replay_workers.write().await;
+            std::mem::take(&mut *replay_workers)
+        };
+        self.runtime_adapter.stop_replay_workers(&mut detached).await;
    }

    pub async fn shutdown(&self) {
+        let _ = self.shutdown_checked().await;
+    }
+
+    pub async fn shutdown_checked(&self) -> Result<(), NotificationError> {
+        let _dispatch_guard = self.pause_dispatch().await;
+        self.legacy_terminated.store(true, Ordering::Release);
        info!(
            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
            component = LOG_COMPONENT_NOTIFY,
@@ -135,7 +300,14 @@ impl NotifyRuntimeFacade {
            "notify runtime lifecycle"
        );

-        let active_targets = self.replay_workers.read().await.len();
+        let (detached_runtime, detached_replay_workers, detached_direct_dispatches) = {
+            // Lock order: replay_workers -> target_list (matches notify AGENTS.md).
+            let mut replay_workers = self.replay_workers.write().await;
+            let mut target_list = self.target_list.write().await;
+            let (runtime, direct_dispatches) = target_list.replace_runtime(TargetRuntimeManager::new());
+            (runtime, std::mem::take(&mut *replay_workers), direct_dispatches)
+        };
+        let active_targets = detached_replay_workers.len();
        info!(
            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
            component = LOG_COMPONENT_NOTIFY,
@@ -145,25 +317,22 @@ impl NotifyRuntimeFacade {
            "notify runtime lifecycle"
        );

-        {
-            // Lock order: replay_workers -> target_list (matches notify AGENTS.md).
-            let mut replay_workers = self.replay_workers.write().await;
-            let mut target_list = self.target_list.write().await;
-            if let Err(err) = self
-                .runtime_adapter
-                .shutdown(target_list.runtime_mut(), &mut replay_workers)
-                .await
-            {
-                tracing::error!(
-                    event = EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED,
-                    component = LOG_COMPONENT_NOTIFY,
-                    subsystem = LOG_SUBSYSTEM_RUNTIME,
-                    error = %err,
-                    "Failed to shutdown notify runtime cleanly"
-                );
-            }
+        let shutdown_result = self
+            .close_detached(DetachedNotifyRuntime {
+                direct_dispatches: detached_direct_dispatches,
+                runtime: detached_runtime,
+                replay_workers: detached_replay_workers,
+            })
+            .await;
+        if let Err(err) = &shutdown_result {
+            tracing::error!(
+                event = EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED,
+                component = LOG_COMPONENT_NOTIFY,
+                subsystem = LOG_SUBSYSTEM_RUNTIME,
+                error = %err,
+                "Failed to shutdown notify runtime cleanly"
+            );
        }
-        tokio::time::sleep(Duration::from_millis(500)).await;

        info!(
            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
@@ -172,6 +341,64 @@ impl NotifyRuntimeFacade {
            state = "stopped",
            "notify runtime lifecycle"
        );
+        shutdown_result
+    }
+
+    fn swap_activation(
+        target_list: &mut crate::notifier::TargetList,
+        replay_workers: &mut ReplayWorkerManager,
+        activation: RuntimeActivation<Event>,
+    ) -> (TargetRuntimeManager<Event>, ReplayWorkerManager, Arc<DirectDispatchTracker>) {
+        let mut replacement = TargetRuntimeManager::new();
+        for target in activation.targets {
+            replacement.add_arc(target);
+        }
+        let (runtime, direct_dispatches) = target_list.replace_runtime(replacement);
+        (runtime, std::mem::replace(replay_workers, activation.replay_workers), direct_dispatches)
+    }
+
+    pub(crate) async fn stop_detached_replay(&self, detached: &mut DetachedNotifyRuntime) {
+        // Replay join is intentionally not wrapped in the target-close timeout:
+        // returning before a worker is confirmed stopped could let a
+        // replacement drain the same persistent queue concurrently.
+        self.runtime_adapter.stop_replay_workers(&mut detached.replay_workers).await;
+    }
+
+    pub(crate) async fn close_detached_targets(&self, mut detached: DetachedNotifyRuntime) -> Result<(), NotificationError> {
+        // Direct sends are allowed to finish after the replacement runtime is
+        // published, but the old targets must remain open until every task that
+        // selected that generation has released its lease.
+        detached.direct_dispatches.wait_idle().await;
+        match tokio::time::timeout(TARGET_CLOSE_TIMEOUT, detached.runtime.clear_and_close()).await {
+            Ok(close_errors) if close_errors.is_empty() => Ok(()),
+            Ok(close_errors) => {
+                let targets = close_errors
+                    .into_iter()
+                    .map(|(target_id, _)| target_id)
+                    .collect::<Vec<_>>()
+                    .join("; ");
+                Err(NotificationError::Target(rustfs_targets::TargetError::Storage(format!(
+                    "Failed to close {targets}"
+                ))))
+            }
+            Err(_) => Err(NotificationError::Target(rustfs_targets::TargetError::Timeout(
+                "Timed out closing replaced notification targets".to_string(),
+            ))),
+        }
+    }
+
+    async fn close_detached(&self, mut detached: DetachedNotifyRuntime) -> Result<(), NotificationError> {
+        self.stop_detached_replay(&mut detached).await;
+        self.close_detached_targets(detached).await
+    }
+
+    pub(crate) async fn close_prepared(&self, prepared: PreparedActivation<Event>) -> Result<(), NotificationError> {
+        match tokio::time::timeout(TARGET_CLOSE_TIMEOUT, self.runtime_adapter.close_prepared(prepared)).await {
+            Ok(result) => result.map_err(NotificationError::Target),
+            Err(_) => Err(NotificationError::Target(rustfs_targets::TargetError::Timeout(
+                "Timed out closing superseded notification targets".to_string(),
+            ))),
+        }
    }
 }

@@ -184,26 +411,50 @@ mod tests {
    };
    use async_trait::async_trait;
    use rustfs_targets::arn::TargetID;
-    use rustfs_targets::store::{Key, Store};
+    use rustfs_targets::store::{Key, QueueStore, Store};
    use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
    use rustfs_targets::{ReplayWorkerManager, SharedTarget, StoreError, Target, TargetError};
    use std::sync::Arc;
    use std::sync::atomic::{AtomicUsize, Ordering};
-    use tokio::sync::{RwLock, Semaphore};
+    use tokio::sync::{Notify, RwLock, Semaphore};

    #[derive(Clone)]
    struct TestTarget {
+        close_entered: Option<Arc<Notify>>,
+        close_error: bool,
+        close_release: Option<Arc<Notify>>,
        close_calls: Arc<AtomicUsize>,
        id: TargetID,
+        store: Option<QueueStore<QueuedPayload>>,
    }

    impl TestTarget {
        fn new(id: &str, name: &str) -> Self {
            Self {
+                close_entered: None,
+                close_error: false,
+                close_release: None,
                close_calls: Arc::new(AtomicUsize::new(0)),
                id: TargetID::new(id.to_string(), name.to_string()),
+                store: None,
            }
        }
+
+        fn with_blocking_close(mut self, entered: Arc<Notify>, release: Arc<Notify>) -> Self {
+            self.close_entered = Some(entered);
+            self.close_release = Some(release);
+            self
+        }
+
+        fn with_close_error(mut self) -> Self {
+            self.close_error = true;
+            self
+        }
+
+        fn with_store(mut self, store: QueueStore<QueuedPayload>) -> Self {
+            self.store = Some(store);
+            self
+        }
    }

    #[async_trait]
@@ -229,11 +480,22 @@ mod tests {

        async fn close(&self) -> Result<(), TargetError> {
            self.close_calls.fetch_add(1, Ordering::SeqCst);
+            if let Some(entered) = &self.close_entered {
+                entered.notify_one();
+            }
+            if let Some(release) = &self.close_release {
+                release.notified().await;
+            }
+            if self.close_error {
+                return Err(TargetError::Storage("forced close failure".to_string()));
+            }
            Ok(())
        }

        fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
-            None
+            self.store
+                .as_ref()
+                .map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
        }

        fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
@@ -254,7 +516,13 @@ mod tests {
        let notifier = Arc::new(EventNotifier::new(metrics.clone(), NotifyRuleEngine::new()));
        let target_list = notifier.target_list();
        let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
-        let facade = NotifyRuntimeFacade::new(target_list, replay_workers.clone(), Arc::new(Semaphore::new(4)), metrics);
+        let facade = NotifyRuntimeFacade::new_with_dispatch_gate(
+            target_list,
+            replay_workers.clone(),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(4)),
+            metrics,
+        );
        (facade, notifier, replay_workers)
    }

@@ -273,6 +541,26 @@ mod tests {
        assert_eq!(activation.replay_workers.len(), 0);
    }

+    #[tokio::test]
+    async fn compatibility_activation_stays_dormant_until_ordered_replace() {
+        let (facade, _, replay_workers) = build_facade();
+        let queue_root = tempfile::tempdir().expect("queue root");
+        let store = QueueStore::new_with_compression(queue_root.path(), 16, ".event", false);
+        let target = TestTarget::new("primary", "webhook").with_store(store);
+
+        let activation = facade.activate_targets_with_replay(vec![Box::new(target)]).await;
+        assert_eq!(activation.targets.len(), 1);
+        assert_eq!(activation.replay_workers.len(), 0, "compatibility prepare must not start replay early");
+        assert_eq!(replay_workers.read().await.len(), 0);
+
+        facade
+            .replace_targets(activation)
+            .await
+            .expect("ordered compatibility replace should succeed");
+        assert_eq!(replay_workers.read().await.len(), 1);
+        facade.shutdown_checked().await.expect("test runtime should shut down");
+    }
+
    #[tokio::test]
    async fn runtime_facade_replace_targets_commits_runtime_state() {
        let (facade, notifier, replay_workers) = build_facade();
@@ -292,4 +580,129 @@ mod tests {
        assert_eq!(active_targets, vec![TargetID::new("primary".to_string(), "webhook".to_string())]);
        assert_eq!(replay_workers.read().await.len(), 0);
    }
+
+    #[tokio::test]
+    async fn runtime_queries_do_not_wait_for_target_close() {
+        let (facade, notifier, replay_workers) = build_facade();
+        let close_entered = Arc::new(Notify::new());
+        let close_release = Arc::new(Notify::new());
+        let target = TestTarget::new("primary", "webhook").with_blocking_close(close_entered.clone(), close_release.clone());
+        facade
+            .replace_targets(rustfs_targets::RuntimeActivation {
+                replay_workers: ReplayWorkerManager::new(),
+                targets: vec![Arc::new(target) as SharedTarget<Event>],
+            })
+            .await
+            .expect("target install should succeed");
+
+        let shutdown = tokio::spawn({
+            let facade = facade.clone();
+            async move { facade.shutdown_checked().await }
+        });
+        close_entered.notified().await;
+
+        let target_list = notifier.target_list();
+        assert!(target_list.try_read().is_ok(), "target list lock must not be held during close");
+        assert!(replay_workers.try_read().is_ok(), "replay manager lock must not be held during close");
+        close_release.notify_one();
+        shutdown
+            .await
+            .expect("shutdown task should not panic")
+            .expect("shutdown should succeed");
+
+        let snapshot = NotifyRuntimeView::new(target_list, replay_workers)
+            .runtime_status_snapshot()
+            .await;
+        assert_eq!(snapshot.target_count, 0);
+        assert_eq!(snapshot.replay_worker_count, 0);
+    }
+
+    #[tokio::test]
+    async fn runtime_locks_are_released_while_replay_worker_joins() {
+        let (facade, notifier, replay_workers) = build_facade();
+        let cancel_received = Arc::new(Notify::new());
+        let release = Arc::new(Notify::new());
+        let (cancel_tx, mut cancel_rx) = tokio::sync::mpsc::channel(1);
+        let join = tokio::spawn({
+            let cancel_received = cancel_received.clone();
+            let release = release.clone();
+            async move {
+                let _ = cancel_rx.recv().await;
+                cancel_received.notify_one();
+                release.notified().await;
+            }
+        });
+        replay_workers
+            .write()
+            .await
+            .insert_with_handle("blocked-worker".to_string(), cancel_tx, join);
+
+        let stop = tokio::spawn({
+            let facade = facade.clone();
+            async move { facade.stop_replay_workers().await }
+        });
+        cancel_received.notified().await;
+
+        assert!(replay_workers.try_read().is_ok(), "replay manager lock must not be held during join");
+        let target_list = notifier.target_list();
+        assert!(target_list.try_read().is_ok(), "target list lock must not be held during replay join");
+
+        release.notify_one();
+        stop.await.expect("stop task should finish");
+    }
+
+    #[tokio::test]
+    async fn shutdown_returns_close_error_after_detaching_runtime() {
+        let (facade, notifier, replay_workers) = build_facade();
+        let target = TestTarget::new("primary", "webhook").with_close_error();
+        facade
+            .replace_targets(rustfs_targets::RuntimeActivation {
+                replay_workers: ReplayWorkerManager::new(),
+                targets: vec![Arc::new(target) as SharedTarget<Event>],
+            })
+            .await
+            .expect("target install should succeed");
+
+        let err = facade.shutdown_checked().await.expect_err("close failure should propagate");
+        assert!(matches!(err, crate::NotificationError::Target(TargetError::Storage(_))));
+
+        let snapshot = NotifyRuntimeView::new(notifier.target_list(), replay_workers)
+            .runtime_status_snapshot()
+            .await;
+        assert_eq!(snapshot.target_count, 0);
+        assert_eq!(snapshot.replay_worker_count, 0);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn shutdown_bounds_a_target_that_never_closes() {
+        let (facade, notifier, replay_workers) = build_facade();
+        let close_entered = Arc::new(Notify::new());
+        let never_release = Arc::new(Notify::new());
+        let target = TestTarget::new("primary", "webhook").with_blocking_close(close_entered.clone(), never_release);
+        facade
+            .replace_targets(rustfs_targets::RuntimeActivation {
+                replay_workers: ReplayWorkerManager::new(),
+                targets: vec![Arc::new(target) as SharedTarget<Event>],
+            })
+            .await
+            .expect("target install should succeed");
+
+        let shutdown = tokio::spawn({
+            let facade = facade.clone();
+            async move { facade.shutdown_checked().await }
+        });
+        close_entered.notified().await;
+        tokio::time::advance(super::TARGET_CLOSE_TIMEOUT).await;
+
+        let err = shutdown
+            .await
+            .expect("shutdown task should not panic")
+            .expect_err("blocked close must time out");
+        assert!(matches!(err, crate::NotificationError::Target(TargetError::Timeout(_))));
+        let snapshot = NotifyRuntimeView::new(notifier.target_list(), replay_workers)
+            .runtime_status_snapshot()
+            .await;
+        assert_eq!(snapshot.target_count, 0);
+        assert_eq!(snapshot.replay_worker_count, 0);
+    }
 }
@@ -61,7 +61,8 @@ impl NotifyRuntimeView {
    }

    pub async fn snapshot_target_health(&self) -> Vec<RuntimeTargetHealthSnapshot> {
-        self.target_list.read().await.runtime_health_snapshots().await
+        let targets = self.target_list.read().await.values();
+        rustfs_targets::health_snapshots_for_targets(targets).await
    }

    pub async fn runtime_status_snapshot(&self) -> rustfs_targets::RuntimeStatusSnapshot {
@@ -84,7 +85,7 @@ mod tests {
        Arc,
        atomic::{AtomicU64, Ordering},
    };
-    use tokio::sync::RwLock;
+    use tokio::sync::{Notify, RwLock};

    #[derive(Clone)]
    struct TestTarget {
@@ -93,6 +94,8 @@ mod tests {
        failed_messages: Arc<AtomicU64>,
        failed_store_length: u64,
        id: TargetID,
+        health_started: Option<Arc<Notify>>,
+        health_release: Option<Arc<Notify>>,
        total_messages: Arc<AtomicU64>,
    }

@@ -104,6 +107,8 @@ mod tests {
                failed_messages: Arc::new(AtomicU64::new(0)),
                failed_store_length: 0,
                id: TargetID::new(id.to_string(), name.to_string()),
+                health_started: None,
+                health_release: None,
                total_messages: Arc::new(AtomicU64::new(0)),
            }
        }
@@ -123,6 +128,12 @@ mod tests {
            self
        }

+        fn with_health_gate(mut self, started: Arc<Notify>, release: Arc<Notify>) -> Self {
+            self.health_started = Some(started);
+            self.health_release = Some(release);
+            self
+        }
+
        fn record_successes(&self, count: u64) {
            self.total_messages.store(count, Ordering::Relaxed);
        }
@@ -142,6 +153,10 @@ mod tests {
        }

        async fn is_active(&self) -> Result<bool, TargetError> {
+            if let (Some(started), Some(release)) = (&self.health_started, &self.health_release) {
+                started.notify_one();
+                release.notified().await;
+            }
            Ok(self.active)
        }

@@ -268,4 +283,29 @@ mod tests {
        assert_eq!(status.target_count, 2);
        assert_eq!(status.replay_worker_count, 0);
    }
+
+    #[tokio::test]
+    async fn health_probe_does_not_hold_the_target_list_read_lock() {
+        let target_list = Arc::new(RwLock::new(TargetList::new()));
+        let started = Arc::new(Notify::new());
+        let release = Arc::new(Notify::new());
+        let target = Arc::new(TestTarget::new("blocked", "webhook").with_health_gate(started.clone(), release.clone()));
+        target_list
+            .write()
+            .await
+            .add(target as Arc<dyn Target<Event> + Send + Sync>)
+            .expect("test target should be added");
+        let runtime_view = NotifyRuntimeView::new(target_list.clone(), Arc::new(RwLock::new(ReplayWorkerManager::new())));
+
+        let snapshot_task = tokio::spawn(async move { runtime_view.snapshot_target_health().await });
+        started.notified().await;
+
+        let write_guard = tokio::time::timeout(std::time::Duration::from_secs(1), target_list.write())
+            .await
+            .expect("network health probe must not retain the target-list read lock");
+        drop(write_guard);
+        release.notify_one();
+
+        assert_eq!(snapshot_task.await.expect("snapshot task should finish").len(), 1);
+    }
 }
@@ -57,7 +57,13 @@ impl NotifyServices {
        live_event_history: Arc<RwLock<LiveEventHistory>>,
    ) -> Self {
        let runtime_view = NotifyRuntimeView::new(target_list.clone(), stream_cancellers.clone());
-        let runtime_facade = NotifyRuntimeFacade::new(target_list, stream_cancellers, concurrency_limiter, metrics.clone());
+        let runtime_facade = NotifyRuntimeFacade::new_with_dispatch_gate(
+            target_list,
+            stream_cancellers,
+            notifier.dispatch_gate(),
+            concurrency_limiter,
+            metrics.clone(),
+        );
        let config_manager = NotifyConfigManager::new(config, registry, rule_engine.clone(), runtime_facade.clone());
        let bucket_config_manager = NotifyBucketConfigManager::new(notifier.clone(), rule_engine, subscriber_view);
        let pipeline = NotifyPipeline::new(notifier, live_event_sender, live_event_history);
@@ -15,35 +15,70 @@
 use std::sync::Arc;

 use rustfs_ecstore::api::config::com::{
-    read_config_without_migrate as read_notify_config_without_migrate_from_backend,
-    save_server_config as save_notify_server_config_to_backend,
+    read_config_without_migrate_no_lock as read_notify_config_without_migrate_from_backend_no_lock,
+    read_existing_server_config_no_lock as read_existing_notify_config_from_backend_no_lock,
+    save_server_config_no_lock as save_notify_server_config_to_backend_no_lock,
+    with_server_config_read_lock as with_notify_server_config_read_lock_from_backend,
+    with_server_config_write_lock as with_notify_server_config_write_lock_from_backend,
 };
 use rustfs_ecstore::api::runtime::object_store_handle as resolve_notify_object_store_handle_from_backend;
-pub(crate) use rustfs_ecstore::api::storage::ECStore as NotifyStore;
+pub use rustfs_ecstore::api::storage::ECStore as NotifyStore;

 pub(crate) fn resolve_notify_object_store_handle() -> Option<Arc<NotifyStore>> {
    resolve_notify_object_store_handle_from_backend()
 }

-pub(crate) async fn read_notify_server_config_without_migrate(
+pub(crate) async fn read_notify_server_config_without_migrate_no_lock(
    store: Arc<NotifyStore>,
 ) -> Result<rustfs_config::server_config::Config, String> {
-    read_notify_config_without_migrate_from_backend(store)
+    read_notify_config_without_migrate_from_backend_no_lock(store)
        .await
        .map_err(|err| err.to_string())
 }

-pub(crate) async fn save_notify_server_config(
+pub(crate) async fn read_existing_notify_server_config_no_lock(
+    store: Arc<NotifyStore>,
+) -> Result<rustfs_config::server_config::Config, String> {
+    read_existing_notify_config_from_backend_no_lock(store)
+        .await
+        .map_err(|err| err.to_string())
+}
+
+pub(crate) async fn save_notify_server_config_no_lock(
    store: Arc<NotifyStore>,
    config: &rustfs_config::server_config::Config,
 ) -> Result<(), String> {
-    save_notify_server_config_to_backend(store, config)
+    save_notify_server_config_to_backend_no_lock(store, config)
+        .await
+        .map_err(|err| err.to_string())
+}
+
+pub(crate) async fn with_notify_server_config_write_lock<F, Fut, T>(store: Arc<NotifyStore>, operation: F) -> Result<T, String>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    with_notify_server_config_write_lock_from_backend(store, operation)
+        .await
+        .map_err(|err| err.to_string())
+}
+
+pub(crate) async fn with_notify_server_config_read_lock<F, Fut, T>(store: Arc<NotifyStore>, operation: F) -> Result<T, String>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    with_notify_server_config_read_lock_from_backend(store, operation)
        .await
        .map_err(|err| err.to_string())
 }

 pub(crate) mod crate_boundary {
    pub(crate) use super::{
-        read_notify_server_config_without_migrate, resolve_notify_object_store_handle, save_notify_server_config,
+        read_existing_notify_server_config_no_lock, read_notify_server_config_without_migrate_no_lock,
+        resolve_notify_object_store_handle, save_notify_server_config_no_lock, with_notify_server_config_read_lock,
+        with_notify_server_config_write_lock,
    };
 }
@@ -0,0 +1,80 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_config::server_config::Config;
+use rustfs_notify::{NotificationError, NotificationRuntimeState, ensure_live_events, notification_system, reconcile};
+use std::sync::Arc;
+
+fn assert_terminated(error: NotificationError) {
+    assert!(matches!(
+        error,
+        NotificationError::Initialization(detail) if detail == "Notification runtime has terminated"
+    ));
+}
+
+#[tokio::test]
+async fn global_singleton_survives_suspend_but_not_process_termination() {
+    let system = ensure_live_events();
+    let same_system = ensure_live_events();
+    assert!(Arc::ptr_eq(&system, &same_system));
+    assert!(Arc::ptr_eq(
+        &system,
+        &notification_system().expect("global notification system should exist")
+    ));
+
+    system
+        .set_targets_enabled(true, Some(Config::new()))
+        .await
+        .expect("empty target runtime should enable");
+    assert!(matches!(
+        system.runtime_lifecycle_state(),
+        NotificationRuntimeState::TargetsEnabled { .. }
+    ));
+
+    system
+        .set_targets_enabled(false, None)
+        .await
+        .expect("disable should suspend targets without terminating the singleton");
+    assert_eq!(system.runtime_lifecycle_state(), NotificationRuntimeState::LiveOnly);
+
+    system
+        .set_targets_enabled(true, None)
+        .await
+        .expect("a suspended target runtime should be restartable");
+    assert!(matches!(
+        system.runtime_lifecycle_state(),
+        NotificationRuntimeState::TargetsEnabled { .. }
+    ));
+
+    system
+        .shutdown_checked()
+        .await
+        .expect("process shutdown should terminate the target runtime");
+    assert_eq!(system.runtime_lifecycle_state(), NotificationRuntimeState::Terminated);
+
+    let lazy_system = ensure_live_events();
+    assert!(Arc::ptr_eq(&system, &lazy_system));
+    assert_terminated(
+        reconcile(Config::new())
+            .await
+            .expect_err("lazy reconciliation must not restart a terminated process runtime"),
+    );
+    assert_terminated(
+        lazy_system
+            .reload_config(Config::new())
+            .await
+            .expect_err("config reload must not restart a terminated process runtime"),
+    );
+    assert_eq!(lazy_system.runtime_lifecycle_state(), NotificationRuntimeState::Terminated);
+}
@@ -0,0 +1,44 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS;
+use rustfs_config::server_config::{Config, KVS};
+use rustfs_config::{ENABLE_KEY, WEBHOOK_ENDPOINT};
+use rustfs_notify::{initialize, notification_system};
+
+#[tokio::test]
+async fn failed_legacy_initialize_can_retry_the_stable_singleton() {
+    let mut invalid_target = KVS::new();
+    invalid_target.insert(ENABLE_KEY.to_string(), "on".to_string());
+    invalid_target.insert(WEBHOOK_ENDPOINT.to_string(), "not-a-url".to_string());
+    let mut invalid_config = Config::new();
+    invalid_config
+        .0
+        .entry(NOTIFY_WEBHOOK_SUB_SYS.to_string())
+        .or_default()
+        .insert("primary".to_string(), invalid_target);
+
+    initialize(invalid_config)
+        .await
+        .expect_err("invalid first target activation should fail");
+    initialize(Config::new())
+        .await
+        .expect("legacy initialize should retry after the configuration is fixed");
+
+    notification_system()
+        .expect("stable singleton should remain available")
+        .shutdown_checked()
+        .await
+        .expect("test runtime should terminate");
+}
@@ -136,7 +136,7 @@ tests read):
 # Pass case ids to override, or "all" for the full default matrix.
 ./capture_via_docker.sh

-RUSTFS_MINIO_STATIC_KMS_KEY_B64=IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= \
+RUSTFS_MINIO_STATIC_KMS_KEY=minio-default-key:IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= \
  cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
 ```

@@ -145,6 +145,28 @@ This is exactly what the nightly `minio-interop` GitHub Actions workflow runs
 SSE-C cases still need the host-`minio` + TLS path above; the Docker helper
 targets the SSE-S3 / SSE-KMS multipart cases the interop tests assert on.

+## RustFS beta.5 KMS Compatibility Fixture
+
+The same CI gate also downloads the pinned RustFS `1.0.0-beta.5` release,
+verifies its published archive SHA-256, starts it with the production local KMS
+backend, writes a real SSE-KMS object, and exports the resulting four-disk
+backend plus its one-time KMS key directory:
+
+```bash
+uv run python ./capture_rustfs_beta5.py
+
+RUSTFS_BETA5_FIXTURE_ROOT=../fixtures/rustfs-beta5-generated \
+  cargo test -p rustfs-ecstore --features rio-v2 \
+    --test minio_generated_read_test \
+    reads_real_rustfs_beta5_sse_kms_fixture_through_production_reader \
+    -- --ignored
+```
+
+Outside Linux x86_64, pass `--rustfs-binary` with the matching beta.5 binary.
+The generated fixture and KMS key stay under the ignored fixture root and are
+recreated for every CI run; neither key material nor plaintext keys are logged
+or committed.
+
 ## Capture Guidance

 For each case, preserve these inputs when possible:
@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import base64
+import hashlib
+import json
+import os
+import platform
+import secrets
+import shutil
+import stat
+import subprocess
+import tempfile
+import time
+import urllib.error
+import urllib.request
+import zipfile
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.parse import urlparse
+
+from lab import (
+    FixtureCase,
+    LabPaths,
+    S3Client,
+    build_payload_file,
+    head_case,
+    stop_process,
+    store_case_artifacts,
+    upload_case,
+    wait_for_s3_ready,
+)
+
+
+RELEASE = "1.0.0-beta.5"
+CASE_ID = "rustfs-beta5-sse-kms-singlepart-64k"
+KMS_KEY_ID = "beta5-test-key"
+LINUX_X86_64_ASSET = "rustfs-linux-x86_64-gnu-v1.0.0-beta.5.zip"
+LINUX_X86_64_SHA256 = "73529e732adc3c2c5c78c7f2c2e4e331a20e5bee1e2db341ee7c762299e4b327"
+DEFAULT_ROOT = Path(__file__).resolve().parents[1] / "fixtures" / "rustfs-beta5-generated"
+
+
+def release_asset_url(asset: str) -> str:
+    return f"https://github.com/rustfs/rustfs/releases/download/{RELEASE}/{asset}"
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def resolve_binary(explicit_binary: Path | None, workdir: Path) -> Path:
+    if explicit_binary is not None:
+        binary = explicit_binary.resolve()
+        if not binary.is_file():
+            raise FileNotFoundError(f"RustFS beta.5 binary not found: {binary}")
+        return binary
+
+    if platform.system() != "Linux" or platform.machine() not in {"x86_64", "AMD64"}:
+        raise RuntimeError("--rustfs-binary is required outside Linux x86_64")
+
+    archive = workdir / LINUX_X86_64_ASSET
+    urllib.request.urlretrieve(release_asset_url(LINUX_X86_64_ASSET), archive)
+    actual = sha256_file(archive)
+    if actual != LINUX_X86_64_SHA256:
+        raise RuntimeError(f"RustFS beta.5 archive SHA-256 mismatch: expected {LINUX_X86_64_SHA256}, got {actual}")
+    with zipfile.ZipFile(archive) as bundle:
+        bundle.extract("rustfs", workdir)
+    binary = workdir / "rustfs"
+    binary.chmod(binary.stat().st_mode | stat.S_IXUSR)
+    return binary
+
+
+def write_local_kms_key(key_dir: Path) -> None:
+    key_dir.mkdir(parents=True, mode=0o700)
+    key_path = key_dir / f"{KMS_KEY_ID}.key"
+    now = datetime.now(timezone.utc).isoformat(timespec="seconds")
+    payload = {
+        "key_id": KMS_KEY_ID,
+        "version": 1,
+        "algorithm": "AES_256",
+        "usage": "EncryptDecrypt",
+        "status": "Active",
+        "description": None,
+        "metadata": {},
+        "created_at": now,
+        "rotated_at": None,
+        "created_by": "fixture-lab",
+        "encrypted_key_material": base64.b64encode(secrets.token_bytes(32)).decode("ascii"),
+        "nonce": [],
+    }
+    key_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    key_path.chmod(0o600)
+
+
+def wait_for_process_s3(client: S3Client, process: subprocess.Popen[str], timeout_seconds: int) -> None:
+    deadline = time.time() + timeout_seconds
+    while time.time() < deadline:
+        if process.poll() is not None:
+            raise RuntimeError("RustFS beta.5 exited before its S3 API became ready")
+        try:
+            client.list_buckets()
+            return
+        except (RuntimeError, urllib.error.URLError):
+            time.sleep(1)
+    wait_for_s3_ready(client, 1)
+
+
+def capture(args: argparse.Namespace) -> Path:
+    root = args.root.resolve()
+    if root.exists():
+        shutil.rmtree(root)
+    root.mkdir(parents=True)
+
+    with tempfile.TemporaryDirectory(prefix="rustfs-beta5-fixture-") as temporary:
+        workdir = Path(temporary)
+        binary = resolve_binary(args.rustfs_binary, workdir)
+        key_dir = workdir / "kms"
+        write_local_kms_key(key_dir)
+        disks = [workdir / f"disk{index}" for index in range(1, 5)]
+        for disk in disks:
+            disk.mkdir()
+
+        endpoint = args.endpoint
+        parsed = urlparse(endpoint)
+        if parsed.scheme != "http" or not parsed.netloc:
+            raise ValueError("beta.5 fixture endpoint must be an http:// URL")
+        command = [
+            str(binary),
+            "server",
+            "--address",
+            parsed.netloc,
+            "--access-key",
+            "minioadmin",
+            "--secret-key",
+            "minioadmin",
+            "--kms-enable",
+            "--kms-backend",
+            "local",
+            "--kms-key-dir",
+            str(key_dir),
+            "--kms-default-key-id",
+            KMS_KEY_ID,
+            *(str(disk) for disk in disks),
+        ]
+        environment = os.environ.copy()
+        environment["RUSTFS_UNSAFE_BYPASS_DISK_CHECK"] = "true"
+        server_log_path = workdir / "server.log"
+        case = FixtureCase(
+            case_id=CASE_ID,
+            bucket="demo",
+            object_name="dir/object.bin",
+            encryption="SSE-KMS",
+            size_bytes=64 * 1024,
+            multipart=False,
+            kms_key_id=KMS_KEY_ID,
+        )
+        payload_file = workdir / "payload.bin"
+        plaintext_sha256 = build_payload_file(payload_file, case.size_bytes)
+
+        with server_log_path.open("w", encoding="utf-8") as server_log:
+            process = subprocess.Popen(
+                command,
+                cwd=str(workdir),
+                env=environment,
+                stdout=server_log,
+                stderr=subprocess.STDOUT,
+                text=True,
+            )
+            try:
+                client = S3Client(endpoint)
+                wait_for_process_s3(client, process, args.timeout_seconds)
+                client.create_bucket(case.bucket)
+                request_payload = upload_case(client, case, payload_file)
+                head_payload = head_case(client, case)
+            finally:
+                stop_process(process)
+
+        export_root = workdir / "export"
+        export_root.mkdir()
+        for index, disk in enumerate(disks, start=1):
+            shutil.copytree(disk, export_root / f"disk{index}")
+        shutil.copytree(key_dir, root / "kms")
+        return store_case_artifacts(
+            paths=LabPaths(root=root, cases=root / "cases"),
+            case_id=case.case_id,
+            bucket=case.bucket,
+            object_name=case.object_name,
+            source_tree=export_root,
+            version_id=head_payload.get("VersionId"),
+            request_payload=request_payload,
+            head_payload=head_payload,
+            plaintext_sha256=plaintext_sha256,
+            notes="Captured from the pinned RustFS 1.0.0-beta.5 release with the production local KMS backend.",
+            capture_payload={
+                "release": RELEASE,
+                "binary_sha256": sha256_file(binary),
+                "disk_count": len(disks),
+                "kms_key_id": KMS_KEY_ID,
+            },
+        )
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Capture a real RustFS beta.5 SSE-KMS fixture")
+    parser.add_argument("--root", type=Path, default=DEFAULT_ROOT)
+    parser.add_argument("--rustfs-binary", type=Path)
+    parser.add_argument("--endpoint", default="http://127.0.0.1:19011")
+    parser.add_argument("--timeout-seconds", type=int, default=90)
+    return parser
+
+
+def main() -> int:
+    case_dir = capture(build_parser().parse_args())
+    print(case_dir)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -2,6 +2,7 @@ from __future__ import annotations

 import importlib.util
 import base64
+import json
 import sys
 import tempfile
 import unittest
@@ -21,6 +22,21 @@ def load_lab_module():


 lab = load_lab_module()
+sys.modules["lab"] = lab
+
+
+def load_beta5_module():
+    module_path = Path(__file__).with_name("capture_rustfs_beta5.py")
+    spec = importlib.util.spec_from_file_location("rustfs_beta5_fixture_capture", module_path)
+    assert spec is not None
+    assert spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+beta5 = load_beta5_module()


 class DiscoverMinioLauncherTests(unittest.TestCase):
@@ -203,5 +219,31 @@ class KmsSecretKeyTests(unittest.TestCase):
        )


+class RustfsBeta5CaptureTests(unittest.TestCase):
+    def test_explicit_beta5_binary_is_used_without_download(self) -> None:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            binary = Path(temp_dir) / "rustfs"
+            binary.write_bytes(b"beta5")
+
+            resolved = beta5.resolve_binary(binary, Path(temp_dir))
+
+            self.assertEqual(resolved, binary.resolve())
+
+    def test_local_kms_key_has_beta5_compatible_shape_and_permissions(self) -> None:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            key_dir = Path(temp_dir) / "kms"
+
+            beta5.write_local_kms_key(key_dir)
+
+            key_path = key_dir / f"{beta5.KMS_KEY_ID}.key"
+            payload = json.loads(key_path.read_text(encoding="utf-8"))
+            self.assertEqual(payload["key_id"], beta5.KMS_KEY_ID)
+            self.assertEqual(payload["algorithm"], "AES_256")
+            self.assertEqual(payload["usage"], "EncryptDecrypt")
+            self.assertEqual(len(base64.b64decode(payload["encrypted_key_material"])), 32)
+            self.assertEqual(payload["nonce"], [])
+            self.assertEqual(key_path.stat().st_mode & 0o777, 0o600)
+
+
 if __name__ == "__main__":
    unittest.main()
@@ -2514,6 +2514,7 @@ mod tests {
                stream: Box::new(Cursor::new(data)),
                object_info: ObjectInfo::default(),
                buffered_body: None,
+                resolved_sse: None,
                body_source: Default::default(),
            })
        }
@@ -833,7 +833,7 @@ mod serial_tests {
        let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
        let backend = register_mock_tier(&tier_name).await;
        backend.set_put_remote_version(Some(Uuid::new_v4().to_string())).await;
-        backend.set_reject_non_empty_remote_versions(true);
+        backend.reject_next_non_empty_remote_version_validation();
        backend.set_remove_failure(true);
        let cleanup_store_barrier = TransitionCleanupStoreBarrier::install();

@@ -943,7 +943,7 @@ mod serial_tests {
            let backend = register_mock_tier(&tier_name).await;
            let remote_version = Uuid::new_v4().to_string();
            backend.set_put_remote_version(Some(remote_version.clone())).await;
-            backend.set_reject_non_empty_remote_versions(true);
+            backend.reject_next_non_empty_remote_version_validation();
            backend.set_remove_failure(matches!(case, CleanupCase::FullyFailed));
            let put_barrier = backend.arm_put_barrier().await;
            let remove_barrier = if matches!(case, CleanupCase::RetryPersisted) {
@@ -18,7 +18,8 @@ rustfs-tls-runtime = { workspace = true }
 rustfs-s3-types = { workspace = true }
 rustfs-utils = { workspace = true, features = ["egress"] }
 async-trait = { workspace = true }
-async-nats = { workspace = true }
+futures-util = { workspace = true }
+async-nats = { workspace = true, features = ["server_2_10", "server_2_11", "server_2_12", "server_2_14", "service", "aws-lc-rs", "nkeys", "object-store", "kv", "websockets", "nuid"] }
 deadpool-postgres = { workspace = true }
 hyper = { workspace = true, features = ["http2", "http1", "server"] }
 hyper-rustls = { workspace = true, default-features = false, features = ["native-tokio", "tls12", "logging", "aws-lc-rs"] }
@@ -26,6 +27,7 @@ lapin = { workspace = true, default-features = false, features = ["tokio", "rust
 libc = { workspace = true }
 pulsar = { workspace = true, default-features = false, features = ["tokio-rustls-runtime", "telemetry"] }
 regex = { workspace = true }
+rayon = { workspace = true }
 reqwest = { workspace = true }
 rumqttc = { workspace = true, features = ["websocket"] }
 redis = { workspace = true, features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
@@ -38,6 +40,7 @@ sha2 = { workspace = true }
 snap = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["fs", "rt-multi-thread", "sync", "time"] }
+tokio-util = { workspace = true }
 tokio-postgres = { workspace = true, default-features = false, features = ["runtime", "with-serde_json-1"] }
 tokio-postgres-rustls = { workspace = true }
 tracing = { workspace = true }
@@ -47,16 +47,13 @@ pub(super) fn split_env_field_and_instance(rest: &str, valid_fields: &HashSet<St
        .max_by_key(|(field, _)| field.len())
 }

-pub(super) fn is_target_enabled(config: &KVS) -> bool {
-    config
-        .lookup(ENABLE_KEY)
-        .map(|v| {
-            EnableState::from_str(v.as_str())
-                .ok()
-                .map(|s| s.is_enabled())
-                .unwrap_or(false)
-        })
-        .unwrap_or(false)
+pub(super) fn is_target_enabled(config: &KVS) -> Result<bool, TargetError> {
+    let Some(value) = config.lookup(ENABLE_KEY) else {
+        return Ok(false);
+    };
+    EnableState::from_str(value.as_str())
+        .map(EnableState::is_enabled)
+        .map_err(|_| TargetError::Configuration(format!("Invalid {ENABLE_KEY} value '{value}'")))
 }

 pub(super) fn parse_target_bool(value: Option<&str>) -> Option<bool> {
@@ -12,7 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use super::loader::collect_merged_target_configs_from_env;
+use super::loader::{
+    MergedTargetConfigRecord, collect_merged_target_configs_compat_from_env, collect_merged_target_configs_from_env,
+};
+use crate::TargetError;
 use crate::domain::TargetDomain;
 use rustfs_config::server_config::{Config, KVS};
 use std::collections::HashSet;
@@ -86,6 +89,13 @@ pub fn normalize_target_plugin_instances(
    normalize_target_plugin_instances_from_env(config, descriptor, std::env::vars())
 }

+pub fn try_normalize_target_plugin_instances(
+    config: &Config,
+    descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
+) -> Result<Vec<TargetPluginInstanceRecord>, TargetError> {
+    try_normalize_target_plugin_instances_from_env(config, descriptor, std::env::vars())
+}
+
 pub fn normalize_target_plugin_instances_from_env<I>(
    config: &Config,
    descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
@@ -100,7 +110,7 @@ where
        .map(|field| (*field).to_string())
        .collect::<HashSet<_>>();

-    collect_merged_target_configs_from_env(
+    collect_merged_target_configs_compat_from_env(
        config,
        descriptor.subsystem,
        descriptor.route_prefix,
@@ -109,7 +119,42 @@ where
        env_vars,
    )
    .into_iter()
-    .map(|record| TargetPluginInstanceRecord {
+    .map(|record| target_plugin_instance_record(descriptor, record))
+    .collect()
+}
+
+pub fn try_normalize_target_plugin_instances_from_env<I>(
+    config: &Config,
+    descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
+    env_vars: I,
+) -> Result<Vec<TargetPluginInstanceRecord>, TargetError>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    let valid_fields = descriptor
+        .valid_fields
+        .iter()
+        .map(|field| (*field).to_string())
+        .collect::<HashSet<_>>();
+
+    Ok(collect_merged_target_configs_from_env(
+        config,
+        descriptor.subsystem,
+        descriptor.route_prefix,
+        descriptor.target_type,
+        &valid_fields,
+        env_vars,
+    )?
+    .into_iter()
+    .map(|record| target_plugin_instance_record(descriptor, record))
+    .collect())
+}
+
+fn target_plugin_instance_record(
+    descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
+    record: MergedTargetConfigRecord,
+) -> TargetPluginInstanceRecord {
+    TargetPluginInstanceRecord {
        domain: descriptor.domain,
        plugin_id: descriptor.plugin_id.to_string(),
        target_type: descriptor.target_type.to_string(),
@@ -123,8 +168,7 @@ where
            has_env_instance: record.has_env_instance,
        },
        effective_config: record.effective_config,
-    })
-    .collect()
+    }
 }

 pub fn normalize_legacy_target_instances(
@@ -149,8 +193,9 @@ where
 mod tests {
    use super::{
        TargetInstanceSourceClass, TargetPluginInstanceCompatDescriptor, normalize_legacy_target_instances_from_env,
-        normalize_target_plugin_instances_from_env,
+        try_normalize_target_plugin_instances_from_env,
    };
+    use crate::TargetError;
    use crate::domain::TargetDomain;
    use crate::manifest::builtin_target_manifest;
    use rustfs_config::audit::{AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
@@ -338,9 +383,50 @@ mod tests {
        let descriptor = notify_webhook_descriptor();
        let env = vec![("RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT".to_string(), "7".to_string())];

-        let canonical = normalize_target_plugin_instances_from_env(&cfg, &descriptor, env.clone());
+        let canonical = try_normalize_target_plugin_instances_from_env(&cfg, &descriptor, env.clone())
+            .expect("canonical normalization should succeed");
        let compatibility = normalize_legacy_target_instances_from_env(&cfg, &descriptor, env);

        assert_eq!(canonical, compatibility);
    }
+
+    #[test]
+    fn normalize_instances_rejects_invalid_enable_value() {
+        let error = try_normalize_target_plugin_instances_from_env(
+            &Config(HashMap::new()),
+            &notify_webhook_descriptor(),
+            vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), "invalid".to_string())],
+        )
+        .expect_err("invalid enable value must be propagated by the public normalizer");
+
+        match error {
+            TargetError::Configuration(detail) => assert_eq!(detail, "Invalid enable value 'invalid'"),
+            other => panic!("expected a configuration error, got {other}"),
+        }
+    }
+
+    #[test]
+    fn legacy_normalizer_keeps_valid_instance_when_one_enable_is_invalid() {
+        let instances = normalize_legacy_target_instances_from_env(
+            &Config(HashMap::new()),
+            &notify_webhook_descriptor(),
+            vec![
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_GOOD".to_string(), "on".to_string()),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_GOOD".to_string(), "https://example.com/good".to_string()),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_BAD".to_string(), "invalid".to_string()),
+            ],
+        );
+
+        assert_eq!(instances.len(), 2);
+        let good = instances
+            .iter()
+            .find(|instance| instance.instance_id == "good")
+            .expect("valid instance should remain present");
+        let bad = instances
+            .iter()
+            .find(|instance| instance.instance_id == "bad")
+            .expect("invalid legacy instance should remain visible");
+        assert!(good.enabled);
+        assert!(!bad.enabled);
+    }
 }
@@ -13,6 +13,7 @@
 // limitations under the License.

 use super::common::{is_target_enabled, split_env_field_and_instance};
+use crate::TargetError;
 use rustfs_config::server_config::{Config, KVS};
 use rustfs_config::{DEFAULT_DELIMITER, ENV_PREFIX};
 use std::collections::{HashMap, HashSet};
@@ -27,6 +28,31 @@ pub fn collect_target_configs(
    collect_target_configs_from_env(config, route_prefix, target_type, valid_fields, std::env::vars())
 }

+pub fn try_collect_target_configs(
+    config: &Config,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+) -> Result<Vec<(String, KVS)>, TargetError> {
+    try_collect_target_configs_from_env(config, route_prefix, target_type, valid_fields, std::env::vars())
+}
+
+/// Collects enabled target instance configs with per-instance fault isolation.
+///
+/// Unlike [`try_collect_target_configs`], an instance with an invalid
+/// configuration (for example an unparseable `enable` value) does not abort the
+/// whole target type. The enabled `(id, config)` pairs are returned alongside a
+/// summary line for every instance that could not be collected, so callers can
+/// load the healthy instances while still surfacing the failed ones.
+pub fn collect_target_config_results(
+    config: &Config,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+) -> (Vec<(String, KVS)>, Vec<String>) {
+    collect_target_config_results_from_env(config, route_prefix, target_type, valid_fields, std::env::vars())
+}
+
 fn is_sensitive_target_field(field_name: &str) -> bool {
    let field_name = field_name.to_ascii_lowercase();
    field_name.contains("password")
@@ -123,7 +149,7 @@ pub fn collect_target_configs_from_env<I>(
 where
    I: IntoIterator<Item = (String, String)>,
 {
-    collect_merged_target_configs_from_env(
+    collect_merged_target_configs_compat_from_env(
        config,
        &format!("{route_prefix}{target_type}").to_lowercase(),
        route_prefix,
@@ -137,6 +163,59 @@ where
    .collect()
 }

+pub fn try_collect_target_configs_from_env<I>(
+    config: &Config,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+    env_vars: I,
+) -> Result<Vec<(String, KVS)>, TargetError>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    Ok(collect_merged_target_configs_from_env(
+        config,
+        &format!("{route_prefix}{target_type}").to_lowercase(),
+        route_prefix,
+        target_type,
+        valid_fields,
+        env_vars,
+    )?
+    .into_iter()
+    .filter(|record| record.enabled)
+    .map(|record| (record.instance_id, record.effective_config))
+    .collect())
+}
+
+pub fn collect_target_config_results_from_env<I>(
+    config: &Config,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+    env_vars: I,
+) -> (Vec<(String, KVS)>, Vec<String>)
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    let mut configs = Vec::new();
+    let mut failures = Vec::new();
+    for result in collect_merged_target_config_results_from_env(
+        config,
+        &format!("{route_prefix}{target_type}").to_lowercase(),
+        route_prefix,
+        target_type,
+        valid_fields,
+        env_vars,
+    ) {
+        match result {
+            Ok(record) if record.enabled => configs.push((record.instance_id, record.effective_config)),
+            Ok(_) => {}
+            Err((record, err)) => failures.push(format!("{target_type}/{}: {err}", record.instance_id)),
+        }
+    }
+    (configs, failures)
+}
+
 pub(crate) fn collect_merged_target_configs_from_env<I>(
    config: &Config,
    section_name: &str,
@@ -144,7 +223,52 @@ pub(crate) fn collect_merged_target_configs_from_env<I>(
    target_type: &str,
    valid_fields: &HashSet<String>,
    env_vars: I,
+) -> Result<Vec<MergedTargetConfigRecord>, TargetError>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    collect_merged_target_config_results_from_env(config, section_name, route_prefix, target_type, valid_fields, env_vars)
+        .into_iter()
+        .map(|result| result.map_err(|(_, err)| err))
+        .collect()
+}
+
+pub(crate) fn collect_merged_target_configs_compat_from_env<I>(
+    config: &Config,
+    section_name: &str,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+    env_vars: I,
 ) -> Vec<MergedTargetConfigRecord>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    collect_merged_target_config_results_from_env(config, section_name, route_prefix, target_type, valid_fields, env_vars)
+        .into_iter()
+        .map(|result| match result {
+            Ok(record) => record,
+            Err((record, err)) => {
+                warn!(
+                    target_type,
+                    instance_id = %record.instance_id,
+                    error = %err,
+                    "Treating target instance with invalid enable configuration as disabled"
+                );
+                record
+            }
+        })
+        .collect()
+}
+
+fn collect_merged_target_config_results_from_env<I>(
+    config: &Config,
+    section_name: &str,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+    env_vars: I,
+) -> Vec<Result<MergedTargetConfigRecord, (MergedTargetConfigRecord, TargetError)>>
 where
    I: IntoIterator<Item = (String, String)>,
 {
@@ -220,14 +344,28 @@ where
            let redacted_config = redacted_target_config(&merged_config);
            debug!(instance_id = %id, ?redacted_config, "Merged target configuration");
        }
-        merged_configs.push(MergedTargetConfigRecord {
-            instance_id: id,
-            enabled: is_target_enabled(&merged_config),
-            effective_config: merged_config,
-            has_file_default,
-            has_file_instance,
-            has_env_default,
-            has_env_instance,
+        merged_configs.push(match is_target_enabled(&merged_config) {
+            Ok(enabled) => Ok(MergedTargetConfigRecord {
+                instance_id: id,
+                enabled,
+                effective_config: merged_config,
+                has_file_default,
+                has_file_instance,
+                has_env_default,
+                has_env_instance,
+            }),
+            Err(err) => Err((
+                MergedTargetConfigRecord {
+                    instance_id: id,
+                    enabled: false,
+                    effective_config: merged_config,
+                    has_file_default,
+                    has_file_instance,
+                    has_env_default,
+                    has_env_instance,
+                },
+                err,
+            )),
        });
    }

@@ -237,9 +375,10 @@ where
 #[cfg(test)]
 mod tests {
    use super::{
-        collect_env_target_instance_ids_from_env, collect_target_configs_from_env, redact_target_field_value,
-        redacted_target_config,
+        collect_env_target_instance_ids_from_env, collect_target_config_results_from_env, collect_target_configs_from_env,
+        redact_target_field_value, redacted_target_config, try_collect_target_configs_from_env,
    };
+    use crate::TargetError;
    use rustfs_config::notify::{
        ENV_NOTIFY_REDIS_ENABLE, ENV_NOTIFY_REDIS_RECONNECT_RETRY_ATTEMPTS, ENV_NOTIFY_REDIS_TLS_ALLOW_INSECURE,
        ENV_NOTIFY_REDIS_URL, NOTIFY_REDIS_KEYS, NOTIFY_ROUTE_PREFIX,
@@ -269,7 +408,7 @@ mod tests {

        cfg.0.insert("notify_webhook".to_string(), subsystem);

-        let configs = collect_target_configs_from_env(
+        let configs = try_collect_target_configs_from_env(
            &cfg,
            NOTIFY_ROUTE_PREFIX,
            "webhook",
@@ -282,7 +421,8 @@ mod tests {
                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE".to_string(), "on".to_string()),
                ("RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT".to_string(), "42".to_string()),
            ],
-        );
+        )
+        .expect("valid env target");

        let configs: HashMap<String, KVS> = configs.into_iter().collect();
        assert_eq!(configs.len(), 2);
@@ -295,7 +435,7 @@ mod tests {
    #[test]
    fn collect_target_configs_discovers_enabled_instance_from_env() {
        let cfg = Config(HashMap::new());
-        let configs = collect_target_configs_from_env(
+        let configs = try_collect_target_configs_from_env(
            &cfg,
            NOTIFY_ROUTE_PREFIX,
            "webhook",
@@ -307,7 +447,8 @@ mod tests {
                    "https://example.com/from-env".to_string(),
                ),
            ],
-        );
+        )
+        .expect("valid target configs");

        assert_eq!(configs.len(), 1);
        assert_eq!(configs[0].0, "primary");
@@ -323,7 +464,7 @@ mod tests {
        subsystem.insert("_".to_string(), default_kvs);
        cfg.0.insert("notify_webhook".to_string(), subsystem);

-        let configs = collect_target_configs_from_env(
+        let configs = try_collect_target_configs_from_env(
            &cfg,
            NOTIFY_ROUTE_PREFIX,
            "webhook",
@@ -332,7 +473,8 @@ mod tests {
                "RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_SECONDARY".to_string(),
                "https://example.com/secondary".to_string(),
            )],
-        );
+        )
+        .expect("valid target configs");

        assert!(configs.is_empty());
    }
@@ -361,7 +503,7 @@ mod tests {
        let cfg = Config(HashMap::new());
        let valid_fields = NOTIFY_REDIS_KEYS.iter().map(|key| (*key).to_string()).collect();

-        let configs = collect_target_configs_from_env(
+        let configs = try_collect_target_configs_from_env(
            &cfg,
            NOTIFY_ROUTE_PREFIX,
            "redis",
@@ -372,7 +514,8 @@ mod tests {
                (format!("{ENV_NOTIFY_REDIS_RECONNECT_RETRY_ATTEMPTS}_PRIMARY"), "9".to_string()),
                (format!("{ENV_NOTIFY_REDIS_TLS_ALLOW_INSECURE}_PRIMARY"), "off".to_string()),
            ],
-        );
+        )
+        .expect("valid redis target config");

        let configs: HashMap<String, KVS> = configs.into_iter().collect();
        let redis_config = configs.get("primary").expect("redis env target should be discovered");
@@ -383,6 +526,99 @@ mod tests {
        assert_eq!(redis_config.lookup(REDIS_TLS_ALLOW_INSECURE).as_deref(), Some("off"));
    }

+    #[test]
+    fn collect_target_configs_rejects_invalid_instance_enable_value() {
+        let err = try_collect_target_configs_from_env(
+            &Config(HashMap::new()),
+            NOTIFY_ROUTE_PREFIX,
+            "webhook",
+            &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
+            vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), "invalid".to_string())],
+        )
+        .expect_err("invalid enable value must not look like a disabled target");
+
+        match err {
+            TargetError::Configuration(detail) => assert_eq!(detail, "Invalid enable value 'invalid'"),
+            other => panic!("expected a configuration error, got {other}"),
+        }
+    }
+
+    #[test]
+    fn legacy_collection_keeps_valid_instances_when_one_enable_is_invalid() {
+        let configs = collect_target_configs_from_env(
+            &Config(HashMap::new()),
+            NOTIFY_ROUTE_PREFIX,
+            "webhook",
+            &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
+            vec![
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_GOOD".to_string(), "on".to_string()),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_GOOD".to_string(), "https://example.com/good".to_string()),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_BAD".to_string(), "invalid".to_string()),
+            ],
+        );
+
+        assert_eq!(configs.len(), 1);
+        assert_eq!(configs[0].0, "good");
+    }
+
+    #[test]
+    fn collect_target_config_results_isolates_invalid_instance_and_reports_it() {
+        let (configs, failures) = collect_target_config_results_from_env(
+            &Config(HashMap::new()),
+            NOTIFY_ROUTE_PREFIX,
+            "webhook",
+            &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
+            vec![
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_GOOD".to_string(), "on".to_string()),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_GOOD".to_string(), "https://example.com/good".to_string()),
+                // "enable" is a common typo: EnableState accepts "enabled"/"on", not "enable".
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_BAD".to_string(), "enable".to_string()),
+            ],
+        );
+
+        // The healthy instance still loads even though a sibling is malformed...
+        assert_eq!(configs.len(), 1);
+        assert_eq!(configs[0].0, "good");
+        // ...and the malformed instance is surfaced (not silently dropped, not
+        // aborting the whole target type) so callers can fail-report it.
+        assert_eq!(failures.len(), 1);
+        assert!(
+            failures[0].contains("webhook/bad") && failures[0].contains("enable"),
+            "unexpected failure summary: {}",
+            failures[0]
+        );
+    }
+
+    #[test]
+    fn collect_target_config_results_does_not_report_validly_disabled_instances() {
+        let (configs, failures) = collect_target_config_results_from_env(
+            &Config(HashMap::new()),
+            NOTIFY_ROUTE_PREFIX,
+            "webhook",
+            &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
+            vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), "off".to_string())],
+        );
+
+        assert!(configs.is_empty());
+        assert!(failures.is_empty(), "a validly disabled instance is not a failure");
+    }
+
+    #[test]
+    fn collect_target_configs_preserves_whitespace_padded_legacy_value() {
+        let configs = try_collect_target_configs_from_env(
+            &Config(HashMap::new()),
+            NOTIFY_ROUTE_PREFIX,
+            "webhook",
+            &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
+            vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), " on ".to_string())],
+        )
+        .expect("the shared enable parser accepts surrounding whitespace");
+
+        assert_eq!(configs.len(), 1);
+        assert_eq!(configs[0].0, "primary");
+        assert_eq!(configs[0].1.lookup(ENABLE_KEY).as_deref(), Some(" on "));
+    }
+
    #[test]
    fn redact_target_field_value_redacts_sensitive_fields() {
        assert_eq!(redact_target_field_value("password", "secret"), "***redacted***");
@@ -21,10 +21,12 @@ pub use instance::{
    LegacyTargetInstanceDescriptor, TargetInstanceSourceClass, TargetInstanceSourceHints, TargetPluginInstance,
    TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, normalize_legacy_target_instances,
    normalize_legacy_target_instances_from_env, normalize_target_plugin_instances, normalize_target_plugin_instances_from_env,
+    try_normalize_target_plugin_instances, try_normalize_target_plugin_instances_from_env,
 };
 pub use loader::{
-    collect_env_target_instance_ids, collect_env_target_instance_ids_from_env, collect_target_configs,
-    collect_target_configs_from_env,
+    collect_env_target_instance_ids, collect_env_target_instance_ids_from_env, collect_target_config_results,
+    collect_target_config_results_from_env, collect_target_configs, collect_target_configs_from_env, try_collect_target_configs,
+    try_collect_target_configs_from_env,
 };
 pub use target_args::{
    build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args,
@@ -43,6 +43,7 @@ pub use config::{
    LegacyTargetInstanceDescriptor, TargetInstanceSourceClass, TargetInstanceSourceHints, TargetPluginInstance,
    TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, normalize_legacy_target_instances,
    normalize_legacy_target_instances_from_env, normalize_target_plugin_instances, normalize_target_plugin_instances_from_env,
+    try_normalize_target_plugin_instances, try_normalize_target_plugin_instances_from_env,
 };
 pub use control_plane::{
    TargetPluginEnableState, TargetPluginExternalAction, TargetPluginExternalActionDecision, TargetPluginExternalActionError,
@@ -65,10 +66,11 @@ pub use plugin::{
    TargetPluginRegistry, TargetRequestValidator, boxed_target,
 };
 pub use runtime::{
-    ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot,
-    RuntimeTargetHealthState, RuntimeTargetSnapshot, SharedTarget, TargetRuntimeManager, activate_targets_with_replay,
+    OpenedActivation, PreparedActivation, ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot,
+    RuntimeTargetHealthReason, RuntimeTargetHealthSnapshot, RuntimeTargetHealthState, RuntimeTargetSnapshot, SharedTarget,
+    TargetRuntimeManager, activate_targets_with_replay,
    adapter::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter},
-    init_target_and_optionally_start_replay,
+    health_snapshots_for_targets, init_target_and_optionally_start_replay,
    ops_diagnostics::{
        OpsDiagnosticsAccessDecision, OpsDiagnosticsReadRequest, OpsDiagnosticsRegistration, OpsDiagnosticsRegistry,
        OpsDiagnosticsRegistryError,
@@ -85,6 +87,7 @@ pub use rustfs_s3_types::EventName;
 use serde::{Deserialize, Serialize};
 pub use sys::user_agent::*;
 pub use target::{Target, TargetDeliverySnapshot};
+pub use target::{TargetHealth, TargetHealthReason, TargetHealthState};

 /// Represents a log of events for sending to targets
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -14,8 +14,9 @@

 use crate::{
    PluginRuntimeAdapter, RuntimeActivation, Target, TargetError,
-    config::collect_target_configs,
+    config::collect_target_config_results,
    manifest::{TargetPluginManifest, builtin_target_manifest},
+    target::with_deferred_queue_store_open,
 };
 use hashbrown::HashMap;
 use rustfs_config::server_config::{Config, KVS};
@@ -317,39 +318,77 @@ where
        config: &Config,
        route_prefix: &str,
    ) -> Result<Vec<BoxedTarget<E>>, TargetError> {
+        self.create_targets_from_config_with_store_mode(config, route_prefix, false)
+            .await
+            .map(|(targets, _)| targets)
+    }
+
+    /// Creates targets while deferring queue-store open until runtime handoff.
+    /// Unlike the compatibility activation API, lifecycle preparation reports
+    /// any invalid or unconstructable configured instance so the originating
+    /// Admin request cannot report a false success.
+    pub async fn create_dormant_targets_from_config(
+        &self,
+        config: &Config,
+        route_prefix: &str,
+    ) -> Result<(Vec<BoxedTarget<E>>, Vec<String>), TargetError> {
+        self.create_targets_from_config_with_store_mode(config, route_prefix, true)
+            .await
+    }
+
+    async fn create_targets_from_config_with_store_mode(
+        &self,
+        config: &Config,
+        route_prefix: &str,
+        defer_store_open: bool,
+    ) -> Result<(Vec<BoxedTarget<E>>, Vec<String>), TargetError> {
        let mut successful_targets = Vec::new();
-        let mut failed_targets = 0usize;
+        let mut failures = Vec::new();

        for (target_type, plugin) in &self.plugins {
            info!(target_type = %target_type, "Start working on target type");
-            for (id, merged_config) in collect_target_configs(config, route_prefix, target_type, plugin.valid_fields_set()) {
+            // Per-instance fault isolation: an invalid instance (e.g. an
+            // unparseable `enable` value) is recorded as a failure and skipped,
+            // never aborting the remaining instances or other target types.
+            let (collected, invalid_instances) =
+                collect_target_config_results(config, route_prefix, target_type, plugin.valid_fields_set());
+            for detail in invalid_instances {
+                error!(target_type = %target_type, reason = "invalid_config", detail = %detail, "Skipping target instance with invalid configuration");
+                failures.push(detail);
+            }
+            for (id, merged_config) in collected {
                info!(target_type = %target_type, instance_id = %id, "Target is enabled, ready to create");
-                match self.create_target(target_type, id.clone(), &merged_config) {
+                let created = if defer_store_open {
+                    with_deferred_queue_store_open(|| self.create_target(target_type, id.clone(), &merged_config))
+                } else {
+                    self.create_target(target_type, id.clone(), &merged_config)
+                };
+                match created {
                    Ok(target) => {
                        info!(target_type = %target.id().name, instance_id = %id, "Create target successfully");
                        successful_targets.push(target);
                    }
-                    Err(err) => {
-                        failed_targets += 1;
-                        error!(target_type = %target_type, instance_id = %id, error = %err, "Failed to create target");
+                    Err(_) => {
+                        failures.push(format!("{target_type}/{id}: target construction failed"));
+                        error!(target_type = %target_type, instance_id = %id, reason = "construction_failed", "Failed to create target");
                    }
                }
            }
        }

-        if failed_targets > 0 {
+        if !failures.is_empty() {
            warn!(
                created = successful_targets.len(),
-                failed = failed_targets,
+                failed = failures.len(),
                "Some configured targets failed to create and were skipped"
            );
        }
        info!(
            count = successful_targets.len(),
-            failed = failed_targets,
+            failed = failures.len(),
            "All target processing completed"
        );
-        Ok(successful_targets)
+        Ok((successful_targets, failures))
    }

    pub async fn create_activation_from_config<A>(
@@ -475,4 +514,61 @@ mod tests {
        assert_eq!(activation.targets[0].id().to_string(), "primary:test");
        assert!(activation.replay_workers.is_empty());
    }
+
+    // Regression: a single instance with a malformed `enable` value must not
+    // abort the remaining instances or unrelated target types. Before this fix
+    // the collector short-circuited the whole create path, so one typo took
+    // down every notify/audit target.
+    #[tokio::test]
+    async fn create_dormant_isolates_invalid_enable_and_still_loads_other_targets() {
+        let mut registry = TargetPluginRegistry::<String>::new();
+        for target_type in ["alpha", "beta"] {
+            registry.register(TargetPluginDescriptor::new(
+                target_type,
+                &[ENABLE_KEY, "endpoint"],
+                |_config| Ok(()),
+                move |id, _config| {
+                    Ok(Box::new(TestTarget {
+                        id: crate::arn::TargetID::new(id, target_type.to_string()),
+                    }))
+                },
+            ));
+        }
+
+        let mut cfg = Config(HashMap::new());
+
+        // alpha: one healthy instance plus one with a malformed `enable` value
+        // ("enable" is a typo -- EnableState accepts "enabled"/"on", not "enable").
+        let mut alpha = HashMap::new();
+        let mut alpha_good = KVS::new();
+        alpha_good.insert(ENABLE_KEY.to_string(), "on".to_string());
+        alpha_good.insert("endpoint".to_string(), "https://example.com/alpha".to_string());
+        alpha.insert("good".to_string(), alpha_good);
+        let mut alpha_bad = KVS::new();
+        alpha_bad.insert(ENABLE_KEY.to_string(), "enable".to_string());
+        alpha.insert("bad".to_string(), alpha_bad);
+        cfg.0.insert("notify_alpha".to_string(), alpha);
+
+        // beta: a healthy instance in a different target type must survive.
+        let mut beta = HashMap::new();
+        let mut beta_primary = KVS::new();
+        beta_primary.insert(ENABLE_KEY.to_string(), "on".to_string());
+        beta_primary.insert("endpoint".to_string(), "https://example.com/beta".to_string());
+        beta.insert("primary".to_string(), beta_primary);
+        cfg.0.insert("notify_beta".to_string(), beta);
+
+        let (targets, failures) = registry
+            .create_dormant_targets_from_config(&cfg, "notify_")
+            .await
+            .expect("a malformed instance must not abort target creation");
+
+        let mut created: Vec<String> = targets.iter().map(|target| target.id().to_string()).collect();
+        created.sort();
+        assert_eq!(created, vec!["good:alpha".to_string(), "primary:beta".to_string()]);
+
+        // The malformed instance is surfaced (so an Admin write can't report a
+        // false success) rather than silently dropped or fatally aborting.
+        assert_eq!(failures.len(), 1);
+        assert!(failures[0].contains("alpha/bad"), "unexpected failure summary: {}", failures[0]);
+    }
 }
@@ -13,21 +13,53 @@
 // limitations under the License.

 use super::{
-    ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot,
-    TargetRuntimeManager, activate_targets_with_replay, init_target_and_optionally_start_replay, start_replay_worker,
+    OpenedActivation, PrepareTargetResult, PreparedActivation, ReplayEvent, ReplayWorkerManager, RuntimeActivation,
+    RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot, TargetActivationFailure, TargetRuntimeManager, prepare_target,
+    start_replay_worker,
 };
 use crate::plugin::PluginEvent;
-use crate::{Target, TargetError};
+use crate::{SharedTarget, Target, TargetError};
 use async_trait::async_trait;
+use rayon::prelude::*;
 use std::future::Future;
+use std::panic::{AssertUnwindSafe, catch_unwind};
 use std::pin::Pin;
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock};
 use std::time::Duration;
 use tokio::sync::Semaphore;
+use tokio_util::sync::CancellationToken;

 type ReplayHook<E> = Arc<dyn Fn(ReplayEvent<E>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
 type ReplayStartObserver = Arc<dyn Fn(&str, bool) + Send + Sync>;

+const MAX_PARALLEL_STORE_OPENS: usize = 4;
+
+static STORE_OPEN_POOL: LazyLock<Result<rayon::ThreadPool, rayon::ThreadPoolBuildError>> = LazyLock::new(|| {
+    rayon::ThreadPoolBuilder::new()
+        .num_threads(MAX_PARALLEL_STORE_OPENS)
+        .thread_name(|index| format!("rustfs-target-store-open-{index}"))
+        .build()
+});
+
+enum StoreOpenOutcome<E>
+where
+    E: PluginEvent,
+{
+    Accepted(SharedTarget<E>),
+    Rejected { panicked: bool, target: SharedTarget<E> },
+}
+
+fn open_target_store<E>(target: SharedTarget<E>) -> StoreOpenOutcome<E>
+where
+    E: PluginEvent,
+{
+    match catch_unwind(AssertUnwindSafe(|| target.store().map(|store| store.open()))) {
+        Ok(None | Some(Ok(()))) => StoreOpenOutcome::Accepted(target),
+        Ok(Some(Err(_))) => StoreOpenOutcome::Rejected { panicked: false, target },
+        Err(_) => StoreOpenOutcome::Rejected { panicked: true, target },
+    }
+}
+
 /// Shared runtime contract for target plugins.
 #[async_trait]
 pub trait PluginRuntimeAdapter<E>: Send + Sync
@@ -96,6 +128,234 @@ where
            stop_log_prefix: stop_log_prefix.into(),
        }
    }
+
+    pub async fn prepare_targets(&self, targets: Vec<Box<dyn Target<E> + Send + Sync>>) -> PreparedActivation<E> {
+        self.prepare_targets_inner(targets, None).await
+    }
+
+    pub async fn prepare_targets_cancellable(
+        &self,
+        targets: Vec<Box<dyn Target<E> + Send + Sync>>,
+        cancellation: &CancellationToken,
+    ) -> PreparedActivation<E> {
+        self.prepare_targets_inner(targets, Some(cancellation)).await
+    }
+
+    async fn prepare_targets_inner(
+        &self,
+        targets: Vec<Box<dyn Target<E> + Send + Sync>>,
+        cancellation: Option<&CancellationToken>,
+    ) -> PreparedActivation<E> {
+        let mut prepared = Vec::with_capacity(targets.len());
+        let mut failures = Vec::new();
+        let mut rejected_targets = Vec::new();
+        let mut targets = targets.into_iter();
+        while let Some(target) = targets.next() {
+            match prepare_target(target, cancellation).await {
+                PrepareTargetResult::Ready(target) => prepared.push(target),
+                PrepareTargetResult::Degraded { error, target } => {
+                    drop(error);
+                    tracing::warn!(
+                        target_id = %target.id(),
+                        reason = "initialization_failed",
+                        "Target initialization failed during lifecycle preparation"
+                    );
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{}: initialization failed", target.id()),
+                    });
+                    prepared.push(target);
+                }
+                PrepareTargetResult::Failed { error, target } => {
+                    drop(error);
+                    let target_id = target.id().to_string();
+                    tracing::warn!(
+                        target_id,
+                        reason = "initialization_failed",
+                        "Target initialization failed during lifecycle preparation"
+                    );
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{target_id}: initialization failed"),
+                    });
+                    rejected_targets.push(Arc::from(target));
+                }
+                PrepareTargetResult::Cancelled(target) => {
+                    prepared.push(Arc::from(target));
+                    prepared.extend(targets.map(Arc::from));
+                    break;
+                }
+            }
+        }
+        PreparedActivation {
+            failures,
+            rejected_targets,
+            targets: prepared,
+        }
+    }
+
+    /// Opens queue stores only after the previous runtime generation has been
+    /// quiesced. Targets whose stores cannot be opened retain the established
+    /// fault-isolation behavior and are returned for lock-free shutdown.
+    pub fn open_prepared_stores(&self, prepared: PreparedActivation<E>) -> (OpenedActivation<E>, PreparedActivation<E>) {
+        let mut accepted = Vec::with_capacity(prepared.targets.len());
+        let mut failures = prepared.failures;
+        let mut rejected = prepared.rejected_targets;
+        let outcomes = if prepared.targets.len() < 2 {
+            prepared.targets.into_iter().map(open_target_store).collect()
+        } else {
+            match STORE_OPEN_POOL.as_ref() {
+                // Vec's indexed parallel iterator preserves configuration
+                // order in collect, keeping failure summaries deterministic.
+                Ok(pool) => pool.install(|| prepared.targets.into_par_iter().map(open_target_store).collect::<Vec<_>>()),
+                Err(err) => {
+                    tracing::warn!(error = %err, "Failed to create target store open pool; opening stores serially");
+                    prepared.targets.into_iter().map(open_target_store).collect()
+                }
+            }
+        };
+        for outcome in outcomes {
+            match outcome {
+                StoreOpenOutcome::Accepted(target) => accepted.push(target),
+                StoreOpenOutcome::Rejected { panicked, target } => {
+                    if panicked {
+                        tracing::error!(
+                            target_id = %target.id(),
+                            reason = "store_open_panicked",
+                            "Target queue store panicked while opening during runtime handoff"
+                        );
+                    } else {
+                        tracing::error!(
+                            target_id = %target.id(),
+                            reason = "store_open_failed",
+                            "Failed to open target queue store during runtime handoff"
+                        );
+                    }
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{}: queue store open failed", target.id()),
+                    });
+                    rejected.push(target);
+                }
+            }
+        }
+        (
+            OpenedActivation { targets: accepted },
+            PreparedActivation {
+                failures,
+                rejected_targets: rejected,
+                targets: Vec::new(),
+            },
+        )
+    }
+
+    pub fn try_activate_prepared(&self, opened: OpenedActivation<E>) -> (RuntimeActivation<E>, PreparedActivation<E>) {
+        let mut replay_workers = ReplayWorkerManager::new();
+        let mut accepted = Vec::with_capacity(opened.targets.len());
+        let mut failures = Vec::new();
+        let mut rejected_targets = Vec::new();
+        for target in opened.targets {
+            let target_id = target.id().to_string();
+            let replay = catch_unwind(AssertUnwindSafe(|| {
+                target.store().filter(|_| target.is_enabled()).map(|store| {
+                    start_replay_worker(
+                        store.boxed_clone(),
+                        Arc::clone(&target),
+                        Arc::clone(&self.replay_hook),
+                        self.replay_semaphore.clone(),
+                        self.batch_timeout,
+                        self.idle_sleep,
+                    )
+                })
+            }));
+            let replay = match replay {
+                Ok(replay) => replay,
+                Err(_) => {
+                    tracing::error!(
+                        target_id,
+                        reason = "replay_activation_panicked",
+                        "Target replay activation panicked during runtime handoff"
+                    );
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{target_id}: replay activation failed"),
+                    });
+                    rejected_targets.push(target);
+                    continue;
+                }
+            };
+            (self.replay_start_observer)(&target_id, replay.is_some());
+            if let Some((cancel_tx, join)) = replay {
+                replay_workers.insert_with_handle(target_id, cancel_tx, join);
+            }
+            accepted.push(target);
+        }
+
+        (
+            RuntimeActivation {
+                replay_workers,
+                targets: accepted,
+            },
+            PreparedActivation {
+                failures,
+                rejected_targets,
+                targets: Vec::new(),
+            },
+        )
+    }
+
+    #[doc(hidden)]
+    pub async fn prepare_dormant_compat_activation(
+        &self,
+        targets: Vec<Box<dyn Target<E> + Send + Sync>>,
+    ) -> RuntimeActivation<E> {
+        let PreparedActivation {
+            failures,
+            rejected_targets,
+            targets,
+        } = self.prepare_targets(targets).await;
+        let rejected = PreparedActivation {
+            failures,
+            rejected_targets,
+            targets: Vec::new(),
+        };
+        if let Err(err) = self.close_prepared(rejected).await {
+            tracing::warn!(error = %err, "Failed to close targets rejected while preparing compatibility activation");
+        }
+        RuntimeActivation {
+            replay_workers: ReplayWorkerManager::new(),
+            targets,
+        }
+    }
+
+    #[doc(hidden)]
+    pub fn start_dormant_compat_activation(
+        &self,
+        activation: RuntimeActivation<E>,
+    ) -> (RuntimeActivation<E>, PreparedActivation<E>, PreparedActivation<E>) {
+        let prepared = PreparedActivation {
+            failures: Vec::new(),
+            rejected_targets: Vec::new(),
+            targets: activation.targets,
+        };
+        let (opened, open_rejected) = self.open_prepared_stores(prepared);
+        let (activation, activation_rejected) = self.try_activate_prepared(opened);
+        (activation, open_rejected, activation_rejected)
+    }
+
+    #[doc(hidden)]
+    pub async fn close_compat_activation(&self, mut activation: RuntimeActivation<E>) -> Result<(), TargetError> {
+        let mut runtime = TargetRuntimeManager::new();
+        for target in activation.targets {
+            runtime.add_arc(target);
+        }
+        self.shutdown(&mut runtime, &mut activation.replay_workers).await
+    }
+
+    pub async fn close_prepared(&self, prepared: PreparedActivation<E>) -> Result<(), TargetError> {
+        let mut runtime = TargetRuntimeManager::new();
+        for target in prepared.targets.into_iter().chain(prepared.rejected_targets) {
+            runtime.add_arc(target);
+        }
+        let mut replay_workers = ReplayWorkerManager::new();
+        self.shutdown(&mut runtime, &mut replay_workers).await
+    }
 }

 #[async_trait]
@@ -104,36 +364,16 @@ where
    E: PluginEvent,
 {
    async fn activate_with_replay(&self, targets: Vec<Box<dyn Target<E> + Send + Sync>>) -> RuntimeActivation<E> {
-        let replay_hook = Arc::clone(&self.replay_hook);
-        let replay_start_observer = Arc::clone(&self.replay_start_observer);
-        let replay_semaphore = self.replay_semaphore.clone();
-        let batch_timeout = self.batch_timeout;
-        let idle_sleep = self.idle_sleep;
-
-        activate_targets_with_replay(targets, move |target| {
-            let replay_hook = Arc::clone(&replay_hook);
-            let replay_start_observer = Arc::clone(&replay_start_observer);
-            let replay_semaphore = replay_semaphore.clone();
-
-            async move {
-                init_target_and_optionally_start_replay(
-                    target,
-                    move |target_id, has_replay| replay_start_observer(target_id, has_replay),
-                    move |store, target| {
-                        start_replay_worker(
-                            store,
-                            target,
-                            Arc::clone(&replay_hook),
-                            replay_semaphore.clone(),
-                            batch_timeout,
-                            idle_sleep,
-                        )
-                    },
-                )
-                .await
-            }
-        })
-        .await
+        let prepared = self.prepare_targets(targets).await;
+        let (opened, rejected) = self.open_prepared_stores(prepared);
+        if let Err(err) = self.close_prepared(rejected).await {
+            tracing::warn!(error = %err, "Failed to close targets whose queue stores could not be opened");
+        }
+        let (activation, rejected) = self.try_activate_prepared(opened);
+        if let Err(err) = self.close_prepared(rejected).await {
+            tracing::warn!(error = %err, "Failed to close targets rejected during replay activation");
+        }
+        activation
    }

    async fn replace_runtime_targets(
@@ -189,7 +429,7 @@ where
        if !close_errors.is_empty() {
            let detail = close_errors
                .into_iter()
-                .map(|(target_id, err)| format!("{target_id}: {err}"))
+                .map(|(target_id, _)| target_id)
                .collect::<Vec<_>>()
                .join("; ");
            return Err(TargetError::Storage(format!("Failed to close {detail}")));
@@ -200,24 +440,143 @@ where

 #[cfg(test)]
 mod tests {
-    use super::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter};
+    use super::{BuiltinPluginRuntimeAdapter, MAX_PARALLEL_STORE_OPENS, PluginRuntimeAdapter};
    use crate::PluginEvent;
    use crate::arn::TargetID;
    use crate::store::{Key, QueueStore, Store};
    use crate::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
    use crate::{StoreError, Target, TargetError};
    use async_trait::async_trait;
-    use std::sync::Arc;
    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::{Arc, Condvar, Mutex};
    use std::time::Duration;
    use tempfile::tempdir;
+    use tokio::sync::Notify;
+    use tokio_util::sync::CancellationToken;
+
+    type TestStore = dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync;
+
+    type BeforeOpen = Arc<dyn Fn() + Send + Sync>;
+
+    #[derive(Clone)]
+    struct TestOpenStore {
+        before_clone: BeforeOpen,
+        before_open: BeforeOpen,
+        store: QueueStore<QueuedPayload>,
+    }
+
+    impl Store<QueuedPayload> for TestOpenStore {
+        type Error = StoreError;
+        type Key = Key;
+
+        fn open(&self) -> Result<(), Self::Error> {
+            (self.before_open)();
+            self.store.open()
+        }
+
+        fn put(&self, item: Arc<QueuedPayload>) -> Result<Self::Key, Self::Error> {
+            self.store.put(item)
+        }
+
+        fn put_multiple(&self, items: Vec<QueuedPayload>) -> Result<Self::Key, Self::Error> {
+            self.store.put_multiple(items)
+        }
+
+        fn put_raw(&self, data: &[u8]) -> Result<Self::Key, Self::Error> {
+            self.store.put_raw(data)
+        }
+
+        fn get(&self, key: &Self::Key) -> Result<QueuedPayload, Self::Error> {
+            self.store.get(key)
+        }
+
+        fn get_multiple(&self, key: &Self::Key) -> Result<Vec<QueuedPayload>, Self::Error> {
+            self.store.get_multiple(key)
+        }
+
+        fn get_raw(&self, key: &Self::Key) -> Result<Vec<u8>, Self::Error> {
+            self.store.get_raw(key)
+        }
+
+        fn del(&self, key: &Self::Key) -> Result<(), Self::Error> {
+            self.store.del(key)
+        }
+
+        fn delete(&self) -> Result<(), Self::Error> {
+            self.store.delete()
+        }
+
+        fn list(&self) -> Vec<Self::Key> {
+            self.store.list()
+        }
+
+        fn len(&self) -> usize {
+            self.store.len()
+        }
+
+        fn is_empty(&self) -> bool {
+            self.store.is_empty()
+        }
+
+        fn boxed_clone(&self) -> Box<dyn Store<QueuedPayload, Error = Self::Error, Key = Self::Key> + Send + Sync> {
+            (self.before_clone)();
+            Box::new(self.clone())
+        }
+    }
+
+    #[derive(Default)]
+    struct StoreOpenGate {
+        changed: Condvar,
+        state: Mutex<StoreOpenGateState>,
+    }
+
+    #[derive(Default)]
+    struct StoreOpenGateState {
+        active: usize,
+        max_active: usize,
+        released: bool,
+    }
+
+    impl StoreOpenGate {
+        fn enter(&self) {
+            let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner());
+            state.active += 1;
+            state.max_active = state.max_active.max(state.active);
+            self.changed.notify_all();
+            while !state.released {
+                state = self.changed.wait(state).unwrap_or_else(|err| err.into_inner());
+            }
+            state.active -= 1;
+        }
+
+        fn wait_for_active(&self, expected: usize, timeout: Duration) -> bool {
+            let state = self.state.lock().unwrap_or_else(|err| err.into_inner());
+            let (state, _) = self
+                .changed
+                .wait_timeout_while(state, timeout, |state| state.max_active < expected)
+                .unwrap_or_else(|err| err.into_inner());
+            state.max_active >= expected
+        }
+
+        fn release(&self) {
+            let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner());
+            state.released = true;
+            self.changed.notify_all();
+        }
+
+        fn max_active(&self) -> usize {
+            self.state.lock().unwrap_or_else(|err| err.into_inner()).max_active
+        }
+    }

    #[derive(Clone)]
    struct TestTarget {
        close_calls: Arc<AtomicUsize>,
        id: TargetID,
+        init_calls: Arc<AtomicUsize>,
+        init_entered: Option<Arc<Notify>>,
        init_fails: bool,
-        store: Option<QueueStore<QueuedPayload>>,
+        store: Option<Arc<TestStore>>,
    }

    impl TestTarget {
@@ -225,6 +584,8 @@ mod tests {
            Self {
                close_calls: Arc::new(AtomicUsize::new(0)),
                id: TargetID::new(id.to_string(), name.to_string()),
+                init_calls: Arc::new(AtomicUsize::new(0)),
+                init_entered: None,
                init_fails: false,
                store: None,
            }
@@ -235,11 +596,16 @@ mod tests {
            self
        }

+        fn with_pending_init(mut self, init_entered: Arc<Notify>) -> Self {
+            self.init_entered = Some(init_entered);
+            self
+        }
+
        fn with_store(mut self) -> Self {
            let dir = tempdir().expect("tempdir should be created for queue store tests");
            let store = QueueStore::<QueuedPayload>::new(dir.path(), 16, ".queue");
            store.open().expect("queue store should open");
-            self.store = Some(store);
+            self.store = Some(Arc::new(store));
            self
        }
    }
@@ -271,9 +637,7 @@ mod tests {
        }

        fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
-            self.store
-                .as_ref()
-                .map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
+            self.store.as_deref()
        }

        fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
@@ -281,6 +645,11 @@ mod tests {
        }

        async fn init(&self) -> Result<(), TargetError> {
+            self.init_calls.fetch_add(1, Ordering::SeqCst);
+            if let Some(init_entered) = &self.init_entered {
+                init_entered.notify_one();
+                return std::future::pending().await;
+            }
            if self.init_fails {
                return Err(TargetError::Configuration("forced init failure".to_string()));
            }
@@ -334,6 +703,212 @@ mod tests {
        assert_eq!(activation.replay_workers.len(), 1);
    }

+    #[tokio::test]
+    async fn prepared_store_target_reports_init_failure_without_dropping_queue_runtime() {
+        let adapter = builtin_adapter();
+        let target = TestTarget::new("primary", "webhook").with_failed_init().with_store();
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        assert_eq!(prepared.targets.len(), 1);
+        assert!(
+            prepared
+                .failure_summary()
+                .is_some_and(|summary| summary.contains("initialization failed") && !summary.contains("forced init failure"))
+        );
+
+        let (opened, rejected) = adapter.open_prepared_stores(prepared);
+        assert!(rejected.failure_summary().is_some());
+        let (mut activation, activation_rejected) = adapter.try_activate_prepared(opened);
+        assert!(activation_rejected.failure_summary().is_none());
+        assert_eq!(activation.targets.len(), 1);
+        assert_eq!(activation.replay_workers.len(), 1);
+        activation.replay_workers.stop_all("stop degraded target replay worker").await;
+    }
+
+    #[tokio::test]
+    async fn cancellable_preparation_returns_current_and_remaining_targets_for_shutdown() {
+        let adapter = builtin_adapter();
+        let init_entered = Arc::new(Notify::new());
+        let first = TestTarget::new("first", "webhook").with_pending_init(init_entered.clone());
+        let first_close_calls = first.close_calls.clone();
+        let second = TestTarget::new("second", "webhook");
+        let second_close_calls = second.close_calls.clone();
+        let second_init_calls = second.init_calls.clone();
+        let cancellation = CancellationToken::new();
+        let prepare_adapter = adapter.clone();
+        let prepare_cancellation = cancellation.clone();
+        let prepare = tokio::spawn(async move {
+            prepare_adapter
+                .prepare_targets_cancellable(vec![Box::new(first), Box::new(second)], &prepare_cancellation)
+                .await
+        });
+
+        init_entered.notified().await;
+        cancellation.cancel();
+        let prepared = tokio::time::timeout(Duration::from_secs(1), prepare)
+            .await
+            .expect("cancellation should interrupt target initialization")
+            .expect("preparation task should finish");
+        assert_eq!(prepared.targets.len(), 2);
+
+        adapter
+            .close_prepared(prepared)
+            .await
+            .expect("cancelled targets should close");
+        assert_eq!(first_close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(second_close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(second_init_calls.load(Ordering::SeqCst), 0);
+    }
+
+    #[tokio::test]
+    async fn prepared_activation_opens_store_before_starting_replay() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let queue_path = dir.path().join("queue");
+        let mut target = TestTarget::new("primary", "webhook");
+        target.store = Some(Arc::new(QueueStore::<QueuedPayload>::new(&queue_path, 16, ".queue")));
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        assert!(!queue_path.exists(), "dormant preparation must not open the queue store");
+
+        let (opened, rejected) = adapter.open_prepared_stores(prepared);
+        assert!(rejected.targets.is_empty());
+        assert!(queue_path.is_dir(), "handoff must open the queue store before activation");
+        let (mut activation, activation_rejected) = adapter.try_activate_prepared(opened);
+        assert!(activation_rejected.failure_summary().is_none());
+        assert_eq!(activation.replay_workers.len(), 1);
+        activation
+            .replay_workers
+            .stop_all("stop prepared activation test worker")
+            .await;
+    }
+
+    #[tokio::test]
+    async fn activation_closes_a_target_when_its_queue_store_cannot_open() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let invalid_base = dir.path().join("not-a-directory");
+        std::fs::write(&invalid_base, b"file").expect("invalid queue base should be created");
+        let mut target = TestTarget::new("primary", "webhook");
+        let close_calls = target.close_calls.clone();
+        target.store = Some(Arc::new(QueueStore::<QueuedPayload>::new(&invalid_base, 16, ".queue")));
+
+        let activation = adapter.activate_with_replay(vec![Box::new(target)]).await;
+
+        assert!(activation.targets.is_empty());
+        assert!(activation.replay_workers.is_empty());
+        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn prepared_stores_open_with_bounded_parallelism_and_stable_order() {
+        const TARGETS: usize = MAX_PARALLEL_STORE_OPENS * 2;
+
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let gate = Arc::new(StoreOpenGate::default());
+        let mut targets: Vec<Box<dyn Target<String> + Send + Sync>> = Vec::with_capacity(TARGETS);
+        let mut expected_ids = Vec::with_capacity(TARGETS);
+        for index in 0..TARGETS {
+            let mut target = TestTarget::new(&format!("target-{index}"), "webhook");
+            expected_ids.push(target.id.to_string());
+            let open_gate = gate.clone();
+            target.store = Some(Arc::new(TestOpenStore {
+                before_clone: Arc::new(|| {}),
+                before_open: Arc::new(move || open_gate.enter()),
+                store: QueueStore::new(dir.path().join(index.to_string()), 16, ".queue"),
+            }));
+            targets.push(Box::new(target));
+        }
+
+        let prepared = adapter.prepare_targets(targets).await;
+        let open_adapter = adapter.clone();
+        let opening = tokio::task::spawn_blocking(move || open_adapter.open_prepared_stores(prepared));
+        let wait_gate = gate.clone();
+        let reached_bound =
+            tokio::task::spawn_blocking(move || wait_gate.wait_for_active(MAX_PARALLEL_STORE_OPENS, Duration::from_secs(30)))
+                .await
+                .expect("store-open observer should not panic");
+        gate.release();
+        let (opened, rejected) = opening.await.expect("bounded store opens should not panic");
+        let opened_ids = opened
+            .targets
+            .iter()
+            .map(|target| target.id().to_string())
+            .collect::<Vec<_>>();
+
+        assert!(reached_bound, "store opens did not use the configured parallelism");
+        assert_eq!(gate.max_active(), MAX_PARALLEL_STORE_OPENS);
+        assert_eq!(opened_ids, expected_ids, "parallel store opens must preserve configuration order");
+        assert!(rejected.targets.is_empty());
+        assert!(rejected.failure_summary().is_none());
+    }
+
+    #[tokio::test]
+    async fn panicking_store_open_rejects_and_closes_only_that_target() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let mut target = TestTarget::new("panicking", "webhook");
+        let close_calls = target.close_calls.clone();
+        target.store = Some(Arc::new(TestOpenStore {
+            before_clone: Arc::new(|| {}),
+            before_open: Arc::new(|| panic!("forced store open panic: do-not-expose-payload")),
+            store: QueueStore::new(dir.path(), 16, ".queue"),
+        }));
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        let (opened, rejected) = adapter.open_prepared_stores(prepared);
+        let summary = rejected
+            .failure_summary()
+            .expect("panicking store should report a generic activation failure");
+        let (activation, activation_rejected) = adapter.try_activate_prepared(opened);
+
+        assert!(activation.targets.is_empty(), "a target without an open store must not become visible");
+        assert!(
+            activation.replay_workers.is_empty(),
+            "a rejected target must not publish without a replay worker"
+        );
+        assert!(activation_rejected.failure_summary().is_none());
+        assert!(summary.contains("queue store open failed"));
+        assert!(!summary.contains("do-not-expose-payload"));
+        adapter
+            .close_prepared(rejected)
+            .await
+            .expect("a target rejected after a store panic should close");
+        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn panicking_store_clone_cannot_publish_target_without_replay_worker() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let mut target = TestTarget::new("panicking-clone", "webhook");
+        let close_calls = target.close_calls.clone();
+        target.store = Some(Arc::new(TestOpenStore {
+            before_clone: Arc::new(|| panic!("forced store clone panic: do-not-expose-payload")),
+            before_open: Arc::new(|| {}),
+            store: QueueStore::new(dir.path(), 16, ".queue"),
+        }));
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        let (opened, open_rejected) = adapter.open_prepared_stores(prepared);
+        assert!(open_rejected.failure_summary().is_none());
+        let (activation, rejected) = adapter.try_activate_prepared(opened);
+        let summary = rejected
+            .failure_summary()
+            .expect("panicking store clone should report a generic activation failure");
+
+        assert!(activation.targets.is_empty(), "a target without a replay worker must not become visible");
+        assert!(activation.replay_workers.is_empty());
+        assert!(summary.contains("replay activation failed"));
+        assert!(!summary.contains("do-not-expose-payload"));
+        adapter
+            .close_prepared(rejected)
+            .await
+            .expect("a target rejected during replay activation should close");
+        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+    }
+
    #[tokio::test]
    async fn builtin_adapter_shutdown_clears_runtime_and_replay_workers() {
        let adapter = builtin_adapter();
@@ -26,12 +26,23 @@ use crate::plugin::PluginEvent;
 use crate::store::{Key, Store, ensure_store_entry_raw_readable};
 use crate::target::QueuedPayload;
 use crate::target::TargetDeliverySnapshot;
+use crate::target::{TargetHealth, TargetHealthReason, TargetHealthState};
 use crate::{StoreError, TargetError};
+use futures_util::stream::{FuturesUnordered, StreamExt};
 use std::sync::Arc;
 use std::{collections::HashMap, fmt::Debug};
 use std::{future::Future, pin::Pin, time::Duration};
 use tokio::sync::{Semaphore, mpsc};
 use tokio::task::JoinHandle;
+use tokio_util::sync::CancellationToken;
+
+fn join_failure_reason(error: &tokio::task::JoinError) -> &'static str {
+    if error.is_cancelled() {
+        "join_cancelled"
+    } else {
+        "join_panicked"
+    }
+}

 /// Maximum number of replay attempts before a stored entry is exhausted. Each attempt runs one full
 /// send (one ack wait at the configured timeout for a JetStream entry), then a backoff sleep before
@@ -68,11 +79,24 @@ pub(crate) fn inter_attempt_backoff_sum(attempts: usize) -> Duration {
 pub type SharedTarget<E> = Arc<dyn Target<E> + Send + Sync>;
 type ReplayHook<E> = Arc<dyn Fn(ReplayEvent<E>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;

-/// Upper bound on how long [`ReplayWorkerManager::stop_all`] waits for a single
-/// replay worker to observe its cancel signal and exit before it is forcibly
-/// aborted. Workers observe cancellation promptly (including during retry
-/// backoff), so this only guards against a wedged task.
-const STOP_JOIN_TIMEOUT: Duration = Duration::from_secs(5);
+const HEALTH_COLLECTION_TIMEOUT: Duration = Duration::from_secs(5);
+const HEALTH_PROBE_CONCURRENCY: usize = 10;
+
+pub(crate) enum PrepareTargetResult<E>
+where
+    E: PluginEvent,
+{
+    Ready(SharedTarget<E>),
+    Degraded {
+        error: TargetError,
+        target: SharedTarget<E>,
+    },
+    Failed {
+        error: TargetError,
+        target: Box<dyn Target<E> + Send + Sync>,
+    },
+    Cancelled(Box<dyn Target<E> + Send + Sync>),
+}

 /// Tracks a running replay worker: its cancel channel and, when the worker was
 /// spawned in-process, the [`JoinHandle`] used to await its exit on shutdown.
@@ -113,7 +137,7 @@ impl ReplayWorkerManager {
    }

    /// Registers a cancel channel together with the worker's join handle so
-    /// `stop_all` can await the worker's exit (bounded by [`STOP_JOIN_TIMEOUT`]).
+    /// `stop_all` can await the worker's exit.
    pub fn insert_with_handle(&mut self, target_id: String, cancel_tx: mpsc::Sender<()>, join: JoinHandle<()>) {
        self.cancellers.insert(
            target_id,
@@ -140,37 +164,36 @@ impl ReplayWorkerManager {
    }

    /// Stops every replay worker: it first signals cancellation to all of them,
-    /// then awaits each worker's exit (bounded by [`STOP_JOIN_TIMEOUT`], after
-    /// which the task is aborted). Signalling before joining lets all workers
-    /// wind down concurrently, and joining guarantees no worker keeps draining
-    /// the shared store after this returns — preventing duplicate delivery and
-    /// orphaned tasks across reloads and shutdown.
+    /// then strictly awaits each worker's exit. A worker already awaiting a
+    /// delivery acknowledgement is allowed to finish; aborting it at an
+    /// arbitrary deadline could leave an acknowledged queue entry undeleted and
+    /// make the replacement worker deliver it again. Signalling before joining
+    /// lets all workers wind down concurrently. Legacy joinless registrations
+    /// can only be signalled.
    pub async fn stop_all(&mut self, log_prefix: &str) {
-        let mut handles: Vec<(String, ReplayWorkerHandle)> = self.cancellers.drain().collect();
+        let handles: Vec<(String, ReplayWorkerHandle)> = self.cancellers.drain().collect();
+        let mut joins = std::collections::VecDeque::new();

        // Phase 1: signal cancellation to all workers.
-        for (target_id, handle) in &handles {
+        for (target_id, handle) in handles {
            tracing::info!(target_id = %target_id, "{log_prefix}");
-            let _ = handle.cancel_tx.send(()).await;
+            let _ = handle.cancel_tx.try_send(());
+            if let Some(join) = handle.join {
+                joins.push_back((target_id, join));
+            } else {
+                tracing::warn!(
+                    target_id = %target_id,
+                    "Replay worker has no join handle; cancellation was signalled but exit cannot be verified"
+                );
+            }
        }

-        // Phase 2: await each worker's exit, forcibly aborting any that overrun.
-        for (target_id, handle) in handles.drain(..) {
-            let Some(mut join) = handle.join else {
-                continue;
-            };
-            match tokio::time::timeout(STOP_JOIN_TIMEOUT, &mut join).await {
-                Ok(Ok(())) => {}
-                Ok(Err(err)) => {
-                    tracing::warn!(target_id = %target_id, error = %err, "Replay worker terminated abnormally");
-                }
-                Err(_) => {
-                    join.abort();
-                    tracing::warn!(
-                        target_id = %target_id,
-                        "Timed out awaiting replay worker exit; task aborted"
-                    );
-                }
+        // Phase 2: strict join. Delivery operations own their own protocol
+        // deadlines; lifecycle must not invent a shorter deadline that turns an
+        // acknowledgement race into duplicate delivery.
+        while let Some((target_id, join)) = joins.pop_front() {
+            if let Err(err) = join.await {
+                tracing::warn!(target_id = %target_id, reason = join_failure_reason(&err), "Replay worker terminated abnormally");
            }
        }
    }
@@ -184,6 +207,55 @@ where
    pub targets: Vec<SharedTarget<E>>,
 }

+/// Targets whose persistent queue stores are open and are ready to start
+/// replay. This distinct stage prevents activation from skipping store open.
+pub struct OpenedActivation<E>
+where
+    E: PluginEvent,
+{
+    pub(crate) targets: Vec<SharedTarget<E>>,
+}
+
+struct TargetActivationFailure {
+    detail: String,
+}
+
+/// Targets that have completed initialization but have not started replay
+/// workers yet. Keeping preparation dormant lets lifecycle orchestration stop
+/// the previous workers before the replacement workers are spawned.
+pub struct PreparedActivation<E>
+where
+    E: PluginEvent,
+{
+    failures: Vec<TargetActivationFailure>,
+    rejected_targets: Vec<SharedTarget<E>>,
+    pub(crate) targets: Vec<SharedTarget<E>>,
+}
+
+impl<E> PreparedActivation<E>
+where
+    E: PluginEvent,
+{
+    pub fn failure_summary(&self) -> Option<String> {
+        if self.failures.is_empty() {
+            return None;
+        }
+
+        Some(
+            self.failures
+                .iter()
+                .map(|failure| failure.detail.clone())
+                .collect::<Vec<_>>()
+                .join("; "),
+        )
+    }
+
+    pub fn extend_creation_failures(&mut self, failures: impl IntoIterator<Item = String>) {
+        self.failures
+            .extend(failures.into_iter().map(|detail| TargetActivationFailure { detail }));
+    }
+}
+
 #[derive(Debug, Clone, Default, PartialEq, Eq)]
 pub struct RuntimeStatusSnapshot {
    pub replay_worker_count: usize,
@@ -201,19 +273,16 @@ pub struct RuntimeTargetSnapshot {
    pub total_messages: u64,
 }

-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum RuntimeTargetHealthState {
-    Disabled,
-    Error,
-    Offline,
-    Online,
-}
+pub type RuntimeTargetHealthState = TargetHealthState;
+pub type RuntimeTargetHealthReason = TargetHealthReason;

 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct RuntimeTargetHealthSnapshot {
+    pub account_id: String,
    pub enabled: bool,
    pub error_message: Option<String>,
    pub state: RuntimeTargetHealthState,
+    pub reason: RuntimeTargetHealthReason,
    pub target_id: String,
    pub target_type: String,
 }
@@ -339,17 +408,19 @@ where
    /// Surfacing them lets a caller fail an explicit shutdown while still tearing down the rest of the
    /// runtime.
    pub async fn clear_and_close(&mut self) -> Vec<(String, TargetError)> {
-        let target_ids: Vec<String> = self.targets.keys().cloned().collect();
+        let targets = std::mem::take(&mut self.targets);
+        let mut closes = FuturesUnordered::new();
+        for (target_id, target) in targets {
+            closes.push(async move { (target_id, target.close().await) });
+        }
+
        let mut errors = Vec::new();
-        for target_id in target_ids {
-            if let Some(target) = self.targets.remove(&target_id)
-                && let Err(err) = target.close().await
-            {
+        while let Some((target_id, result)) = closes.next().await {
+            if let Err(err) = result {
                tracing::error!(target_id = %target_id, error = %err, "Failed to close target during shutdown");
                errors.push((target_id, err));
            }
        }
-        self.targets.clear();
        errors
    }

@@ -389,33 +460,57 @@ where
    }

    pub async fn health_snapshots(&self) -> Vec<RuntimeTargetHealthSnapshot> {
-        let mut snapshots = Vec::with_capacity(self.targets.len());
-        for target in self.targets.values() {
-            let enabled = target.is_enabled();
-            let target_id = target.id();
-            let (state, error_message) = if !enabled {
-                (RuntimeTargetHealthState::Disabled, None)
-            } else {
-                match target.is_active().await {
-                    Ok(true) => (RuntimeTargetHealthState::Online, None),
-                    Ok(false) => (RuntimeTargetHealthState::Offline, None),
-                    Err(err) => (RuntimeTargetHealthState::Error, Some(err.to_string())),
-                }
-            };
-
-            snapshots.push(RuntimeTargetHealthSnapshot {
-                enabled,
-                error_message,
-                state,
-                target_id: target_id.to_string(),
-                target_type: target_id.name,
-            });
-        }
-        snapshots.sort_by(|a, b| a.target_id.cmp(&b.target_id));
-        snapshots
+        health_snapshots_for_targets(self.values()).await
    }
 }

+pub async fn health_snapshots_for_targets<E>(targets: Vec<SharedTarget<E>>) -> Vec<RuntimeTargetHealthSnapshot>
+where
+    E: PluginEvent,
+{
+    let deadline = tokio::time::Instant::now() + HEALTH_COLLECTION_TIMEOUT;
+    let permits = Arc::new(Semaphore::new(HEALTH_PROBE_CONCURRENCY));
+    let mut probes = FuturesUnordered::new();
+    for target in targets {
+        let permits = Arc::clone(&permits);
+        let enabled = target.is_enabled();
+        let target_id = target.id();
+        probes.push(async move {
+            let health = if !enabled {
+                TargetHealth::disabled()
+            } else if let Ok(Ok(_permit)) = tokio::time::timeout_at(deadline, permits.acquire_owned()).await {
+                if tokio::time::Instant::now() >= deadline {
+                    TargetHealth::error(TargetHealthReason::HealthCheckFailed)
+                } else {
+                    match tokio::time::timeout_at(deadline, target.health()).await {
+                        Ok(health) => health,
+                        Err(_) => TargetHealth::error(TargetHealthReason::TimedOut),
+                    }
+                }
+            } else {
+                TargetHealth::error(TargetHealthReason::HealthCheckFailed)
+            };
+            (enabled, target_id, health)
+        });
+    }
+
+    let mut snapshots = Vec::with_capacity(probes.len());
+    while let Some((enabled, target_id, health)) = probes.next().await {
+        snapshots.push(RuntimeTargetHealthSnapshot {
+            account_id: target_id.id.clone(),
+            enabled,
+            error_message: (health.state == TargetHealthState::Error).then(|| health.reason.as_str().to_string()),
+            state: health.state,
+            reason: health.reason,
+            target_id: target_id.to_string(),
+            target_type: target_id.name,
+        });
+    }
+
+    snapshots.sort_by(|a, b| a.target_id.cmp(&b.target_id));
+    snapshots
+}
+
 fn snapshot_from_delivery(target_id: TargetID, delivery: TargetDeliverySnapshot) -> RuntimeTargetSnapshot {
    RuntimeTargetSnapshot {
        failed_messages: delivery.failed_messages,
@@ -440,21 +535,16 @@ where
        SharedTarget<E>,
    ) -> (mpsc::Sender<()>, JoinHandle<()>),
 {
-    let target_id = target.id().to_string();
-    let has_store = target.store().is_some();
-
-    if let Err(err) = target.init().await {
-        tracing::error!(target_id = %target_id, error = %err, "Failed to initialize target");
-        if !has_store {
+    let shared = match prepare_target(target, None).await {
+        PrepareTargetResult::Ready(target) => target,
+        PrepareTargetResult::Degraded { target, .. } => target,
+        PrepareTargetResult::Failed { target, .. } => {
+            let _ = target.close().await;
            return None;
        }
-        tracing::warn!(
-            target_id = %target_id,
-            "Proceeding with store-backed target despite init failure"
-        );
-    }
-
-    let shared: SharedTarget<E> = Arc::from(target);
+        PrepareTargetResult::Cancelled(_) => unreachable!("preparation without a cancellation token cannot be cancelled"),
+    };
+    let target_id = shared.id().to_string();
    if !shared.is_enabled() {
        on_replay_start(&target_id, false);
        return Some((shared, None));
@@ -467,6 +557,45 @@ where
    Some((shared, cancel))
 }

+pub(crate) async fn prepare_target<E>(
+    target: Box<dyn Target<E> + Send + Sync>,
+    cancellation: Option<&CancellationToken>,
+) -> PrepareTargetResult<E>
+where
+    E: PluginEvent,
+{
+    let target_id = target.id().to_string();
+    let has_store = target.store().is_some();
+
+    let init_result = match cancellation {
+        Some(cancellation) => {
+            tokio::select! {
+                biased;
+                _ = cancellation.cancelled() => return PrepareTargetResult::Cancelled(target),
+                result = target.init() => result,
+            }
+        }
+        None => target.init().await,
+    };
+
+    if let Err(err) = init_result {
+        tracing::error!(target_id = %target_id, reason = "initialization_failed", "Failed to initialize target");
+        if !has_store {
+            return PrepareTargetResult::Failed { error: err, target };
+        }
+        tracing::warn!(
+            target_id = %target_id,
+            "Proceeding with store-backed target despite init failure"
+        );
+        return PrepareTargetResult::Degraded {
+            error: err,
+            target: Arc::from(target),
+        };
+    }
+
+    PrepareTargetResult::Ready(Arc::from(target))
+}
+
 type ActivatedTarget<E> = (SharedTarget<E>, Option<(mpsc::Sender<()>, JoinHandle<()>)>);

 pub async fn activate_targets_with_replay<E, F, Fut>(
@@ -584,7 +713,11 @@ async fn stream_replay_worker<E>(
                }
                Ok(Ok(_)) => {}
                Err(join_err) => {
-                    tracing::warn!(target_id = %target.id(), error = %join_err, "The failed-events maintenance task failed to join");
+                    tracing::warn!(
+                        target_id = %target.id(),
+                        reason = join_failure_reason(&join_err),
+                        "The failed-events maintenance task failed to join"
+                    );
                }
            }
            last_prune = tokio::time::Instant::now();
@@ -826,16 +959,18 @@ where

 #[cfg(test)]
 mod tests {
-    use super::TargetRuntimeManager;
+    use super::{HEALTH_PROBE_CONCURRENCY, TargetRuntimeManager, health_snapshots_for_targets};
    use crate::PluginEvent;
    use crate::StoreError;
    use crate::arn::TargetID;
    use crate::store::{Key, Store};
    use crate::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
-    use crate::{Target, TargetError};
+    use crate::{SharedTarget, Target, TargetError};
    use async_trait::async_trait;
    use std::sync::Arc;
-    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+    use std::time::Duration;
+    use tokio::sync::{Notify, Semaphore};

    #[tokio::test(start_paused = true)]
    async fn seed_interval_start_backdates_by_one_interval() {
@@ -900,14 +1035,50 @@ mod tests {
    #[derive(Clone)]
    struct TestTarget {
        id: TargetID,
+        block_on_close: Arc<AtomicBool>,
+        close_gate: Arc<Semaphore>,
        close_calls: Arc<AtomicUsize>,
+        enabled: bool,
+        health_delay: Duration,
+        health_drops: Arc<AtomicUsize>,
+        health_started: Arc<Notify>,
+        close_started: Arc<Notify>,
+    }
+
+    struct HealthDropGuard(Arc<AtomicUsize>);
+
+    impl Drop for HealthDropGuard {
+        fn drop(&mut self) {
+            self.0.fetch_add(1, Ordering::SeqCst);
+        }
    }

    impl TestTarget {
        fn new(id: &str, name: &str) -> Self {
            Self {
                id: TargetID::new(id.to_string(), name.to_string()),
+                block_on_close: Arc::new(AtomicBool::new(false)),
+                close_gate: Arc::new(Semaphore::new(0)),
                close_calls: Arc::new(AtomicUsize::new(0)),
+                enabled: true,
+                health_delay: Duration::ZERO,
+                health_drops: Arc::new(AtomicUsize::new(0)),
+                health_started: Arc::new(Notify::new()),
+                close_started: Arc::new(Notify::new()),
+            }
+        }
+
+        fn with_health_delay(id: &str, delay: Duration) -> Self {
+            Self {
+                health_delay: delay,
+                ..Self::new(id, "webhook")
+            }
+        }
+
+        fn disabled(id: &str) -> Self {
+            Self {
+                enabled: false,
+                ..Self::new(id, "webhook")
            }
        }
    }
@@ -922,6 +1093,9 @@ mod tests {
        }

        async fn is_active(&self) -> Result<bool, TargetError> {
+            self.health_started.notify_one();
+            let _drop_guard = HealthDropGuard(Arc::clone(&self.health_drops));
+            tokio::time::sleep(self.health_delay).await;
            Ok(true)
        }

@@ -935,6 +1109,10 @@ mod tests {

        async fn close(&self) -> Result<(), TargetError> {
            self.close_calls.fetch_add(1, Ordering::SeqCst);
+            self.close_started.notify_one();
+            if self.block_on_close.load(Ordering::SeqCst) {
+                let _permit = self.close_gate.acquire().await.expect("close gate should remain open");
+            }
            Ok(())
        }

@@ -947,7 +1125,7 @@ mod tests {
        }

        fn is_enabled(&self) -> bool {
-            true
+            self.enabled
        }
    }

@@ -966,6 +1144,45 @@ mod tests {
        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
    }

+    #[tokio::test(start_paused = true)]
+    async fn runtime_manager_starts_all_target_closes_before_waiting_for_completion() {
+        let mut manager = TargetRuntimeManager::<String>::new();
+        let first = TestTarget::new("first", "webhook");
+        let second = TestTarget::new("second", "webhook");
+        let first_observer = first.clone();
+        let second_observer = second.clone();
+        manager.add_boxed(Box::new(first));
+        manager.add_boxed(Box::new(second));
+
+        let first_close_key = manager
+            .keys()
+            .into_iter()
+            .next()
+            .expect("two targets should have a first close key");
+        let (blocked, unblocked) = if first_close_key == first_observer.id.to_string() {
+            (first_observer, second_observer)
+        } else {
+            (second_observer, first_observer)
+        };
+        blocked.block_on_close.store(true, Ordering::SeqCst);
+
+        let close_task = tokio::spawn(async move { manager.clear_and_close().await });
+
+        tokio::time::timeout(std::time::Duration::from_secs(1), blocked.close_started.notified())
+            .await
+            .expect("the first target close should start");
+        tokio::time::timeout(std::time::Duration::from_secs(1), unblocked.close_started.notified())
+            .await
+            .expect("a blocked first close must not prevent the second close from starting");
+        assert!(!close_task.is_finished(), "clear_and_close must still await the blocked target");
+
+        blocked.close_gate.add_permits(1);
+        let errors = close_task.await.expect("clear_and_close task should join");
+        assert!(errors.is_empty());
+        assert_eq!(blocked.close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(unblocked.close_calls.load(Ordering::SeqCst), 1);
+    }
+
    #[test]
    fn runtime_manager_snapshots_targets() {
        let mut manager = TargetRuntimeManager::<String>::new();
@@ -977,6 +1194,104 @@ mod tests {
        assert_eq!(snapshots[0].target_type, "webhook");
    }

+    #[tokio::test(start_paused = true)]
+    async fn health_snapshot_allows_a_four_second_probe() {
+        let mut manager = TargetRuntimeManager::<String>::new();
+        manager.add_boxed(Box::new(TestTarget::with_health_delay("slow", Duration::from_secs(4))));
+
+        let snapshots = manager.health_snapshots().await;
+
+        assert_eq!(snapshots.len(), 1);
+        assert_eq!(snapshots[0].state, crate::TargetHealthState::Online);
+        assert_eq!(snapshots[0].reason, crate::TargetHealthReason::Reachable);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn health_snapshot_times_out_after_five_seconds() {
+        let mut manager = TargetRuntimeManager::<String>::new();
+        manager.add_boxed(Box::new(TestTarget::with_health_delay("stalled", Duration::from_secs(6))));
+
+        let snapshots = manager.health_snapshots().await;
+
+        assert_eq!(snapshots.len(), 1);
+        assert_eq!(snapshots[0].state, crate::TargetHealthState::Error);
+        assert_eq!(snapshots[0].reason, crate::TargetHealthReason::TimedOut);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn health_collection_deadline_does_not_scale_with_target_count() {
+        let mut manager = TargetRuntimeManager::<String>::new();
+        for index in 0..24 {
+            manager.add_boxed(Box::new(TestTarget::with_health_delay(
+                &format!("stalled-{index}"),
+                Duration::from_secs(30),
+            )));
+        }
+        let started = tokio::time::Instant::now();
+
+        let snapshots = manager.health_snapshots().await;
+
+        assert_eq!(started.elapsed(), Duration::from_secs(5));
+        assert_eq!(snapshots.len(), 24);
+        assert!(
+            snapshots
+                .iter()
+                .all(|snapshot| snapshot.state == crate::TargetHealthState::Error)
+        );
+        assert_eq!(
+            snapshots
+                .iter()
+                .filter(|snapshot| snapshot.reason == crate::TargetHealthReason::TimedOut)
+                .count(),
+            HEALTH_PROBE_CONCURRENCY
+        );
+        assert_eq!(
+            snapshots
+                .iter()
+                .filter(|snapshot| snapshot.reason == crate::TargetHealthReason::HealthCheckFailed)
+                .count(),
+            24 - HEALTH_PROBE_CONCURRENCY
+        );
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn disabled_target_does_not_wait_for_probe_capacity() {
+        let mut targets: Vec<SharedTarget<String>> = (0..HEALTH_PROBE_CONCURRENCY)
+            .map(|index| {
+                Arc::new(TestTarget::with_health_delay(&format!("stalled-{index}"), Duration::from_secs(30)))
+                    as SharedTarget<String>
+            })
+            .collect();
+        targets.push(Arc::new(TestTarget::disabled("disabled")));
+
+        let snapshots = health_snapshots_for_targets(targets).await;
+        let disabled = snapshots
+            .iter()
+            .find(|snapshot| snapshot.target_id == "disabled:webhook")
+            .expect("disabled target snapshot");
+
+        assert_eq!(disabled.state, crate::TargetHealthState::Disabled);
+        assert_eq!(disabled.reason, crate::TargetHealthReason::Disabled);
+    }
+
+    #[tokio::test]
+    async fn cancelling_health_collection_drops_in_flight_probe() {
+        let target = TestTarget::with_health_delay("slow", Duration::from_secs(30));
+        let health_drops = Arc::clone(&target.health_drops);
+        let health_started = Arc::clone(&target.health_started);
+        let targets: Vec<SharedTarget<String>> = vec![Arc::new(target)];
+        let collector = tokio::spawn(health_snapshots_for_targets(targets));
+
+        tokio::time::timeout(Duration::from_secs(1), health_started.notified())
+            .await
+            .expect("health probe should start");
+        collector.abort();
+        let join_error = collector.await.expect_err("health collector should be cancelled");
+
+        assert!(join_error.is_cancelled());
+        assert_eq!(health_drops.load(Ordering::SeqCst), 1);
+    }
+
    #[tokio::test]
    async fn sleep_or_cancelled_returns_immediately_on_cancel() {
        let (cancel_tx, mut cancel_rx) = tokio::sync::mpsc::channel::<()>(1);
@@ -1030,6 +1345,51 @@ mod tests {
        assert!(exited.load(Ordering::SeqCst), "stop_all must await the worker to completion");
    }

+    #[tokio::test(start_paused = true)]
+    async fn stop_all_does_not_abort_delivery_awaiting_acknowledgement() {
+        use super::ReplayWorkerManager;
+        use std::sync::atomic::{AtomicBool, Ordering};
+        use tokio::sync::Notify;
+
+        let mut manager = ReplayWorkerManager::new();
+        let acknowledgement = Arc::new(Notify::new());
+        let worker_started = Arc::new(Notify::new());
+        let exited = Arc::new(AtomicBool::new(false));
+        let (cancel_tx, mut cancel_rx) = tokio::sync::mpsc::channel::<()>(1);
+        let worker_acknowledgement = acknowledgement.clone();
+        let worker_started_signal = worker_started.clone();
+        let worker_exited = exited.clone();
+        let join = tokio::spawn(async move {
+            worker_started_signal.notify_one();
+            let _ = cancel_rx.recv().await;
+            // Model a protocol operation that has accepted the request but has
+            // not returned its acknowledgement yet. Lifecycle must not abort
+            // this future or the same durable entry can be sent twice.
+            worker_acknowledgement.notified().await;
+            worker_exited.store(true, Ordering::SeqCst);
+        });
+        manager.insert_with_handle("primary:webhook".to_string(), cancel_tx, join);
+        worker_started.notified().await;
+
+        let mut stop = Box::pin(manager.stop_all("stopping ack-pending test worker"));
+        tokio::select! {
+            biased;
+            _ = &mut stop => panic!("stop_all returned before the pending acknowledgement"),
+            _ = std::future::ready(()) => {}
+        }
+        tokio::time::advance(std::time::Duration::from_secs(60)).await;
+        tokio::select! {
+            biased;
+            _ = &mut stop => panic!("stop_all aborted an acknowledgement-pending delivery"),
+            _ = std::future::ready(()) => {}
+        }
+
+        acknowledgement.notify_one();
+        stop.await;
+        assert!(exited.load(Ordering::SeqCst));
+        assert!(manager.is_empty());
+    }
+
    mod classifier {
        use super::super::{ReplayEvent, stream_replay_worker};
        use crate::arn::TargetID;
@@ -24,7 +24,7 @@ use crate::{
    store::{Key, Store},
    target::{
        ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, invalidate_cache_on_connectivity_error,
+        TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, is_connectivity_error,
        open_target_queue_store, persist_queued_payload_to_store,
    },
 };
@@ -32,13 +32,84 @@ use async_trait::async_trait;
 use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError, KafkaCode};
 use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, Record, RequiredAcks, SaslConfig, SecurityConfig};
 use rustfs_tls_runtime::{load_cert_bundle_der_bytes, load_private_key};
-use std::{fmt, marker::PhantomData, sync::Arc, time::Duration};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::{fmt, future::Future, marker::PhantomData, sync::Arc, time::Duration};
 use tokio::sync::Mutex;
 use tracing::{debug, error, info, instrument, warn};

 pub(crate) const KAFKA_SASL_PLAIN: &str = "PLAIN";
 pub(crate) const KAFKA_SASL_SCRAM_SHA_256: &str = "SCRAM-SHA-256";
 pub(crate) const KAFKA_SASL_SCRAM_SHA_512: &str = "SCRAM-SHA-512";
+const KAFKA_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
+
+struct KafkaDeliveryAttempt<'a> {
+    armed: bool,
+    poisoned: &'a AtomicBool,
+}
+
+impl KafkaDeliveryAttempt<'_> {
+    fn disarm(&mut self) {
+        self.armed = false;
+    }
+}
+
+impl Drop for KafkaDeliveryAttempt<'_> {
+    fn drop(&mut self) {
+        if self.armed {
+            self.poisoned.store(true, Ordering::Release);
+        }
+    }
+}
+
+fn kafka_delivery_timeout() -> TargetError {
+    TargetError::Timeout(format!("Kafka delivery timed out after {KAFKA_DELIVERY_TIMEOUT:?}"))
+}
+
+async fn with_serialized_kafka_delivery<P, T, Select, SelectFuture, Deliver, DeliveryFuture, Invalidate, InvalidateFuture>(
+    delivery_lock: &Mutex<()>,
+    delivery_poisoned: &AtomicBool,
+    select_producer: Select,
+    deliver: Deliver,
+    invalidate: Invalidate,
+) -> Result<T, TargetError>
+where
+    P: Send,
+    T: Send,
+    Select: FnOnce() -> SelectFuture + Send,
+    SelectFuture: Future<Output = Result<P, TargetError>> + Send,
+    Deliver: FnOnce(P) -> DeliveryFuture + Send,
+    DeliveryFuture: Future<Output = Result<T, TargetError>> + Send,
+    Invalidate: Fn() -> InvalidateFuture + Send,
+    InvalidateFuture: Future<Output = ()> + Send,
+{
+    let deadline = tokio::time::Instant::now() + KAFKA_DELIVERY_TIMEOUT;
+    let _delivery_guard = tokio::time::timeout_at(deadline, delivery_lock.lock())
+        .await
+        .map_err(|_| kafka_delivery_timeout())?;
+    let mut attempt = KafkaDeliveryAttempt {
+        armed: true,
+        poisoned: delivery_poisoned,
+    };
+
+    if delivery_poisoned.load(Ordering::Acquire) {
+        tokio::time::timeout_at(deadline, invalidate())
+            .await
+            .map_err(|_| kafka_delivery_timeout())?;
+        delivery_poisoned.store(false, Ordering::Release);
+    }
+
+    let result = tokio::time::timeout_at(deadline, async { deliver(select_producer().await?).await })
+        .await
+        .map_err(|_| kafka_delivery_timeout())?;
+    if result.as_ref().is_err_and(is_connectivity_error) {
+        tokio::time::timeout_at(deadline, invalidate())
+            .await
+            .map_err(|_| kafka_delivery_timeout())?;
+        delivery_poisoned.store(false, Ordering::Release);
+    }
+    attempt.disarm();
+    result
+}

 /// Arguments for configuring a Kafka target
 #[derive(Clone)]
@@ -233,6 +304,8 @@ where
    args: KafkaArgs,
    store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
    producer: Arc<Mutex<Option<Arc<AsyncProducer>>>>,
+    delivery_lock: Arc<Mutex<()>>,
+    delivery_poisoned: Arc<AtomicBool>,
    tls_state: Arc<Mutex<TargetTlsState>>,
    /// Adapter that bridges this target to the TLS reload coordinator.
    /// When `Some`, the target uses coordinator-managed material; when `None`,
@@ -291,6 +364,8 @@ where
            args,
            store: queue_store,
            producer: Arc::new(Mutex::new(None)),
+            delivery_lock: Arc::new(Mutex::new(())),
+            delivery_poisoned: Arc::new(AtomicBool::new(false)),
            tls_state: Arc::new(Mutex::new(TargetTlsState::default())),
            tls_adapter: None,
            delivery_counters: Arc::new(TargetDeliveryCounters::default()),
@@ -307,7 +382,7 @@ where
        };

        let mut config = AsyncProducerConfig::new()
-            .with_ack_timeout(Duration::from_secs(30))
+            .with_ack_timeout(KAFKA_DELIVERY_TIMEOUT)
            .with_required_acks(acks);

        if let Some(security) = self.args.security_config(true)? {
@@ -388,20 +463,26 @@ where
            "Sending Kafka payload"
        );

-        let producer = self.get_or_build_producer().await?;
-
-        // Use "<bucket>/<object>" as the message key so all events for the same
-        // object hash to the same partition and preserve per-object ordering
-        // across multiple partitions (backlog#983).
-        let partition_key = format!("{}/{}", meta.bucket_name, meta.object_name);
-        if let Err(err) = producer
-            .send(&Record::from_key_value(&self.args.topic, partition_key, body.as_slice()))
-            .await
-        {
-            let mapped = Self::map_kafka_error(err, "Failed to send message to Kafka");
-            invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_producer()).await;
-            return Err(mapped);
-        }
+        // rustfs-kafka-async does not validate response correlation IDs. Keep
+        // producer selection, send, and timeout invalidation serialized so a
+        // waiter cannot reuse a connection with an unread timed-out response.
+        with_serialized_kafka_delivery(
+            &self.delivery_lock,
+            &self.delivery_poisoned,
+            || self.get_or_build_producer(),
+            |producer| async move {
+                // Use "<bucket>/<object>" as the message key so all events for the same
+                // object hash to the same partition and preserve per-object ordering
+                // across multiple partitions (backlog#983).
+                let partition_key = format!("{}/{}", meta.bucket_name, meta.object_name);
+                producer
+                    .send(&Record::from_key_value(&self.args.topic, partition_key, body.as_slice()))
+                    .await
+                    .map_err(|err| Self::map_kafka_error(err, "Failed to send message to Kafka"))
+            },
+            || self.invalidate_cached_producer(),
+        )
+        .await?;

        debug!(target_id = %self.id, topic = %self.args.topic, "Event published to Kafka topic");
        self.delivery_counters.record_success();
@@ -415,6 +496,8 @@ where
            args: self.args.clone(),
            store: self.store.as_ref().map(|s| s.boxed_clone()),
            producer: Arc::clone(&self.producer),
+            delivery_lock: Arc::clone(&self.delivery_lock),
+            delivery_poisoned: Arc::clone(&self.delivery_poisoned),
            tls_state: Arc::clone(&self.tls_state),
            tls_adapter: self.tls_adapter.clone(),
            delivery_counters: Arc::clone(&self.delivery_counters),
@@ -559,6 +642,8 @@ where
 #[cfg(test)]
 mod tests {
    use super::*;
+    use std::sync::atomic::AtomicUsize;
+    use tokio::sync::Notify;

    fn base_args() -> KafkaArgs {
        KafkaArgs {
@@ -580,6 +665,158 @@ mod tests {
        }
    }

+    #[tokio::test(start_paused = true)]
+    async fn timeout_invalidates_before_the_next_delivery_selects_a_producer() {
+        let delivery_lock = Arc::new(Mutex::new(()));
+        let delivery_poisoned = Arc::new(AtomicBool::new(false));
+        let generation = Arc::new(AtomicUsize::new(1));
+        let first_entered = Arc::new(Notify::new());
+
+        let first = {
+            let delivery_lock = Arc::clone(&delivery_lock);
+            let delivery_poisoned = Arc::clone(&delivery_poisoned);
+            let generation = Arc::clone(&generation);
+            let first_entered = Arc::clone(&first_entered);
+            tokio::spawn(async move {
+                with_serialized_kafka_delivery(
+                    &delivery_lock,
+                    &delivery_poisoned,
+                    {
+                        let generation = Arc::clone(&generation);
+                        move || async move { Ok(generation.load(Ordering::SeqCst)) }
+                    },
+                    move |selected| async move {
+                        assert_eq!(selected, 1);
+                        first_entered.notify_one();
+                        std::future::pending::<Result<usize, TargetError>>().await
+                    },
+                    move || {
+                        let generation = Arc::clone(&generation);
+                        async move { generation.store(2, Ordering::SeqCst) }
+                    },
+                )
+                .await
+            })
+        };
+
+        first_entered.notified().await;
+        tokio::time::advance(Duration::from_secs(1)).await;
+        let second = {
+            let delivery_lock = Arc::clone(&delivery_lock);
+            let delivery_poisoned = Arc::clone(&delivery_poisoned);
+            let generation = Arc::clone(&generation);
+            tokio::spawn(async move {
+                with_serialized_kafka_delivery(
+                    &delivery_lock,
+                    &delivery_poisoned,
+                    {
+                        let generation = Arc::clone(&generation);
+                        move || async move { Ok(generation.load(Ordering::SeqCst)) }
+                    },
+                    |selected| async move { Ok(selected) },
+                    move || {
+                        let generation = Arc::clone(&generation);
+                        async move { generation.store(2, Ordering::SeqCst) }
+                    },
+                )
+                .await
+            })
+        };
+
+        assert!(matches!(
+            first.await.expect("first delivery task should not panic"),
+            Err(TargetError::Timeout(_))
+        ));
+        assert_eq!(
+            second
+                .await
+                .expect("second delivery task should not panic")
+                .expect("second delivery should succeed"),
+            2,
+            "the waiter must select a fresh producer generation after timeout invalidation"
+        );
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn delivery_deadline_includes_waiting_for_the_serialization_lock() {
+        let delivery_lock = Arc::new(Mutex::new(()));
+        let delivery_poisoned = AtomicBool::new(false);
+        let selected = Arc::new(AtomicBool::new(false));
+        let _held = delivery_lock.lock().await;
+
+        let error = with_serialized_kafka_delivery(
+            &delivery_lock,
+            &delivery_poisoned,
+            {
+                let selected = Arc::clone(&selected);
+                move || async move {
+                    selected.store(true, Ordering::SeqCst);
+                    Ok(())
+                }
+            },
+            |()| async { Ok(()) },
+            || async {},
+        )
+        .await
+        .expect_err("lock admission must share the absolute delivery deadline");
+
+        assert!(matches!(error, TargetError::Timeout(_)));
+        assert!(!selected.load(Ordering::SeqCst), "a timed-out waiter must not select a producer");
+        assert!(!delivery_poisoned.load(Ordering::SeqCst));
+    }
+
+    #[tokio::test]
+    async fn cancelled_delivery_poisons_the_connection_before_the_next_selection() {
+        let delivery_lock = Arc::new(Mutex::new(()));
+        let delivery_poisoned = Arc::new(AtomicBool::new(false));
+        let generation = Arc::new(AtomicUsize::new(1));
+        let first_entered = Arc::new(Notify::new());
+        let first = {
+            let delivery_lock = Arc::clone(&delivery_lock);
+            let delivery_poisoned = Arc::clone(&delivery_poisoned);
+            let first_entered = Arc::clone(&first_entered);
+            tokio::spawn(async move {
+                with_serialized_kafka_delivery(
+                    &delivery_lock,
+                    &delivery_poisoned,
+                    || async { Ok(1usize) },
+                    move |_| async move {
+                        first_entered.notify_one();
+                        std::future::pending::<Result<(), TargetError>>().await
+                    },
+                    || async {},
+                )
+                .await
+            })
+        };
+        first_entered.notified().await;
+        first.abort();
+        assert!(first.await.expect_err("first delivery should be cancelled").is_cancelled());
+        assert!(delivery_poisoned.load(Ordering::Acquire));
+
+        let selected = with_serialized_kafka_delivery(
+            &delivery_lock,
+            &delivery_poisoned,
+            {
+                let generation = Arc::clone(&generation);
+                move || async move { Ok(generation.load(Ordering::SeqCst)) }
+            },
+            |selected| async move { Ok(selected) },
+            {
+                let generation = Arc::clone(&generation);
+                move || {
+                    let generation = Arc::clone(&generation);
+                    async move { generation.store(2, Ordering::SeqCst) }
+                }
+            },
+        )
+        .await
+        .expect("the next delivery should recover from cancellation poisoning");
+
+        assert_eq!(selected, 2);
+        assert!(!delivery_poisoned.load(Ordering::Acquire));
+    }
+
    #[test]
    fn test_validate_empty_brokers() {
        let args = KafkaArgs {
@@ -19,12 +19,14 @@ use crate::{StoreError, TargetError, TargetLog};
 use async_trait::async_trait;
 use rustfs_s3_types::EventName;
 use serde::{Deserialize, Serialize};
+use std::cell::Cell;
 use std::fmt::Formatter;
 use std::future::Future;
 use std::path::PathBuf;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
-use std::time::{SystemTime, UNIX_EPOCH};
+use std::thread_local;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
 use tracing::{debug, warn};

 pub mod amqp;
@@ -70,6 +72,125 @@ pub struct TargetDeliveryCounters {
    total_messages: AtomicU64,
 }

+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TargetHealthState {
+    Disabled,
+    Error,
+    Offline,
+    Online,
+}
+
+impl TargetHealthState {
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::Disabled => "disabled",
+            Self::Error => "error",
+            Self::Offline => "offline",
+            Self::Online => "online",
+        }
+    }
+
+    pub const fn status(self) -> &'static str {
+        match self {
+            Self::Online => "online",
+            Self::Disabled | Self::Error | Self::Offline => "offline",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TargetHealthReason {
+    AuthenticationFailed,
+    ConfigurationInvalid,
+    ConnectionRefused,
+    Disabled,
+    DnsFailure,
+    HealthCheckFailed,
+    InitializationFailed,
+    NotLoadedInRuntime,
+    Reachable,
+    RequestFailed,
+    TimedOut,
+    TlsFailure,
+    Unreachable,
+}
+
+impl TargetHealthReason {
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::AuthenticationFailed => "authentication_failed",
+            Self::ConfigurationInvalid => "configuration_invalid",
+            Self::ConnectionRefused => "connection_refused",
+            Self::Disabled => "disabled",
+            Self::DnsFailure => "dns_failure",
+            Self::HealthCheckFailed => "health_check_failed",
+            Self::InitializationFailed => "initialization_failed",
+            Self::NotLoadedInRuntime => "not_loaded_in_runtime",
+            Self::Reachable => "reachable",
+            Self::RequestFailed => "request_failed",
+            Self::TimedOut => "timed_out",
+            Self::TlsFailure => "tls_failure",
+            Self::Unreachable => "unreachable",
+        }
+    }
+
+    fn from_target_error(err: &TargetError) -> Self {
+        match err {
+            TargetError::Authentication(_) => Self::AuthenticationFailed,
+            TargetError::Configuration(_) | TargetError::ParseError(_) => Self::ConfigurationInvalid,
+            TargetError::Initialization(_) | TargetError::ServerNotInitialized(_) => Self::InitializationFailed,
+            TargetError::Network(_) | TargetError::NotConnected => Self::Unreachable,
+            TargetError::Request(_) => Self::RequestFailed,
+            TargetError::Timeout(_) => Self::TimedOut,
+            TargetError::Storage(_)
+            | TargetError::JetStreamPublish { .. }
+            | TargetError::Encoding(_)
+            | TargetError::Serialization(_)
+            | TargetError::InvalidARN(_)
+            | TargetError::Unknown(_)
+            | TargetError::Disabled
+            | TargetError::Dropped(_)
+            | TargetError::SaveConfig(_) => Self::HealthCheckFailed,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct TargetHealth {
+    pub state: TargetHealthState,
+    pub reason: TargetHealthReason,
+}
+
+impl TargetHealth {
+    pub const fn disabled() -> Self {
+        Self {
+            state: TargetHealthState::Disabled,
+            reason: TargetHealthReason::Disabled,
+        }
+    }
+
+    pub const fn error(reason: TargetHealthReason) -> Self {
+        Self {
+            state: TargetHealthState::Error,
+            reason,
+        }
+    }
+
+    pub const fn offline(reason: TargetHealthReason) -> Self {
+        Self {
+            state: TargetHealthState::Offline,
+            reason,
+        }
+    }
+
+    pub const fn online(reason: TargetHealthReason) -> Self {
+        Self {
+            state: TargetHealthState::Online,
+            reason,
+        }
+    }
+}
+
 pub(crate) type BoxedQueuedStore = Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>;

 impl TargetDeliveryCounters {
@@ -111,7 +232,25 @@ where
    /// Checks if the target is active and reachable
    async fn is_active(&self) -> Result<bool, TargetError>;

-    /// Saves an event (either sends it immediately or stores it for later)
+    /// Returns a credential-free, machine-readable health result.
+    async fn health(&self) -> TargetHealth {
+        if !self.is_enabled() {
+            return TargetHealth::disabled();
+        }
+
+        match self.is_active().await {
+            Ok(true) => TargetHealth::online(TargetHealthReason::Reachable),
+            Ok(false) => TargetHealth::offline(TargetHealthReason::Unreachable),
+            Err(err) => TargetHealth::error(TargetHealthReason::from_target_error(&err)),
+        }
+    }
+
+    /// Saves an event (either sends it immediately or stores it for later).
+    ///
+    /// A target whose [`Self::store`] returns `Some` must only persist the event
+    /// here; network delivery belongs to its replay worker. Runtime lifecycle
+    /// handoff drains these durable enqueues while allowing a direct network
+    /// send to finish against a detached target.
    async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError>;

    /// Sends an event from the store using the queued raw body and metadata.
@@ -605,6 +744,24 @@ pub(crate) fn open_target_queue_store(
    Ok(store.map(|store| Box::new(store) as BoxedQueuedStore))
 }

+thread_local! {
+    static DEFER_QUEUE_STORE_OPEN: Cell<bool> = const { Cell::new(false) };
+}
+
+pub(crate) fn with_deferred_queue_store_open<T>(operation: impl FnOnce() -> T) -> T {
+    struct Reset(bool);
+
+    impl Drop for Reset {
+        fn drop(&mut self) {
+            DEFER_QUEUE_STORE_OPEN.with(|deferred| deferred.set(self.0));
+        }
+    }
+
+    let previous = DEFER_QUEUE_STORE_OPEN.with(|deferred| deferred.replace(true));
+    let _reset = Reset(previous);
+    operation()
+}
+
 /// Opens the queue store and returns the concrete QueueStore, so a target that needs its typed
 /// failed-store capability holds it directly rather than through the type-erased Store handle.
 pub(crate) fn open_target_queue_store_typed(
@@ -625,9 +782,11 @@ pub(crate) fn open_target_queue_store_typed(
        TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
    };
    let store = QueueStore::<QueuedPayload>::new(queue_dir, queue_limit, extension);
-    store
-        .open()
-        .map_err(|err| TargetError::Storage(format!("{open_context}: {err}")))?;
+    if !DEFER_QUEUE_STORE_OPEN.with(Cell::get) {
+        store
+            .open()
+            .map_err(|err| TargetError::Storage(format!("{open_context}: {err}")))?;
+    }

    Ok(Some(store))
 }
@@ -649,6 +808,26 @@ pub(crate) fn is_connectivity_error(err: &TargetError) -> bool {
    matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_))
 }

+/// Applies an absolute deadline to one protocol delivery attempt.
+///
+/// Target clients expose different timeout controls, and several of them only
+/// place a timeout value in the wire request without bounding the local socket
+/// future. Keeping the outer deadline here gives every caller the same typed,
+/// retryable timeout without changing the target-specific error mapping.
+pub(crate) async fn with_delivery_deadline<T, F>(
+    deadline: Duration,
+    operation: &'static str,
+    delivery: F,
+) -> Result<T, TargetError>
+where
+    F: Future<Output = Result<T, TargetError>>,
+{
+    match tokio::time::timeout(deadline, delivery).await {
+        Ok(result) => result,
+        Err(_) => Err(TargetError::Timeout(format!("{operation} timed out after {deadline:?}"))),
+    }
+}
+
 pub(crate) async fn invalidate_cache_on_connectivity_error<F, Fut>(err: &TargetError, invalidate: F)
 where
    F: FnOnce() -> Fut,
@@ -1137,6 +1316,29 @@ mod tests {
        let _ = fs::remove_file(base);
    }

+    #[test]
+    fn deferred_queue_store_creation_does_not_touch_the_filesystem() {
+        let base = std::env::temp_dir().join(format!("rustfs-target-store-deferred-{}", Uuid::new_v4()));
+        fs::write(&base, b"not-a-directory").expect("failed to create file base");
+        let target_id = TargetID::new("target-a".to_string(), ChannelTargetType::Kafka.as_str().to_string());
+
+        let store = with_deferred_queue_store_open(|| {
+            open_target_queue_store(
+                base.to_str().unwrap(),
+                100,
+                TargetType::NotifyEvent,
+                ChannelTargetType::Kafka.as_str(),
+                &target_id,
+                "deferred open",
+            )
+        })
+        .expect("deferred construction must not open the queue directory")
+        .expect("non-empty queue directory should create a dormant store");
+
+        assert!(store.open().is_err(), "the invalid path must fail when handoff explicitly opens it");
+        let _ = fs::remove_file(base);
+    }
+
    #[test]
    fn persist_queued_payload_to_store_writes_encoded_payload() {
        let store = MockQueuedStore::new(false);
@@ -1182,6 +1384,19 @@ mod tests {
        assert!(!is_connectivity_error(&TargetError::Serialization("serialization".to_string())));
    }

+    #[tokio::test(start_paused = true)]
+    async fn delivery_deadline_cuts_off_a_stalled_protocol_operation() {
+        let error = with_delivery_deadline(
+            Duration::from_secs(30),
+            "test delivery",
+            std::future::pending::<Result<(), TargetError>>(),
+        )
+        .await
+        .expect_err("a stalled delivery must hit its hard deadline");
+
+        assert!(matches!(error, TargetError::Timeout(message) if message == "test delivery timed out after 30s"));
+    }
+
    #[tokio::test]
    async fn invalidate_cache_on_connectivity_error_only_runs_for_connectivity_failures() {
        let marker = Arc::new(AtomicBool::new(false));
@@ -765,11 +765,6 @@ where

    #[instrument(skip(self, body, meta), fields(target_id = %self.id))]
    async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
-        let client_guard = self.client.lock().await;
-        let client = client_guard
-            .as_ref()
-            .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
-
        debug!(
            event = EVENT_MQTT_DELIVERY_STATE,
            component = LOG_COMPONENT_TARGETS,
@@ -790,10 +785,22 @@ where
        // silently dropped the event while its durable copy was already deleted
        // (backlog#971). Error classification now matches on the typed error
        // instead of substring matching on the display string.
-        let notice = match client.publish_tracked(&self.args.topic, self.args.qos, false, body).await {
-            Ok(notice) => notice,
-            Err(e) => {
-                let err = classify_mqtt_client_error(&e);
+        let notice = match tokio::time::timeout(MQTT_PUBLISH_CONFIRM_TIMEOUT, async {
+            let client_guard = self.client.lock().await;
+            let client = client_guard
+                .as_ref()
+                .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
+            let notice = client
+                .publish_tracked(&self.args.topic, self.args.qos, false, body)
+                .await
+                .map_err(|error| classify_mqtt_client_error(&error))?;
+            drop(client_guard);
+            Ok(notice)
+        })
+        .await
+        {
+            Ok(Ok(notice)) => notice,
+            Ok(Err(err)) => {
                warn!(
                    event = EVENT_MQTT_DELIVERY_STATE,
                    component = LOG_COMPONENT_TARGETS,
@@ -801,18 +808,29 @@ where
                    target_id = %self.id,
                    state = "publish_failed",
                    reason = "enqueue_error",
-                    error = %e,
+                    error = %err,
                    "mqtt delivery state"
                );
                mark_target_disconnected_on_connectivity_error(&self.connected, &err);
                return Err(err);
            }
+            Err(_) => {
+                warn!(
+                    event = EVENT_MQTT_DELIVERY_STATE,
+                    component = LOG_COMPONENT_TARGETS,
+                    subsystem = LOG_SUBSYSTEM_MQTT,
+                    target_id = %self.id,
+                    state = "publish_failed",
+                    reason = "enqueue_timeout",
+                    "mqtt delivery state"
+                );
+                // Admission can time out because the local bounded request
+                // channel is full while the MQTT session remains connected.
+                // Only protocol/client failures are evidence of disconnect.
+                return Err(TargetError::Timeout("MQTT publish enqueue timed out".to_string()));
+            }
        };

-        // Release the client lock before awaiting the broker acknowledgement so a
-        // slow/hung broker never blocks other senders from queueing publishes.
-        drop(client_guard);
-
        match tokio::time::timeout(MQTT_PUBLISH_CONFIRM_TIMEOUT, notice.wait_completion_async()).await {
            Ok(Ok(())) => {
                debug!(
@@ -1708,9 +1726,9 @@ where
 #[cfg(test)]
 mod tests {
    use super::{
-        ClientError, MQTT_RECONNECT_BACKOFF_MAX, MQTT_RECONNECT_BACKOFF_MIN, MQTTArgs, MQTTTlsConfig, PublishNoticeError, QoS,
-        classify_mqtt_client_error, classify_mqtt_notice_error, next_reconnect_backoff, reconnect_supervisor,
-        validate_mqtt_broker_url,
+        AsyncClient, ClientError, MQTT_RECONNECT_BACKOFF_MAX, MQTT_RECONNECT_BACKOFF_MIN, MQTTArgs, MQTTTarget, MQTTTlsConfig,
+        MqttOptions, PublishNoticeError, QoS, QueuedPayloadMeta, classify_mqtt_client_error, classify_mqtt_notice_error,
+        next_reconnect_backoff, reconnect_supervisor, validate_mqtt_broker_url,
    };
    use crate::error::TargetError;
    use crate::target::{REDACTED_SECRET, TargetType};
@@ -1720,6 +1738,23 @@ mod tests {
    use tokio::sync::mpsc;
    use url::Url;

+    fn base_mqtt_args() -> MQTTArgs {
+        MQTTArgs {
+            enable: true,
+            broker: Url::parse("mqtt://broker.example.com:1883").expect("valid broker"),
+            topic: "rustfs/events".to_string(),
+            qos: QoS::AtLeastOnce,
+            username: String::new(),
+            password: String::new(),
+            tls: MQTTTlsConfig::default(),
+            max_reconnect_interval: Duration::from_secs(1),
+            keep_alive: Duration::from_secs(30),
+            queue_dir: String::new(),
+            queue_limit: 0,
+            target_type: TargetType::NotifyEvent,
+        }
+    }
+
    #[test]
    fn mqtt_client_error_classified_as_not_connected() {
        // A publish that cannot be handed to the event loop means the client is
@@ -1752,6 +1787,38 @@ mod tests {
        assert!(matches!(classify_mqtt_notice_error(&err), TargetError::Request(_)));
    }

+    #[tokio::test(start_paused = true)]
+    async fn enqueue_timeout_keeps_a_live_session_connected() {
+        let target = MQTTTarget::<String>::new("mqtt:test".to_string(), base_mqtt_args()).expect("target should build");
+        let (client, _event_loop) = AsyncClient::builder(MqttOptions::new("mqtt-timeout-test", ("localhost", 1883)))
+            .capacity(1)
+            .build();
+        client
+            .publish("fill", QoS::AtLeastOnce, false, b"fill".as_slice())
+            .await
+            .expect("first publish should fill the local channel");
+        *target.client.lock().await = Some(client);
+        target.connected.store(true, Ordering::SeqCst);
+        let meta = QueuedPayloadMeta::new(
+            rustfs_s3_types::EventName::ObjectCreatedPut,
+            "bucket".to_string(),
+            "object".to_string(),
+            "application/json",
+            2,
+        );
+
+        let error = target
+            .send_body(b"{}".to_vec(), &meta)
+            .await
+            .expect_err("a full local request channel should hit the enqueue deadline");
+
+        assert!(matches!(error, TargetError::Timeout(_)));
+        assert!(
+            target.connected.load(Ordering::SeqCst),
+            "local admission pressure is not evidence that the MQTT session disconnected"
+        );
+    }
+
    #[test]
    fn next_reconnect_backoff_doubles_until_capped() {
        let mut backoff = MQTT_RECONNECT_BACKOFF_MIN;
@@ -1864,21 +1931,13 @@ mod tests {
    #[test]
    fn debug_redacts_mqtt_secret_fields() {
        let args = MQTTArgs {
-            enable: true,
-            broker: Url::parse("mqtt://broker.example.com:1883").expect("valid broker"),
-            topic: "rustfs/events".to_string(),
-            qos: QoS::AtLeastOnce,
            username: "mqtt-user".to_string(),
            password: "mqtt-password".to_string(),
            tls: MQTTTlsConfig {
                client_key_path: "/etc/rustfs/mqtt.key".to_string(),
                ..MQTTTlsConfig::default()
            },
-            max_reconnect_interval: Duration::from_secs(1),
-            keep_alive: Duration::from_secs(30),
-            queue_dir: String::new(),
-            queue_limit: 0,
-            target_type: TargetType::NotifyEvent,
+            ..base_mqtt_args()
        };

        let rendered = format!("{args:?}");
@@ -25,7 +25,7 @@ use crate::{
    target::{
        ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
        TargetType, build_queued_payload, delete_stored_payload, is_connectivity_error, open_target_queue_store,
-        persist_queued_payload_to_store, redacted_secret,
+        persist_queued_payload_to_store, redacted_secret, with_delivery_deadline,
    },
 };
 use async_trait::async_trait;
@@ -47,6 +47,8 @@ use uuid::Uuid;
 /// `TargetError::Timeout`, a connectivity error, so the payload stays queued
 /// for replay.
 const MYSQL_CONN_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(15);
+/// Absolute ceiling for one INSERT, including pool checkout and server execution.
+const MYSQL_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);

 /// Name of the optional idempotency-key column / primary key. Present on tables
 /// created by this target; absent on legacy two-column tables.
@@ -784,29 +786,33 @@ where
            "Inserting MySQL event"
        );

-        let pool = self.get_or_init_pool().await?;
-        // At this point the pool has already been initialized (get_or_init_pool
-        // succeeded above), so get_conn() failures are always transient: the
-        // connection was lost or the pool is temporarily exhausted.
-        let mut conn = checkout_conn(&pool).await?;
-
        let event_time = extract_event_time(body)?;
        let event_data =
            std::str::from_utf8(body).map_err(|e| TargetError::Serialization(format!("Event body is not valid UTF-8: {e}")))?;

        let quoted_table = quote_table_name(&self.args.table)?;
+        with_delivery_deadline(MYSQL_DELIVERY_TIMEOUT, "MySQL delivery", async {
+            let pool = self.get_or_init_pool().await?;
+            // At this point the pool has already been initialized (get_or_init_pool
+            // succeeded above), so get_conn() failures are always transient: the
+            // connection was lost or the pool is temporarily exhausted.
+            let mut conn = checkout_conn(&pool).await?;

-        if self.idempotency_supported.load(Ordering::Relaxed) {
-            let sql = mysql_insert_sql_with_event_id(&quoted_table);
-            conn.exec_drop(sql, (event_id, event_time.as_str(), event_data))
-                .await
-                .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
-        } else {
-            let sql = mysql_insert_sql_legacy(&quoted_table);
-            conn.exec_drop(sql, (event_time.as_str(), event_data))
-                .await
-                .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
-        }
+            if self.idempotency_supported.load(Ordering::Relaxed) {
+                let sql = mysql_insert_sql_with_event_id(&quoted_table);
+                conn.exec_drop(sql, (event_id, event_time.as_str(), event_data))
+                    .await
+                    .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
+            } else {
+                let sql = mysql_insert_sql_legacy(&quoted_table);
+                conn.exec_drop(sql, (event_time.as_str(), event_data))
+                    .await
+                    .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
+            }
+
+            Ok(())
+        })
+        .await?;

        self.delivery_counters.record_success();
        debug!(target_id = %self.id, "MySQL event inserted");
@@ -25,7 +25,7 @@ use crate::{
    target::{
        ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
        TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error,
-        open_target_queue_store_typed, persist_queued_payload_to_store, redacted_secret,
+        open_target_queue_store_typed, persist_queued_payload_to_store, redacted_secret, with_delivery_deadline,
    },
 };
 use async_trait::async_trait;
@@ -52,6 +52,8 @@ use publish_error::{classify_nats_flush_error, classify_nats_publish_error};
 pub(crate) use jetstream::resolve_dedup_id;
 pub(crate) use validation::{validate_jetstream_settings, validate_jetstream_stream};

+const NATS_CORE_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
+
 #[derive(Clone)]
 pub struct NATSArgs {
    pub enable: bool,
@@ -397,9 +399,21 @@ where
    }

    async fn send_body(&self, body: Vec<u8>) -> Result<(), TargetError> {
-        let client = self.get_or_connect().await?;
-        if let Err(e) = client.publish(self.args.subject.clone(), body.into()).await {
-            let err = classify_nats_publish_error(&e);
+        let result = with_delivery_deadline(NATS_CORE_DELIVERY_TIMEOUT, "NATS delivery", async {
+            let client = self.get_or_connect().await?;
+            client
+                .publish(self.args.subject.clone(), body.into())
+                .await
+                .map_err(|err| classify_nats_publish_error(&err))?;
+
+            // publish only enqueues the message on the client's outbound channel. Flush to confirm the
+            // message reached the server before delivery is treated as successful (backlog#971).
+            client.flush().await.map_err(|err| classify_nats_flush_error(&err))?;
+            Ok(())
+        })
+        .await;
+
+        if let Err(err) = result {
            if is_connectivity_error(&err) {
                self.invalidate_cached_client_connection().await;
                self.connected.store(false, Ordering::SeqCst);
@@ -407,15 +421,6 @@ where
            return Err(err);
        }

-        // publish only enqueues the message on the client's outbound channel. Flush to confirm the
-        // message reached the server before delivery is treated as successful (backlog#971).
-        if let Err(e) = client.flush().await {
-            let err = classify_nats_flush_error(&e);
-            self.invalidate_cached_client_connection().await;
-            self.connected.store(false, Ordering::SeqCst);
-            return Err(err);
-        }
-
        self.delivery_counters.record_success();
        Ok(())
    }
@@ -38,7 +38,7 @@ use crate::{
    target::{
        ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
        TargetType, build_queued_payload, open_target_queue_store, persist_queued_payload_to_store, redacted_optional_secret,
-        redacted_secret,
+        redacted_secret, with_delivery_deadline,
    },
 };
 use async_trait::async_trait;
@@ -70,6 +70,8 @@ const POSTGRES_POOL_RECYCLE_TIMEOUT: Duration = Duration::from_secs(10);
 /// Absolute ceiling on a single checkout, wrapping `pool.get()` in a Tokio
 /// timeout as a belt-and-suspenders guard on top of the deadpool timeouts.
 const POSTGRES_POOL_CHECKOUT_HARD_LIMIT: Duration = Duration::from_secs(20);
+/// Absolute ceiling for one SQL delivery, including pool checkout and execution.
+const POSTGRES_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);

 /// Returns `true` for any `s3:ObjectRemoved:*` event.
 ///
@@ -730,30 +732,29 @@ where

        let key = resolve_payload_key(&payload, meta);

-        let result = match self.args.format {
-            // For the single-row `namespace` format, an object removal must
-            // delete the row rather than UPSERT it, otherwise stale state
-            // lingers in the table after the object is gone.
-            PostgresFormat::Namespace if is_object_removed_event(&meta.event_name) => {
-                client.execute(&self.namespace_delete_sql, &[&key]).await
+        with_delivery_deadline(POSTGRES_DELIVERY_TIMEOUT, "PostgreSQL delivery", async {
+            match self.args.format {
+                // For the single-row `namespace` format, an object removal must
+                // delete the row rather than UPSERT it, otherwise stale state
+                // lingers in the table after the object is gone.
+                PostgresFormat::Namespace if is_object_removed_event(&meta.event_name) => {
+                    client.execute(&self.namespace_delete_sql, &[&key]).await
+                }
+                PostgresFormat::Namespace => client.execute(&self.namespace_sql, &[&key, &payload]).await,
+                PostgresFormat::Access => {
+                    let event_name_str = meta.event_name.to_string();
+                    let queued_at_ms = meta.queued_at_unix_ms as i64;
+                    client
+                        .execute(&self.access_sql, &[&event_id, &event_name_str, &key, &payload, &queued_at_ms])
+                        .await
+                }
            }
-            PostgresFormat::Namespace => client.execute(&self.namespace_sql, &[&key, &payload]).await,
-            PostgresFormat::Access => {
-                let event_name_str = meta.event_name.to_string();
-                let queued_at_ms = meta.queued_at_unix_ms as i64;
-                client
-                    .execute(&self.access_sql, &[&event_id, &event_name_str, &key, &payload, &queued_at_ms])
-                    .await
-            }
-        };
+            .map_err(|err| map_pg_error(&err, "PostgreSQL insert failed"))
+        })
+        .await?;

-        match result {
-            Ok(_) => {
-                self.delivery_counters.record_success();
-                Ok(())
-            }
-            Err(err) => Err(map_pg_error(&err, "PostgreSQL insert failed")),
-        }
+        self.delivery_counters.record_success();
+        Ok(())
    }

    /// Probes the table from `init()`. Failure is non-fatal when a queue is
@@ -24,8 +24,9 @@ use crate::{
    store::{Key, Store},
    target::{
        ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, open_target_queue_store,
-        persist_queued_payload_to_store, redacted_secret, sanitize_queue_dir_component,
+        TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error,
+        open_target_queue_store, persist_queued_payload_to_store, redacted_secret, sanitize_queue_dir_component,
+        with_delivery_deadline,
    },
 };
 use async_trait::async_trait;
@@ -39,11 +40,15 @@ use std::fmt;
 use std::path::Path;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
 use tokio::sync::Mutex as AsyncMutex;
-use tracing::{info, instrument};
+use tracing::{info, instrument, warn};
 use url::Url;
 use uuid::Uuid;

+const PULSAR_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
+const PULSAR_FAILED_DELIVERY_CLEANUP_TIMEOUT: Duration = Duration::from_secs(1);
+
 #[derive(Clone)]
 pub struct PulsarArgs {
    pub enable: bool,
@@ -276,6 +281,23 @@ where
        self.tls_state.lock().reset();
    }

+    async fn clear_failed_delivery_state(&self) {
+        match tokio::time::timeout(PULSAR_FAILED_DELIVERY_CLEANUP_TIMEOUT, self.producer.lock()).await {
+            Ok(mut producer) => {
+                producer.take();
+            }
+            Err(_) => {
+                warn!(
+                    target_id = %self.id,
+                    reason = "producer_cleanup_lock_timeout",
+                    "Timed out clearing the Pulsar producer after a failed delivery"
+                );
+            }
+        }
+        self.clear_cached_client();
+        self.connected.store(false, Ordering::SeqCst);
+    }
+
    async fn get_or_connect_client(&self) -> Result<Pulsar<TokioExecutor>, TargetError> {
        // When a TLS reload adapter is attached, it drives client rebuilds
        // in the background. The inline per-send fingerprint check is skipped.
@@ -334,20 +356,30 @@ where
    }

    async fn send_body(&self, body: Vec<u8>) -> Result<(), TargetError> {
-        self.init_producer().await?;
-        let mut guard = self.producer.lock().await;
-        let producer = guard
-            .as_mut()
-            .ok_or_else(|| TargetError::Configuration("Pulsar producer not initialized".to_string()))?;
-        let receipt = producer
-            .send_non_blocking(body)
-            .await
-            .map_err(|e| TargetError::Request(format!("Failed to send Pulsar message: {e}")))?;
-        receipt
-            .await
-            .map_err(|e| TargetError::Request(format!("Failed to receive Pulsar receipt: {e}")))?;
-        self.delivery_counters.record_success();
-        Ok(())
+        let result = with_delivery_deadline(PULSAR_DELIVERY_TIMEOUT, "Pulsar delivery", async {
+            self.init_producer().await?;
+            let mut guard = self.producer.lock().await;
+            let producer = guard
+                .as_mut()
+                .ok_or_else(|| TargetError::Configuration("Pulsar producer not initialized".to_string()))?;
+            let receipt = producer
+                .send_non_blocking(body)
+                .await
+                .map_err(|e| TargetError::Request(format!("Failed to send Pulsar message: {e}")))?;
+            receipt
+                .await
+                .map_err(|e| TargetError::Request(format!("Failed to receive Pulsar receipt: {e}")))?;
+            self.delivery_counters.record_success();
+            Ok(())
+        })
+        .await;
+
+        if let Err(err) = &result
+            && is_connectivity_error(err)
+        {
+            self.clear_failed_delivery_state().await;
+        }
+        result
    }
 }

@@ -525,6 +557,22 @@ mod tests {
        }
    }

+    #[tokio::test(start_paused = true)]
+    async fn failed_delivery_cleanup_is_bounded_when_the_producer_lock_is_busy() {
+        let target = Arc::new(PulsarTarget::<String>::new("pulsar:test".to_string(), base_args()).expect("target should build"));
+        target.connected.store(true, Ordering::SeqCst);
+        let producer_guard = target.producer.lock().await;
+        let cleanup = {
+            let target = Arc::clone(&target);
+            tokio::spawn(async move { target.clear_failed_delivery_state().await })
+        };
+
+        cleanup.await.expect("cleanup task should not panic");
+
+        assert!(!target.connected.load(Ordering::SeqCst));
+        drop(producer_guard);
+    }
+
    #[test]
    fn debug_redacts_pulsar_secret_fields() {
        let args = PulsarArgs {
@@ -26,6 +26,7 @@ use crate::{
        ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
        TargetType, build_queued_payload, invalidate_cache_on_connectivity_error, is_connectivity_error,
        mark_target_disconnected_on_connectivity_error, open_target_queue_store, persist_queued_payload_to_store,
+        with_delivery_deadline,
    },
 };
 use async_trait::async_trait;
@@ -48,6 +49,19 @@ use tokio::sync::Mutex;
 use tracing::{debug, info, instrument, warn};
 use url::Url;

+const REDIS_CONNECTION_TIMEOUT_DEFAULT: Duration = Duration::from_secs(5);
+const REDIS_RESPONSE_TIMEOUT_DEFAULT: Duration = Duration::from_secs(5);
+
+fn redis_total_delivery_timeout(args: &RedisArgs) -> Duration {
+    let attempts = u32::try_from(args.max_retry_attempts).unwrap_or(u32::MAX);
+    let per_attempt = args
+        .connection_timeout
+        .unwrap_or(REDIS_CONNECTION_TIMEOUT_DEFAULT)
+        .saturating_add(args.response_timeout.unwrap_or(REDIS_RESPONSE_TIMEOUT_DEFAULT))
+        .saturating_add(args.max_retry_delay.unwrap_or(Duration::from_secs(2)));
+    per_attempt.saturating_mul(attempts)
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum RedisTlsPolicy {
    SystemCa,
@@ -214,6 +228,16 @@ impl RedisArgs {
            ));
        }

+        if self.connection_timeout == Some(Duration::ZERO) {
+            return Err(TargetError::Configuration(
+                "Redis connection_timeout must be greater than zero".to_string(),
+            ));
+        }
+
+        if self.response_timeout == Some(Duration::ZERO) {
+            return Err(TargetError::Configuration("Redis response_timeout must be greater than zero".to_string()));
+        }
+
        if self.pipeline_buffer_size == Some(0) {
            return Err(TargetError::Configuration(
                "Redis pipeline_buffer_size must be greater than zero".to_string(),
@@ -464,71 +488,97 @@ where
            "Sending Redis payload"
        );

-        let mut attempt = 0usize;
-        let mut last_error = None;
-        while attempt < self.args.max_retry_attempts {
-            attempt += 1;
+        let result = with_delivery_deadline(redis_total_delivery_timeout(&self.args), "Redis delivery", async {
+            let mut attempt = 0usize;
+            let mut last_error = None;
+            while attempt < self.args.max_retry_attempts {
+                attempt += 1;

-            let mut publisher = self.get_or_create_publisher().await?;
-            match publisher
-                .publish::<_, _, i64>(self.args.channel.as_str(), body.as_slice())
+                let connection_timeout = self.args.connection_timeout.unwrap_or(REDIS_CONNECTION_TIMEOUT_DEFAULT);
+                let mut publisher = match with_delivery_deadline(
+                    connection_timeout,
+                    "Redis connection",
+                    self.get_or_create_publisher(),
+                )
                .await
-            {
-                Ok(receiver_count) => {
-                    // PUBLISH returns the number of subscribers that received the
-                    // message. Redis pub/sub is best-effort: with zero subscribers
-                    // the event is delivered to no one, yet the durable copy is
-                    // deleted. Warn so operators relying on reliable delivery are
-                    // not silently losing events (backlog#982).
-                    if receiver_count == 0 {
+                {
+                    Ok(publisher) => publisher,
+                    Err(err) => {
+                        invalidate_cache_on_connectivity_error(&err, || self.invalidate_cached_publisher()).await;
+                        return Err(err);
+                    }
+                };
+                let response_timeout = self.args.response_timeout.unwrap_or(REDIS_RESPONSE_TIMEOUT_DEFAULT);
+                match with_delivery_deadline(response_timeout, "Redis publish response", async {
+                    publisher
+                        .publish::<_, _, i64>(self.args.channel.as_str(), body.as_slice())
+                        .await
+                        .map_err(map_redis_error)
+                })
+                .await
+                {
+                    Ok(receiver_count) => {
+                        // PUBLISH returns the number of subscribers that received the
+                        // message. Redis pub/sub is best-effort: with zero subscribers
+                        // the event is delivered to no one, yet the durable copy is
+                        // deleted. Warn so operators relying on reliable delivery are
+                        // not silently losing events (backlog#982).
+                        if receiver_count == 0 {
+                            warn!(
+                                target_id = %self.id,
+                                channel = %self.args.channel,
+                                "Redis PUBLISH reached 0 subscribers; the event was not received by any consumer (pub/sub is best-effort)"
+                            );
+                        }
+                        debug!(
+                            target_id = %self.id,
+                            channel = %self.args.channel,
+                            attempt,
+                            receiver_count,
+                            "Event published to Redis channel"
+                        );
+                        self.delivery_counters.record_success();
+                        return Ok(());
+                    }
+                    Err(mapped) => {
+                        invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_publisher()).await;
+
                        warn!(
                            target_id = %self.id,
                            channel = %self.args.channel,
-                            "Redis PUBLISH reached 0 subscribers; the event was not received by any consumer (pub/sub is best-effort)"
+                            attempt,
+                            max_attempts = self.args.max_retry_attempts,
+                            error = %mapped,
+                            "Redis publish attempt failed"
                        );
-                    }
-                    debug!(
-                        target_id = %self.id,
-                        channel = %self.args.channel,
-                        attempt,
-                        receiver_count,
-                        "Event published to Redis channel"
-                    );
-                    self.delivery_counters.record_success();
-                    return Ok(());
-                }
-                Err(err) => {
-                    let mapped = map_redis_error(err);
-                    invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_publisher()).await;

-                    warn!(
-                        target_id = %self.id,
-                        channel = %self.args.channel,
-                        attempt,
-                        max_attempts = self.args.max_retry_attempts,
-                        error = %mapped,
-                        "Redis publish attempt failed"
-                    );
+                        if !is_connectivity_error(&mapped) || attempt >= self.args.max_retry_attempts {
+                            last_error = Some(mapped);
+                            break;
+                        }

-                    if !is_connectivity_error(&mapped) || attempt >= self.args.max_retry_attempts {
                        last_error = Some(mapped);
-                        break;
+                        tokio::time::sleep(compute_retry_delay(
+                            attempt,
+                            self.args.min_retry_delay.unwrap_or(Duration::from_millis(100)),
+                            self.args.max_retry_delay.unwrap_or(Duration::from_secs(2)),
+                        ))
+                        .await;
                    }
-
-                    last_error = Some(mapped);
-                    tokio::time::sleep(compute_retry_delay(
-                        attempt,
-                        self.args.min_retry_delay.unwrap_or(Duration::from_millis(100)),
-                        self.args.max_retry_delay.unwrap_or(Duration::from_secs(2)),
-                    ))
-                    .await;
                }
            }
+
+            Err(last_error.unwrap_or(TargetError::Unknown(
+                "Redis publish failed without a captured error".to_string(),
+            )))
+        })
+        .await;
+
+        if let Err(err) = &result {
+            invalidate_cache_on_connectivity_error(err, || self.invalidate_cached_publisher()).await;
+            self.connected.store(false, Ordering::SeqCst);
        }
-
-        self.connected.store(false, Ordering::SeqCst);
-
-        Err(last_error.unwrap_or(TargetError::Unknown("Redis publish failed without a captured error".to_string())))
+        result
    }
 }

@@ -551,7 +601,7 @@ where
        // thus a fresh TCP+TLS handshake — on every health check (backlog#982).
        // ensure_publisher_ready already invalidates the cached manager on a
        // connectivity error so the next attempt rebuilds it.
-        match tokio::time::timeout(Duration::from_secs(5), self.ensure_publisher_ready()).await {
+        match tokio::time::timeout(REDIS_CONNECTION_TIMEOUT_DEFAULT, self.ensure_publisher_ready()).await {
            Ok(Ok(())) => {
                self.connected.store(true, Ordering::SeqCst);
                Ok(true)
@@ -917,6 +967,26 @@ mod tests {
        assert!(args.validate().is_err());
    }

+    #[test]
+    fn validate_rejects_zero_connection_timeout() {
+        let args = RedisArgs {
+            connection_timeout: Some(Duration::ZERO),
+            ..base_args()
+        };
+
+        assert!(args.validate().is_err());
+    }
+
+    #[test]
+    fn validate_rejects_zero_response_timeout() {
+        let args = RedisArgs {
+            response_timeout: Some(Duration::ZERO),
+            ..base_args()
+        };
+
+        assert!(args.validate().is_err());
+    }
+
    #[test]
    fn validate_accepts_custom_ca_tls_policy() {
        let args = RedisArgs {
@@ -1237,6 +1307,23 @@ mod tests {
        assert_eq!(target.delivery_snapshot().total_messages, 1);
    }

+    #[tokio::test(start_paused = true)]
+    async fn delivery_budget_respects_response_timeout_longer_than_sixty_seconds() {
+        let mut args = base_args();
+        args.max_retry_attempts = 1;
+        args.response_timeout = Some(Duration::from_secs(90));
+        let manager_config = build_redis_connection_manager_config(&args);
+
+        assert_eq!(manager_config.response_timeout(), Some(Duration::from_secs(90)));
+        assert_eq!(redis_total_delivery_timeout(&args), Duration::from_secs(97));
+        with_delivery_deadline(redis_total_delivery_timeout(&args), "Redis delivery", async {
+            tokio::time::sleep(Duration::from_secs(70)).await;
+            Ok::<_, TargetError>(())
+        })
+        .await
+        .expect("the configured delivery budget must not impose a fixed sixty-second cap");
+    }
+
    #[tokio::test]
    async fn send_body_sets_connected_false_after_retry_exhaustion() {
        let listener = match TcpListener::bind("127.0.0.1:0").await {
@@ -24,8 +24,8 @@ use crate::{
    store::{Key, Store},
    target::{
        ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, open_target_queue_store,
-        persist_queued_payload_to_store, redacted_secret,
+        TargetHealth, TargetHealthReason, TargetHealthState, TargetTlsState, TargetType, build_queued_payload,
+        build_target_tls_fingerprint, open_target_queue_store, persist_queued_payload_to_store, redacted_secret,
    },
 };
 use async_trait::async_trait;
@@ -34,6 +34,7 @@ use reqwest::{Client, StatusCode, Url};
 use rustfs_tls_runtime::load_cert_bundle_der_bytes;
 use rustfs_utils::egress::validate_outbound_url;
 use std::{
+    error::Error as StdError,
    fmt,
    marker::PhantomData,
    sync::{
@@ -49,6 +50,69 @@ const LOG_COMPONENT_TARGETS: &str = "targets";
 const LOG_SUBSYSTEM_WEBHOOK: &str = "webhook";
 const EVENT_WEBHOOK_TARGET_STATE: &str = "webhook_target_state";
 const EVENT_WEBHOOK_DELIVERY_STATE: &str = "webhook_delivery_state";
+const WEBHOOK_HEALTH_TIMEOUT: Duration = Duration::from_secs(5);
+
+fn classify_probe_error(err: &reqwest::Error) -> TargetHealthReason {
+    if err.is_timeout() {
+        return TargetHealthReason::TimedOut;
+    }
+
+    let mut source = err.source();
+    while let Some(cause) = source {
+        if cause.downcast_ref::<rustls::Error>().is_some() {
+            return TargetHealthReason::TlsFailure;
+        }
+        if let Some(io_error) = cause.downcast_ref::<std::io::Error>() {
+            match io_error.kind() {
+                std::io::ErrorKind::ConnectionRefused => return TargetHealthReason::ConnectionRefused,
+                std::io::ErrorKind::NotFound | std::io::ErrorKind::AddrNotAvailable => {
+                    return TargetHealthReason::DnsFailure;
+                }
+                _ => {}
+            }
+        }
+
+        let label = cause.to_string().to_ascii_lowercase();
+        if label.contains("dns error") || label.contains("failed to lookup") || label.contains("name or service not known") {
+            return TargetHealthReason::DnsFailure;
+        }
+        if label.contains("certificate") || label.contains("tls") {
+            return TargetHealthReason::TlsFailure;
+        }
+        source = cause.source();
+    }
+
+    TargetHealthReason::Unreachable
+}
+
+async fn probe_health_url(client: &Client, health_check_url: &Url) -> TargetHealth {
+    match tokio::time::timeout(WEBHOOK_HEALTH_TIMEOUT, client.head(health_check_url.as_str()).send()).await {
+        Ok(Ok(_)) => TargetHealth::online(TargetHealthReason::Reachable),
+        Ok(Err(err)) => TargetHealth::error(classify_probe_error(&err)),
+        Err(_) => TargetHealth::error(TargetHealthReason::TimedOut),
+    }
+}
+
+fn classify_delivery_status(status: StatusCode) -> Result<(), TargetError> {
+    if status.is_success() {
+        Ok(())
+    } else if status.is_redirection() {
+        Err(TargetError::Request(format!(
+            "Webhook endpoint returned redirect '{}'; redirects are not followed for webhook delivery",
+            status
+        )))
+    } else if status == StatusCode::FORBIDDEN || status == StatusCode::UNAUTHORIZED {
+        Err(TargetError::Authentication(format!(
+            "Webhook endpoint returned '{}', please check if your auth token is correctly set",
+            status
+        )))
+    } else {
+        Err(TargetError::Request(format!(
+            "Webhook endpoint returned '{}', please check your endpoint configuration",
+            status
+        )))
+    }
+}

 /// Arguments for configuring a Webhook target
 #[derive(Clone)]
@@ -79,7 +143,7 @@ impl fmt::Debug for WebhookArgs {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("WebhookArgs")
            .field("enable", &self.enable)
-            .field("endpoint", &self.endpoint)
+            .field("endpoint_origin", &self.endpoint.origin().ascii_serialization())
            .field("auth_token", &redacted_secret(&self.auth_token))
            .field("queue_dir", &self.queue_dir)
            .field("queue_limit", &self.queue_limit)
@@ -243,7 +307,6 @@ where
                event = EVENT_WEBHOOK_TARGET_STATE,
                component = LOG_COMPONENT_TARGETS,
                subsystem = LOG_SUBSYSTEM_WEBHOOK,
-                endpoint = %args.endpoint,
                state = "tls_verification_skipped",
                fallback = "danger_accept_invalid_certs",
                "webhook target state"
@@ -308,8 +371,14 @@ where
    fn health_check_url(endpoint: &Url) -> Result<Url, TargetError> {
        endpoint
            .host()
-            .ok_or_else(|| TargetError::Configuration(format!("Webhook endpoint '{}' is missing a host", endpoint)))?;
+            .ok_or_else(|| TargetError::Configuration("Webhook endpoint is missing a host".to_string()))?;
        let mut health_check_url = endpoint.clone();
+        health_check_url
+            .set_username("")
+            .map_err(|_| TargetError::Configuration("Webhook endpoint contains invalid user information".to_string()))?;
+        health_check_url
+            .set_password(None)
+            .map_err(|_| TargetError::Configuration("Webhook endpoint contains invalid user information".to_string()))?;
        health_check_url.set_path("/");
        health_check_url.set_query(None);
        health_check_url.set_fragment(None);
@@ -317,39 +386,34 @@ where
        Ok(health_check_url)
    }

-    async fn probe_reachability(&self) -> Result<bool, TargetError> {
+    async fn probe_health(&self) -> TargetHealth {
        let Some(health_check_url) = self.health_check_url.as_ref() else {
-            return Ok(false);
+            return TargetHealth::offline(TargetHealthReason::Unreachable);
        };
-
        let client = self.http_client.lock().clone();
-        match tokio::time::timeout(Duration::from_secs(5), client.head(health_check_url.as_str()).send()).await {
-            Ok(Ok(resp)) => {
-                debug!(
-                    event = EVENT_WEBHOOK_TARGET_STATE,
-                    component = LOG_COMPONENT_TARGETS,
-                    subsystem = LOG_SUBSYSTEM_WEBHOOK,
-                    target_id = %self.id,
-                    status = %resp.status(),
-                    health_check_url = %health_check_url,
-                    state = "reachability_probe_succeeded",
-                    "webhook target state"
-                );
-                Ok(true)
-            }
-            Ok(Err(err)) if err.is_timeout() => Err(TargetError::Timeout(format!(
-                "Webhook health check request to {} timed out",
-                health_check_url
-            ))),
-            Ok(Err(err)) if err.is_connect() => Ok(false),
-            Ok(Err(err)) => Err(TargetError::Network(format!(
-                "Webhook health check request to {} failed: {}",
-                health_check_url, err
-            ))),
-            Err(_) => Err(TargetError::Timeout(format!(
-                "Webhook health check request to {} timed out",
-                health_check_url
-            ))),
+        let health = probe_health_url(&client, health_check_url).await;
+        if health.state == TargetHealthState::Online {
+            debug!(
+                event = EVENT_WEBHOOK_TARGET_STATE,
+                component = LOG_COMPONENT_TARGETS,
+                subsystem = LOG_SUBSYSTEM_WEBHOOK,
+                target_id = %self.id,
+                state = "reachability_probe_succeeded",
+                "webhook target state"
+            );
+        }
+        health
+    }
+
+    async fn probe_reachability(&self) -> Result<bool, TargetError> {
+        let health = self.probe_health().await;
+        match health.state {
+            TargetHealthState::Online => Ok(true),
+            TargetHealthState::Offline | TargetHealthState::Disabled => Ok(false),
+            TargetHealthState::Error => match health.reason {
+                TargetHealthReason::TimedOut => Err(TargetError::Timeout("Webhook health check timed out".to_string())),
+                _ => Err(TargetError::Network(format!("Webhook health check failed: {}", health.reason.as_str()))),
+            },
        }
    }

@@ -371,7 +435,6 @@ where
                    component = LOG_COMPONENT_TARGETS,
                    subsystem = LOG_SUBSYSTEM_WEBHOOK,
                    target_id = %self.id,
-                    health_check_url = ?self.health_check_url,
                    state = "reachable",
                    "webhook target state"
                );
@@ -447,7 +510,7 @@ where
            if e.is_timeout() || e.is_connect() {
                TargetError::NotConnected
            } else {
-                TargetError::Request(format!("Failed to send request: {e}"))
+                TargetError::Request("Webhook delivery request failed".to_string())
            }
        })?;

@@ -456,7 +519,8 @@ where
        // pool and can be reused (keep-alive) instead of being closed mid-stream
        // (backlog#983). The body content is not needed for delivery accounting.
        let _ = resp.bytes().await;
-        if status.is_success() {
+        let result = classify_delivery_status(status);
+        if result.is_ok() {
            debug!(
                event = EVENT_WEBHOOK_DELIVERY_STATE,
                component = LOG_COMPONENT_TARGETS,
@@ -467,26 +531,8 @@ where
                "webhook delivery state"
            );
            self.delivery_counters.record_success();
-            Ok(())
-        } else if status.is_redirection() {
-            // SSRF hardening (backlog#974): redirects are intentionally not followed
-            // (see build_http_client). Treat a 3xx response as a delivery failure rather
-            // than silently chasing the redirect to a potentially internal target.
-            Err(TargetError::Request(format!(
-                "{} returned redirect '{}'; redirects are not followed for webhook delivery",
-                self.args.endpoint, status
-            )))
-        } else if status == StatusCode::FORBIDDEN {
-            Err(TargetError::Authentication(format!(
-                "{} returned '{}', please check if your auth token is correctly set",
-                self.args.endpoint, status
-            )))
-        } else {
-            Err(TargetError::Request(format!(
-                "{} returned '{}', please check your endpoint configuration",
-                self.args.endpoint, status
-            )))
        }
+        result
    }
 }

@@ -507,6 +553,13 @@ where
        self.probe_reachability().await
    }

+    async fn health(&self) -> TargetHealth {
+        if !self.args.enable {
+            return TargetHealth::disabled();
+        }
+        self.probe_health().await
+    }
+
    async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
        let queued = match self.build_queued_payload(&event) {
            Ok(queued) => queued,
@@ -700,8 +753,9 @@ where

 #[cfg(test)]
 mod tests {
-    use super::{WebhookArgs, WebhookTarget};
-    use crate::target::{REDACTED_SECRET, Target, TargetType, decode_object_name};
+    use super::{WebhookArgs, WebhookTarget, classify_delivery_status, probe_health_url};
+    use crate::target::{REDACTED_SECRET, Target, TargetHealthReason, TargetHealthState, TargetType, decode_object_name};
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
    use url::Url;
    use url::form_urlencoded;

@@ -720,9 +774,25 @@ mod tests {
        }
    }

+    async fn http_status_url(status: u16) -> Url {
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind test server");
+        let address = listener.local_addr().expect("test server address");
+        tokio::spawn(async move {
+            let (mut stream, _) = listener.accept().await.expect("accept health probe");
+            let mut request = [0u8; 1024];
+            let _ = stream.read(&mut request).await;
+            stream
+                .write_all(format!("HTTP/1.1 {status} Test\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").as_bytes())
+                .await
+                .expect("write health response");
+        });
+        Url::parse(&format!("http://{address}/")).expect("health probe URL")
+    }
+
    #[test]
    fn debug_redacts_webhook_secret_fields() {
        let args = WebhookArgs {
+            endpoint: Url::parse("https://user:password@example.com/private?token=query-secret").expect("debug URL"),
            auth_token: "webhook-token".to_string(),
            client_key: "/etc/rustfs/webhook.key".to_string(),
            ..base_args()
@@ -732,7 +802,11 @@ mod tests {

        assert!(!rendered.contains("webhook-token"));
        assert!(!rendered.contains("/etc/rustfs/webhook.key"));
+        assert!(!rendered.contains("password"));
+        assert!(!rendered.contains("/private"));
+        assert!(!rendered.contains("query-secret"));
        assert!(rendered.contains(REDACTED_SECRET));
+        assert!(rendered.contains("https://example.com"));
        assert!(rendered.contains("WebhookArgs"));
    }

@@ -824,12 +898,124 @@ mod tests {

    #[test]
    fn test_health_check_url_ignores_endpoint_path() {
-        let endpoint = Url::parse("https://example.com:9443/hook/path").unwrap();
-        let health_check_url = WebhookTarget::<serde_json::Value>::health_check_url(&endpoint).unwrap();
+        let endpoint = Url::parse("https://user:password@example.com:9443/hook/path?token=secret").expect("webhook endpoint URL");
+        let health_check_url = WebhookTarget::<serde_json::Value>::health_check_url(&endpoint).expect("webhook health-check URL");

        assert_eq!(health_check_url.as_str(), "https://example.com:9443/");
    }

+    #[tokio::test]
+    async fn head_http_responses_only_measure_reachability() {
+        let client = WebhookTarget::<serde_json::Value>::build_http_client(&base_args()).expect("build client");
+        for status in [401, 404, 500] {
+            let health = probe_health_url(&client, &http_status_url(status).await).await;
+            assert_eq!(health.state, TargetHealthState::Online, "HEAD {status} is reachable");
+            assert_eq!(health.reason, TargetHealthReason::Reachable);
+        }
+    }
+
+    #[tokio::test]
+    async fn refused_connection_has_stable_health_reason() {
+        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve refused port");
+        let address = listener.local_addr().expect("refused address");
+        drop(listener);
+        let url = Url::parse(&format!("http://{address}/")).expect("refused URL");
+        let client = WebhookTarget::<serde_json::Value>::build_http_client(&base_args()).expect("build client");
+
+        let health = probe_health_url(&client, &url).await;
+
+        assert_eq!(health.state, TargetHealthState::Error);
+        assert_eq!(health.reason, TargetHealthReason::ConnectionRefused);
+    }
+
+    #[tokio::test]
+    async fn dns_failure_has_stable_health_reason() {
+        let url = Url::parse("http://rustfs-health-check.invalid/").expect("invalid test domain URL");
+        let client = WebhookTarget::<serde_json::Value>::build_http_client(&base_args()).expect("build client");
+
+        let health = probe_health_url(&client, &url).await;
+
+        assert_eq!(health.state, TargetHealthState::Error);
+        assert_eq!(health.reason, TargetHealthReason::DnsFailure);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn webhook_health_probe_has_a_five_second_total_budget() {
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
+            .await
+            .expect("bind stalled server");
+        let address = listener.local_addr().expect("stalled server address");
+        let server = tokio::spawn(async move {
+            let (_stream, _) = listener.accept().await.expect("accept stalled probe");
+            std::future::pending::<()>().await;
+        });
+        let url = Url::parse(&format!("http://{address}/")).expect("stalled URL");
+        let client = WebhookTarget::<serde_json::Value>::build_http_client(&base_args()).expect("build client");
+        let started = tokio::time::Instant::now();
+
+        let health = probe_health_url(&client, &url).await;
+
+        assert_eq!(started.elapsed(), super::WEBHOOK_HEALTH_TIMEOUT);
+        assert_eq!(health.state, TargetHealthState::Error);
+        assert_eq!(health.reason, TargetHealthReason::TimedOut);
+        server.abort();
+    }
+
+    #[tokio::test]
+    async fn tls_failure_has_stable_health_reason() {
+        use rustls::{
+            ServerConfig, ServerConnection, StreamOwned,
+            pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer},
+        };
+        use std::io::Read;
+        use std::sync::{Arc, Once};
+
+        static INSTALL_CRYPTO_PROVIDER: Once = Once::new();
+        INSTALL_CRYPTO_PROVIDER.call_once(|| {
+            let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
+        });
+        let rcgen::CertifiedKey { cert, signing_key } =
+            rcgen::generate_simple_self_signed(vec!["localhost".to_string()]).expect("cert should generate");
+        let server_config = Arc::new(
+            ServerConfig::builder()
+                .with_no_client_auth()
+                .with_single_cert(
+                    vec![cert.der().clone()],
+                    PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(signing_key.serialize_der())),
+                )
+                .expect("server cert should be valid"),
+        );
+        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind TLS server");
+        let address = listener.local_addr().expect("TLS server address");
+        let server = std::thread::spawn(move || {
+            let (stream, _) = listener.accept().expect("accept TLS client");
+            let connection = ServerConnection::new(server_config).expect("server connection");
+            let mut tls_stream = StreamOwned::new(connection, stream);
+            let mut request = [0u8; 1024];
+            let _ = tls_stream.read(&mut request);
+        });
+        let url = Url::parse(&format!("https://localhost:{}/", address.port())).expect("TLS health URL");
+        let client = WebhookTarget::<serde_json::Value>::build_http_client(&base_args()).expect("build client");
+
+        let health = probe_health_url(&client, &url).await;
+
+        assert_eq!(health.state, TargetHealthState::Error);
+        assert_eq!(health.reason, TargetHealthReason::TlsFailure);
+        server.join().expect("TLS server thread");
+    }
+
+    #[test]
+    fn post_status_classification_requires_success() {
+        assert!(classify_delivery_status(reqwest::StatusCode::NO_CONTENT).is_ok());
+        for status in [
+            reqwest::StatusCode::MOVED_PERMANENTLY,
+            reqwest::StatusCode::UNAUTHORIZED,
+            reqwest::StatusCode::INTERNAL_SERVER_ERROR,
+        ] {
+            assert!(classify_delivery_status(status).is_err(), "POST {status} must fail");
+        }
+    }
+
    #[tokio::test]
    async fn test_disabled_target_can_be_constructed_without_origin_probe() {
        let args = WebhookArgs {
@@ -952,7 +1138,7 @@ mod tests {
            .expect("https webhook probe should trust configured ca");

        assert_eq!(resp.status(), reqwest::StatusCode::OK);
-        assert_eq!(resp.text_with_charset("utf-8").await.expect("read response body"), "");
+        assert!(resp.bytes().await.expect("read response body").is_empty());
        handle.join().expect("tls server thread");
    }
 }
@@ -20,12 +20,26 @@

 use http::{HeaderMap, HeaderValue};
 use std::borrow::Cow;
+use std::collections::HashMap;
+use std::error::Error;
+use std::fmt::{Display, Formatter};

 const RUSTFS_PREFIX: &str = "x-rustfs-";
 const MINIO_PREFIX: &str = "x-minio-";
 const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-";
 const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-";

+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ConflictingMetadataValue;
+
+impl Display for ConflictingMetadataValue {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_str("metadata contains conflicting values for the same case-insensitive key")
+    }
+}
+
+impl Error for ConflictingMetadataValue {}
+
 // Suffix constants (part after x-rustfs- or x-minio-). Use with get_header/insert_header.
 pub const SUFFIX_FORCE_DELETE: &str = "force-delete";
 pub const SUFFIX_INCLUDE_DELETED: &str = "include-deleted";
@@ -84,6 +98,24 @@ pub fn get_header_map(map: &std::collections::HashMap<String, String>, suffix: &
    map.get(&rk).cloned().or_else(|| map.get(&mk).cloned())
 }

+/// Returns the value when every case-insensitive occurrence of `name` agrees.
+pub fn get_consistent_metadata_value<'a>(
+    metadata: &'a HashMap<String, String>,
+    name: &str,
+) -> Result<Option<&'a str>, ConflictingMetadataValue> {
+    let mut value = None;
+    for (candidate, candidate_value) in metadata {
+        if !candidate.eq_ignore_ascii_case(name) {
+            continue;
+        }
+        if value.is_some_and(|existing| existing != candidate_value) {
+            return Err(ConflictingMetadataValue);
+        }
+        value = Some(candidate_value.as_str());
+    }
+    Ok(value)
+}
+
 /// Insert into HashMap with both x-rustfs-{suffix} and x-minio-{suffix}.
 pub fn insert_header_map(map: &mut std::collections::HashMap<String, String>, suffix: &str, value: impl Into<String>) {
    let v = value.into();
@@ -120,4 +152,28 @@ mod tests {
        headers2.insert("X-Rustfs-Force-Delete", HeaderValue::from_static("true"));
        assert_eq!(get_header(&headers2, SUFFIX_FORCE_DELETE).as_deref(), Some("true"));
    }
+
+    #[test]
+    fn test_get_consistent_metadata_value() {
+        let mut metadata = HashMap::new();
+        assert_eq!(
+            get_consistent_metadata_value(&metadata, "x-test").expect("missing value should be valid"),
+            None
+        );
+
+        metadata.insert("X-Test".to_string(), "value".to_string());
+        assert_eq!(
+            get_consistent_metadata_value(&metadata, "x-test").expect("single value should be valid"),
+            Some("value")
+        );
+
+        metadata.insert("x-test".to_string(), "value".to_string());
+        assert_eq!(
+            get_consistent_metadata_value(&metadata, "x-test").expect("matching values should be valid"),
+            Some("value")
+        );
+
+        metadata.insert("x-TEST".to_string(), "other".to_string());
+        assert!(get_consistent_metadata_value(&metadata, "x-test").is_err());
+    }
 }
@@ -30,6 +30,7 @@ Two rules keep this directory healthy:
 ## Contracts & invariants

 - [erasure-coding.md](erasure-coding.md) — normative erasure-coding algorithm and on-disk (`xl.meta`) compatibility contract; the frozen invariants for all user-data read/write, encode/decode, quorum, heal, and decode tolerance
+- [minio-sse-key-hierarchy.md](minio-sse-key-hierarchy.md) — MinIO-compatible SSE key hierarchy, wrapping algorithms, DARE 2.0 stream format, metadata persistence, and RustFS layer boundaries
 - [placement-repair-invariants.md](placement-repair-invariants.md)
 - [unified-object-generation.md](unified-object-generation.md) — single per-object generation authority (fencing epoch, transport/encoding/proto/mixed-version contracts)
 - [runtime-capability-contracts.md](runtime-capability-contracts.md)
@@ -12,6 +12,7 @@ for later deletion.

 ## Open Items

+- `rustfs-5063` legacy SSE-S3 KMS envelopes: releases before provider routing was separated could wrap an AES256 object's DEK with KMS, so readers identify the persisted KMS envelope and retain KMS decryption for those objects. Remove this compatibility path after the SSE-S3 migration has rewrapped every referenced legacy DEK with the local SSE-S3 key provider.
 - `#4648` walk-dir stream completion capability: old clients can append fallback output to an already-used metacache writer after a terminal body error, so servers emit terminal walk errors only to clients that sign the `walk_dir_stream_completion=error-v1` query capability and its request-body digest. Remove the legacy clean-EOF path after the minimum supported RustFS peer version always advertises this capability.
 - `heal-rpc-auth-v2` internode gRPC authentication: servers temporarily accept legacy prefix signatures so old peers remain available during rolling upgrades. Remove the legacy fallback after the minimum supported RustFS peer version sends v2 authentication on every internode gRPC request.
 - `heal-status-rpc-v1` node heal status capability: new peers treat an unimplemented BackgroundHealStatus RPC as an explicitly incomplete rolling-upgrade response. Remove the fallback after the minimum supported RustFS peer version implements BackgroundHealStatus.
@@ -0,0 +1,983 @@
+# MinIO SSE Key Hierarchy & RustFS Compatibility Contract
+
+This document is the core reference for RustFS implementation and review of
+MinIO-compatible Server-Side Encryption (SSE). It defines key terminology,
+derivation relationships, inter-layer responsibilities, persistence formats,
+data encryption formats for single-part and multipart objects, and the
+compatibility behavior required when reading MinIO objects.
+
+This document describes a long-term architecture contract, not a migration
+plan for a specific implementation phase. The historical RustFS direct-stream
+key format belongs to the read-compatibility surface and does not alter the
+MinIO compatibility target specified herein.
+
+## 1. Basis and Scope
+
+This document takes the following source snapshots as its factual baseline:
+
+- MinIO commit
+  [`7aac2a2c5b7c882e68c1ce017d8256be2feea27f`](https://github.com/minio/minio/tree/7aac2a2c5b7c882e68c1ce017d8256be2feea27f):
+  `cmd/encryption-v1.go`, `internal/crypto/key.go`,
+  `internal/crypto/metadata.go`, `internal/crypto/sse-*.go`.
+- RustFS commit `5ea9a1fd8f3c1b39154b27fd419cde76abc68f3b`:
+  [SSE boundary](../../rustfs/src/storage/sse.rs),
+  [RIO v2 encryption stream](../../crates/rio-v2/src/encrypt_reader.rs), and
+  [MinIO generated fixtures](../../crates/rio-v2/tests/minio_generated_fixtures.rs).
+- MinIO `sio v0.4.1` DARE 2.0 package format; the RustFS compatible
+  implementation is fixed in
+  [RIO v2 encryption stream](../../crates/rio-v2/src/encrypt_reader.rs).
+
+This document covers:
+
+- SSE-C, SSE-S3, and SSE-KMS;
+- KMS envelope encryption, object key derivation, and wrapping;
+- The exact meanings of `ObjectKey`, `sealingKey`, and `SealedKey`;
+- The SIO/RIO DARE 2.0 data stream format;
+- Multipart part keys;
+- MinIO internal object metadata;
+- RustFS write, read, and historical compatibility boundaries.
+
+This document does not specify how a KMS service internally protects its KMS
+master key, nor does it replace the S3 public request header and error code
+API compatibility specification.
+
+## 2. Normative Language
+
+The terms "MUST", "MUST NOT", "SHOULD", and "MAY" in this document carry
+their RFC 2119 meanings. If an implementation conflicts with this document,
+the behavior that can read real MinIO objects without degrading the security
+boundary takes precedence, and this document SHALL be revised accordingly.
+
+## 3. Terminology: Named by Cryptographic Role
+
+`key` and `data key` often refer to different roles in KMS APIs versus the
+object encryption layer. Implementations and reviews MUST use the role names
+in the table below as their primary reference and MUST NOT infer purpose from
+variable names alone.
+
+| Canonical Term | MinIO / RustFS Source Name | Length | Lifetime | Cryptographic Role |
+|---|---|---|---|---|
+| KMS master key | KMS key / master key | KMS-defined | KMS-internal only | Root key; protects the KMS data key |
+| KMS data key plaintext | MinIO `key.Plaintext`; RustFS `DataKey.plaintext_key` | 32 B | Transient, in-process only | Parent/external key of the object envelope; NOT the `ObjectKey` that directly encrypts object content |
+| KMS data key ciphertext | MinIO `key.Ciphertext`; RustFS `encrypted_data_key` | KMS-defined | Persisted | KMS data key plaintext wrapped under the KMS master key |
+| Local SSE-S3 master key | RustFS local SSE-S3 provider master key | 32 B | Process-lifetime secret | Parent/external key for local/K/V SSE-S3; MUST NOT be persisted in object metadata |
+| SSE-C customer key | SSE-C key from the client request | 32 B | In-memory for the request lifetime | Parent/external key for SSE-C |
+| object-key generation nonce | Random value inside `crypto.GenerateKey` | 32 B | Discarded after derivation | Ensures distinct `ObjectKey` values under the same parent key |
+| **content-encryption DEK / CEK** | **MinIO `ObjectKey`; RustFS object-key mode `key_bytes`** | **32 B** | **In-memory only** | **The key that actually encrypts object content; derives part keys for multipart** |
+| key-wrapping IV | MinIO `SealedKey.IV` / `MetaIV`; RustFS `ManagedSealedKey.iv` | 32 B | Persisted | Contributes to deriving the per-object KEK |
+| **per-object KEK** | **MinIO local variable `sealingKey`; RustFS `sealing_key`** | **32 B** | **Derived transiently, never persisted** | **Wraps and unwraps `ObjectKey`** |
+| wrapped ObjectKey / wrapped content DEK | MinIO `SealedKey.Key`; Base64-decoded value of the corresponding metadata field | 64 B | Persisted | DARE package of `ObjectKey` AEAD-encrypted under the per-object KEK; new writes are DARE 2.0, legacy SSE-C may be DARE 1.0 |
+| `SealedKey` logical record | MinIO `SealedKey { Key, IV, Algorithm }`; RustFS `ManagedSealedKey` plus algorithm constant | 64 B + 32 B + algorithm name | Persisted as multiple metadata fields | The complete ObjectKey wrapping record; NOT an object content encryption key |
+| DARE stream nonce | Nonce material in the SIO/RIO DARE header | 12 B | Carried in every package header | Combined with the package sequence number to form the AEAD nonce; independent of the key-wrapping IV |
+| multipart part key | MinIO `DerivePartKey`; RustFS `derive_part_key` | 32 B | In-memory only | Content-encryption DEK for each part |
+
+### 3.1 Ambiguities That MUST Be Avoided
+
+1. `SealedKey` is NOT "the key that participates in object content
+   encryption"; it is the logical record of a wrapped `ObjectKey` and its
+   unwrapping parameters.
+2. `sealingKey` IS the per-object KEK; it is not an additional key layer
+   beyond the KEK.
+3. KMS calls its generation output a data key, but in MinIO's object
+   encryption layer that plaintext data key serves as the parent/external
+   key; the key that actually encrypts object content is the subsequently
+   derived `ObjectKey`.
+4. `SealedKey.IV` is a 32 B key-wrapping IV; the DARE data stream nonce is
+   12 B. The two MUST NOT be reused, stored interchangeably, or derived from
+   each other.
+
+## 4. Overall Key Hierarchy
+
+```mermaid
+flowchart TB
+    subgraph ROOT["Upstream Key Sources"]
+        KMSMK["KMS master key<br/>root KEK; KMS-internal only"]
+        KMSGEN["KMS GenerateKey<br/>carrying KMS AssociatedData"]
+        KMSPT["KMS data key plaintext<br/>32 B; parent/external key; in-memory only"]
+        KMSCT["KMS data key ciphertext<br/>KMS-wrapped; persisted"]
+        SSES3LOCAL["Local SSE-S3 master key<br/>32 B; server-managed parent key"]
+        SSECK["SSE-C customer key<br/>32 B; parent/external key; MUST NOT be persisted"]
+        KMSMK --> KMSGEN
+        KMSGEN --> KMSPT
+        KMSGEN --> KMSCT
+    end
+
+    subgraph SSE["SSE Layer: Key Hierarchy and Metadata Semantics"]
+        PARENT["parent/external key"]
+        OGN["object-key generation nonce<br/>random 32 B; discarded after derivation"]
+        OKEY["ObjectKey<br/>content-encryption DEK / CEK<br/>32 B; in-memory only"]
+        WIV["key-wrapping IV<br/>random 32 B; persisted"]
+        BIND["domain + algorithm + bucket/object"]
+        KEK["sealingKey<br/>per-object KEK<br/>32 B; never persisted"]
+        SEALED["SealedKey.Key<br/>wrapped ObjectKey<br/>64 B DARE 2.0 package; persisted"]
+
+        PARENT -->|"HMAC-SHA256"| OKEY
+        OGN -->|"context ∥ nonce"| OKEY
+        PARENT -->|"HMAC-SHA256"| KEK
+        WIV --> KEK
+        BIND --> KEK
+        KEK -->|"AEAD key"| SEALED
+        OKEY -->|"32 B plaintext"| SEALED
+    end
+
+    subgraph RIO["SIO / RIO Layer: Object Data Stream"]
+        PLAIN["Object plaintext"]
+        PART["multipart only<br/>partKey = HMAC-SHA256(ObjectKey, LE32(partNumber))"]
+        DARE["DARE 2.0 AEAD packages<br/>AES-256-GCM / ChaCha20-Poly1305"]
+        CIPHER["Object ciphertext stream"]
+        OKEY -->|"single-part key"| DARE
+        OKEY --> PART
+        PART -->|"multipart part key"| DARE
+        PLAIN --> DARE
+        DARE --> CIPHER
+    end
+
+    KMSPT --> PARENT
+    SSES3LOCAL --> PARENT
+    SSECK --> PARENT
+
+    subgraph STORE["Persistence Boundary"]
+        META["Object SSE metadata<br/>wrapped ObjectKey + wrapping IV + algorithm<br/>+ KMS key ID/ciphertext/context"]
+        BODY["Object data region<br/>DARE 2.0 ciphertext stream"]
+        FORBID["Forbidden to persist or log<br/>KMS data key plaintext<br/>SSE-C customer key<br/>ObjectKey / part key<br/>sealingKey"]
+    end
+
+    KMSCT --> META
+    WIV --> META
+    SEALED --> META
+    CIPHER --> BODY
+```
+
+The minimal equivalence relationships are:
+
+```text
+ObjectKey  = content-encryption DEK / CEK
+sealingKey = per-object KEK
+SealedKey.Key = Wrap(sealingKey, ObjectKey) = wrapped content DEK
+```
+
+## 5. Precise Key Transformations
+
+The concatenation operator `||` below denotes byte-string concatenation, and
+integers use little-endian encoding.
+
+### 5.1 Origin of the Parent/External Key
+
+| SSE Mode / Provider | Parent/External Key | Upstream Action |
+|---|---|---|
+| SSE-C | 32 B customer key, client-provided and verified | No KMS call; customer key MUST NOT be persisted |
+| SSE-S3 local/K/V | 32 B local SSE-S3 master key | No KMS call; the local parent key seals the per-object `ObjectKey` |
+| SSE-S3 KMS compatibility | KMS data key plaintext generated by KMS/provider | Calls `GenerateKey` with the default service-side key ID and object context |
+| SSE-KMS | KMS data key plaintext generated by KMS | Calls `GenerateKey` with the requested or default KMS key ID and KMS encryption context |
+
+Provider choice and object write format are independent. RustFS SSE-S3 writes
+MUST use the local SSE-S3 provider even when the MinIO ObjectKey write format
+is enabled; configuring KMS MUST NOT change SSE-S3 write routing. SSE-KMS
+writes MUST use KMS and MUST NOT fall back to the local SSE-S3 provider.
+
+KMS-backed SSE-S3 compatibility objects and SSE-KMS objects MUST preserve
+both the KMS key ID and KMS data key ciphertext so that the read path can
+recover the same parent/external key via KMS. Local/K/V SSE-S3 ObjectKey
+objects omit both fields simultaneously and seal `ObjectKey` directly under
+the local SSE-S3 parent key. Having only one KMS field present, or either
+present field empty, is corrupt metadata and MUST be rejected.
+
+KMS AssociatedData MUST exactly reproduce MinIO:
+
+```text
+objectPath = path.Join(bucket, object)
+
+SSE-S3 KMS compatibility:
+    kmsContext = { bucket: objectPath }
+
+SSE-KMS:
+    storedContext = exact copy of the client-provided context
+    kmsContext    = copy(storedContext)
+    if bucket is absent from kmsContext:
+        kmsContext[bucket] = objectPath
+```
+
+For SSE-KMS, if the client has explicitly provided an entry whose key equals
+the current bucket name, MinIO preserves that value and does not overwrite
+it; reads MUST reconstruct the same rule using the persisted `storedContext`.
+The augmented `kmsContext` used for the KMS call MUST NOT overwrite the
+client context that SHALL be persisted as-is. Copy, rename, and rewrap MUST
+reproduce the same context rules for the corresponding source/target
+operations and MUST NOT transparently forward a temporary map that lacks the
+required object binding.
+
+### 5.2 ObjectKey: Object Content DEK
+
+MinIO's computation is:
+
+```text
+objectKeyGenerationNonce = CSPRNG(32)
+
+ObjectKey = HMAC-SHA256(
+    key  = parentExternalKey,
+    data = UTF8("object-encryption-key generation")
+           || objectKeyGenerationNonce
+)
+```
+
+Constraints:
+
+- `ObjectKey` MUST be 32 B.
+- `objectKeyGenerationNonce` is used for exactly one derivation and is not
+  persisted.
+- `ObjectKey` does not include bucket/object; path binding occurs in the
+  per-object KEK derivation.
+- `ObjectKey` MUST NOT appear in object metadata, logs, error text, or debug
+  output.
+
+### 5.3 sealingKey: Per-Object KEK
+
+```text
+keyWrappingIV = CSPRNG(32)
+
+sealingKey = HMAC-SHA256(
+    key  = parentExternalKey,
+    data = keyWrappingIV
+           || UTF8(domain)
+           || UTF8("DAREv2-HMAC-SHA256")
+           || UTF8(canonicalBucketObjectPath)
+)
+```
+
+`domain` MUST exactly match the SSE mode:
+
+| SSE Mode | domain |
+|---|---|
+| SSE-C | `SSE-C` |
+| SSE-S3 | `SSE-S3` |
+| SSE-KMS | `SSE-KMS` |
+
+`canonicalBucketObjectPath` MUST be consistent with MinIO's
+`path.Join(bucket, object)` semantics. Any change to the path, domain,
+algorithm identifier, or IV will cause unwrapping authentication failure.
+Therefore, copying or renaming an object cannot simply carry over the old
+`SealedKey`; it MUST re-seal against the target path, or perform a
+semantically equivalent key rotation.
+
+This canonicalization is part of the MinIO sealed-key format and SHALL only
+be reproduced exactly within the sealing compatibility boundary; general
+path-cleaning functions that resolve `.` or `..` MUST NOT be applied to
+ordinary S3 object key paths based on this.
+
+For KMS-backed SSE-S3 compatibility objects and SSE-KMS, a target path change
+may also change the KMS AssociatedData. In that case, merely re-sealing
+`ObjectKey` while keeping the old KMS data key ciphertext is insufficient;
+the parent data key MUST be generated or rewrapped under the target effective
+KMS context, and that parent key MUST then be used to produce a new
+`SealedKey` for the target path. Only when the client's explicit SSE-KMS
+context makes the source and target effective AssociatedData identical may
+the corresponding KMS envelope be reused after KMS semantics have been
+verified. MinIO's normal key rotation path generates a new KMS data key and
+then re-seals `ObjectKey`.
+
+### 5.4 SealedKey: Wrapped ObjectKey
+
+```text
+SealedKey.Key = DAREv2-AEAD-Encrypt(
+    key       = sealingKey,
+    plaintext = ObjectKey,
+    aad       = DARE header[0:4],
+    nonce     = DARE package nonce
+)
+```
+
+The 32 B `ObjectKey` is encoded as a single DARE 2.0 package:
+
+```text
+16 B DARE header
+ 32 B ciphertext
+ 16 B AEAD tag
+= 64 B SealedKey.Key
+```
+
+The MinIO writer may write AES-256-GCM or ChaCha20-Poly1305 depending on
+`sio` cipher-suite selection. Compatible readers MUST accept both valid
+cipher IDs:
+
+| Cipher Suite | DARE Cipher ID |
+|---|---:|
+| AES-256-GCM | `0x00` |
+| ChaCha20-Poly1305 | `0x01` |
+
+The complete logical record also includes:
+
+```text
+SealedKey {
+    Key:       [u8; 64], // wrapped ObjectKey package
+    IV:        [u8; 32], // keyWrappingIV
+    Algorithm: "DAREv2-HMAC-SHA256"
+}
+```
+
+The three components are persisted separately in object metadata, not
+serialized as a single structure.
+
+### 5.5 Legacy MinIO SSE-C Seal
+
+The MinIO reader also accepts the historical SSE-C algorithm `DARE-SHA256`:
+
+```text
+legacySealingKey = SHA256(parentExternalKey || keyWrappingIV)
+ObjectKey        = sio.Decrypt(
+    key        = legacySealingKey,
+    minVersion = DARE 1.0,
+    input      = SealedKey.Key
+)
+```
+
+This format has no domain or bucket/object binding and is permitted only as
+a read-only compatibility path for historical objects:
+
+- MinIO sets `MinVersion = DARE 1.0`, so this algorithm branch may read
+  DARE 1.0 or DARE 2.0 packages produced with the legacy key derivation; it
+  MUST NOT be implemented as "header must be v1";
+- New writes, copy targets, and rewraps MUST NOT produce `DARE-SHA256`;
+- SSE-S3 and SSE-KMS MUST NOT accept this algorithm;
+- Legacy identification MUST be conditioned on the SSE-C metadata shape and
+  MUST NOT become a downgrade channel for arbitrarily corrupted v2 metadata;
+- After a successful read, if a rewrite occurs, the object SHOULD be
+  upgraded to the current `DAREv2-HMAC-SHA256` format.
+
+## 6. DARE 2.0 Format for Object Data
+
+The SSE layer hands the plaintext `ObjectKey` to SIO/RIO. SIO/RIO is not
+responsible for:
+
+- Calling KMS;
+- Parsing SSE-C request headers;
+- Generating or unwrapping `SealedKey`;
+- Interpreting SSE-S3, SSE-KMS, bucket/object, or KMS context;
+- Deciding SSE metadata fields.
+
+SIO/RIO is only responsible for encrypting and decrypting the DARE data
+stream using the supplied content DEK.
+
+### 6.1 Physical Format of Each Data Package
+
+```text
+DARE 2.0 package
+┌──────────────────┬──────────────────────┬──────────────────┐
+│ 16-byte header   │ 1..65536 B ciphertext│ 16-byte AEAD tag │
+└──────────────────┴──────────────────────┴──────────────────┘
+```
+
+Header:
+
+| Byte | Field | Meaning |
+|---:|---|---|
+| `0` | version | `0x20`, DARE 2.0 |
+| `1` | cipher suite | `0x00` AES-256-GCM; `0x01` ChaCha20-Poly1305 |
+| `2–3` (`[2:4]`) | payload length minus one | `LE16(plaintextLength - 1)` |
+| `4–15` (`[4:16]`) | stream nonce material | 12 B; the high bit also carries the final-package flag |
+
+A zero-length plaintext is a special valid case: its ciphertext stream is
+also zero bytes and produces no DARE packages, hence no final-package flag.
+Only after at least one package has been observed and EOF is reached MUST
+the last package be required to carry the final flag; an empty stream MUST
+NOT be misclassified as truncated.
+
+For package sequence number `sequenceNumber`:
+
+```text
+packageNonce = header[4:16]
+packageNonce[8:12] ^= LE32(sequenceNumber)
+
+ciphertext || tag = AEAD-Encrypt(
+    key       = ObjectKey or partKey,
+    nonce     = packageNonce,
+    plaintext = packagePlaintext,
+    aad       = header[0:4]
+)
+```
+
+Constraints:
+
+- The first package of a given DARE stream has sequence number `0`,
+  monotonically increasing thereafter.
+- The final-package flag MUST be covered by AEAD authentication.
+- The reader MUST verify the complete AEAD tag before releasing that
+  package's plaintext to the upper layer.
+- The header nonce is the DARE stream nonce, NOT `SealedKey.IV`.
+- Maximum plaintext per package is 64 KiB; each package adds a fixed 32 B
+  framing overhead.
+
+### 6.2 Multipart Part Key
+
+Multipart objects do not directly use `ObjectKey` to encrypt all parts.
+Each part first derives:
+
+```text
+partKey = HMAC-SHA256(
+    key  = ObjectKey,
+    data = LE32(partNumber)
+)
+```
+
+Each part is then an independent DARE stream:
+
+- Uses the corresponding `partKey`;
+- The MinIO writer uses
+  `SHA256(fmt.Append(nil, uploadID, partNumber))[0:12]` as the stream
+  nonce; here `fmt.Append` is Go textual concatenation of upload ID and part
+  number;
+- Sequence number restarts from `0`;
+- Part boundaries MUST be accurately recovered from the actual/encrypted
+  part sizes in metadata.
+
+Decimal strings, network byte order, or zero-based part indices MUST NOT
+replace `LE32(partNumber)`.
+
+The part number in the partKey derivation is `LE32`, but the part number in
+the MinIO stream nonce input is the Go textual representation; the two
+encodings differ. The DARE header carries its own nonce, so readers verify
+against the header for interoperability. MinIO's deterministic nonce is a
+recognized historical write behavior, not a requirement for new RustFS
+writes: re-uploading different plaintext under the same upload ID and part
+number would reuse the same AEAD key/nonce with that same partKey. New
+RustFS writes MUST generate a random, non-repeating stream nonce for each
+part write; this is DARE-semantically interoperable with the MinIO reader
+but does not target byte-level identical output with the MinIO writer.
+
+### 6.3 Derivation Keys for ETag and Other Internal Object Metadata
+
+In addition to encrypting object content, `ObjectKey` also serves as the
+root for several internal metadata sub-keys. This logic belongs at the
+SSE/object-metadata boundary, not in RIO's `SealedKey` management.
+
+MinIO's derivation for a non-empty ETag:
+
+```text
+etagKey    = HMAC-SHA256(ObjectKey, UTF8("SSE-etag"))
+sealedETag = DARE-Encrypt(etagKey, etag)
+```
+
+A typical 16 B MD5 ETag becomes a `16 B header + 16 B ciphertext + 16 B tag
+= 48 B` sealed ETag. An empty ETag remains empty; on read, a 16 B backend
+ETag is treated as an unsealed plain ETag and no decryption is performed.
+The ETag visible to S3 clients MUST be correctly unsealed/formatted and MUST
+NOT directly expose backend sealed bytes.
+
+MinIO's general-purpose internal metadata encrypter uses:
+
+```text
+metadataKey = HMAC-SHA256(ObjectKey, UTF8(baseKey))
+sealedValue = DARE-Encrypt(metadataKey, value)
+```
+
+For example, the compression index uses an independent `baseKey`. Each
+`baseKey` is a key-domain label and MUST exactly match MinIO; empty values
+remain empty. Implementations MUST NOT directly reuse `ObjectKey` with the
+same AEAD nonce to encrypt these metadata values.
+
+## 7. MinIO Internal Metadata Persistence Contract
+
+These fields are internal object metadata; not all of them should be
+returned to S3 clients. Binary values are standard Base64 encoded.
+
+RustFS writes of internal object metadata MUST additionally observe the
+repository's twin-key rule: write the same semantic value under both
+`x-rustfs-internal-<suffix>` and `x-minio-internal-<suffix>`, and on read
+prefer the RustFS key while remaining compatible with objects that have only
+the MinIO key. The table below lists the MinIO keys required for MinIO
+interoperability; the RustFS twin key does not replace them. Write, delete,
+and rotate operations MUST handle both keys simultaneously to prevent stale
+values from "resurrecting" on the next read.
+
+### 7.1 Common Fields
+
+| MinIO Metadata Key | Value | Required When |
+|---|---|---|
+| `X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm` | `DAREv2-HMAC-SHA256` | MinIO-style sealed ObjectKey |
+| `X-Minio-Internal-Server-Side-Encryption-Iv` | Base64(32 B key-wrapping IV) | MinIO-style sealed ObjectKey |
+| `X-Minio-Internal-Encrypted-Multipart` | multipart marker | Encrypted multipart object |
+
+### 7.2 Per-SSE-Mode Fields
+
+| Mode | Wrapped ObjectKey Field | KMS Key ID | KMS Data Key Ciphertext | KMS Context |
+|---|---|---|---|---|
+| SSE-C | `X-Minio-Internal-Server-Side-Encryption-Sealed-Key` | Not stored | Not stored | Not stored |
+| SSE-S3 local/K/V | `X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key` | Not stored | Not stored | Not stored |
+| SSE-S3 KMS compatibility | `X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key` | `X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id` | `X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key` | Typically no client context |
+| SSE-KMS | `X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key` | `X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id` | `X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key` | `X-Minio-Internal-Server-Side-Encryption-Context`, Base64(JSON) |
+
+Note two easily confused fields:
+
+```text
+...-Kms-Sealed-Key
+    = wrapped ObjectKey
+    = content DEK wrapped under the per-object sealingKey
+
+...-S3-Kms-Sealed-Key
+    = KMS data key ciphertext
+    = parent/external key wrapped under the KMS master key
+```
+
+These are at different envelope layers and MUST NOT be interchanged.
+
+The KMS encryption context is a Base64-encoded JSON AssociatedData that
+provides only KMS authentication binding, not confidentiality. It appears in
+`xl.meta`, backups, and diagnostic data, and therefore MUST NOT contain
+tokens, passwords, plaintext keys, personally sensitive information, or
+other secrets; it MUST NOT be written to logs, audit events, notification
+events, or client responses.
+
+### 7.3 Materials That MUST NOT Be Persisted
+
+The following materials MUST NOT enter `xl.meta`, user metadata, bucket
+metadata, logs, audit events, notification events, error messages, or
+metrics labels:
+
+- KMS data key plaintext;
+- SSE-C customer key;
+- `ObjectKey`;
+- Multipart `partKey`;
+- `sealingKey`;
+- Any debug structure from which the above plaintext keys could be
+  recovered.
+
+Types that carry the above plaintext materials MUST additionally:
+
+- Use a secret wrapper and zeroize on drop in a way that the compiler cannot
+  optimize away;
+- Zeroize on error, cancellation, and early-return paths as well;
+- Not implement or derive `Debug`, `Display`, or `Serialize` in a way that
+  would expose their contents;
+- Avoid unnecessary `Clone` and MUST NOT copy bare `[u8; 32]` or `Vec<u8>`
+  across async tasks;
+- Treat core dumps, swap, telemetry samples, and crash diagnostics as
+  sensitive memory.
+
+## 8. Write and Read Sequence
+
+All three modes share the same main flow:
+
+```text
+Prepare parent/external key
+  -> Derive ObjectKey and sealingKey
+  -> Generate and persist the complete envelope metadata
+  -> Hand ObjectKey/partKey to RIO to produce DARE ciphertext
+  -> Atomically commit metadata and ciphertext in the same object generation
+  -> Wipe all plaintext keys
+```
+
+Mode differences are only in the parent key and persisted envelope:
+
+| Mode | Write Preparation | Additional Persistence | Read Recovery |
+|---|---|---|---|
+| SSE-C | Verify customer key and MD5 | No KMS fields | Client provides customer key again |
+| SSE-S3 local/K/V | Load local SSE-S3 parent key | No KMS fields | Local SSE-S3 provider supplies the parent key |
+| SSE-S3 KMS compatibility | Call KMS/provider with default service-side key ID | KMS key ID + KMS ciphertext | KMS unwraps parent key |
+| SSE-KMS | Request/default key ID, construct context per §5.1 | KMS key ID + ciphertext + original client context | Reconstruct effective context, then KMS unwraps |
+
+Reads execute the strict reverse: parse the complete envelope, recover the
+parent key, derive sealingKey, AEAD-unwrap `ObjectKey`, then RIO uses
+`ObjectKey`/partKey to authenticate each package and return plaintext. Any
+missing field, illegal Base64, wrong length, unsupported algorithm/cipher,
+path or KMS context mismatch, or AEAD tag failure MUST fail explicitly; MUST
+NOT fabricate success with a default key, zero key, truncated value, or
+legacy fallback.
+
+## 9. Layered Responsibility Contract
+
+### 9.1 S3 API / Application Layer
+
+Responsible for:
+
+- Parsing and validating public SSE request headers;
+- Deciding SSE-C, SSE-S3, or SSE-KMS;
+- Passing bucket, object, KMS key ID, and context;
+- Organizing copy, multipart, range, and error mapping;
+- Ensuring the encryption reader is actually in the final write-to-disk
+  chain.
+
+MUST NOT:
+
+- Implement key derivation on its own;
+- Expose internal SSE metadata as user metadata;
+- Rely solely on metadata claiming the object is encrypted without verifying
+  actual on-disk bytes.
+
+### 9.2 SSE / Key-Management Layer
+
+Solely responsible for:
+
+- Calling KMS/provider;
+- Generating and recovering parent/external keys;
+- Deriving `ObjectKey`;
+- Deriving per-object `sealingKey`;
+- Sealing/unsealing `ObjectKey`;
+- Generating, parsing, and validating SSE metadata;
+- Binding domain, algorithm, and bucket/object;
+- Managing the lifecycle of plaintext keys.
+
+### 9.3 SIO / RIO Layer
+
+Only responsible for:
+
+- Receiving `ObjectKey` or `partKey`;
+- Generating and managing DARE stream nonces;
+- Maintaining sequence numbers;
+- Producing and parsing DARE 2.0 packages;
+- Performing AEAD encryption/decryption and tag verification;
+- Supporting sequence/boundary positioning as required by parts and ranges.
+
+The RIO API SHALL NOT receive `SealedKey`, KMS key ID, KMS ciphertext, or
+KMS context.
+
+### 9.4 Storage / Erasure Layer
+
+Responsible for persisting:
+
+- The encrypted DARE data stream;
+- The internal SSE metadata, intact and complete;
+- Information needed for reads, such as part sizes and object actual size.
+
+The Storage layer MUST NOT have persistence semantics for plaintext keys,
+nor attempt to reconstruct `ObjectKey` on its own.
+
+## 10. RustFS Implementation Navigation
+
+The following anchors locate implementations and do not, by their mere
+presence, prove complete interoperability:
+
+| Specification Role | RustFS Implementation Anchor |
+|---|---|
+| SSE boundary | [rustfs/src/storage/sse.rs](../../rustfs/src/storage/sse.rs) |
+| ObjectKey derivation | `derive_object_key` |
+| Per-object KEK derivation | `derive_sealing_key` |
+| ObjectKey seal/unseal | `seal_object_key` / `unseal_object_key` |
+| Complete wrapping record | `ManagedSealedKey` |
+| SSE-to-RIO material | `EncryptionMaterial` / `DecryptionMaterial` |
+| Key semantic label | `EncryptionKeyKind::Object` |
+| DARE data stream | [crates/rio-v2/src/encrypt_reader.rs](../../crates/rio-v2/src/encrypt_reader.rs) |
+| Multipart part key | `derive_part_key` |
+| MinIO metadata/`xl.meta` fixtures | [crates/rio-v2/tests/minio_generated_fixtures.rs](../../crates/rio-v2/tests/minio_generated_fixtures.rs) |
+| MinIO end-to-end read fixtures | [crates/ecstore/tests/minio_generated_read_test.rs](../../crates/ecstore/tests/minio_generated_read_test.rs) |
+
+`EncryptionKeyKind::Direct` and the `x-rustfs-encryption-*` fields belong to
+the historical RustFS direct-stream key format. The compatibility principles
+are:
+
+1. Existing historical objects SHALL remain readable unless a verified
+   migration and rollback plan exists.
+2. MinIO-compatible object-key mode MUST follow the `ObjectKey` /
+   `SealedKey` hierarchy defined in this document.
+3. Legacy fallback MUST NOT swallow format corruption, wrong keys, or
+   authentication failures.
+4. New MinIO compatibility tests MUST use real MinIO generated fixtures, not
+   only RustFS self-write-self-read round trips.
+
+## 11. Compatibility and Security Invariants
+
+Any SSE/RIO modification MUST preserve the following invariants.
+
+### 11.1 Format Invariants
+
+- `ObjectKey`, parent/external key, sealingKey, and partKey are all 32 B.
+- Wrapping IV is 32 B; DARE stream nonce is 12 B.
+- Wrapped `ObjectKey` is fixed at 64 B; new writes are DARE 2.0 packages,
+  and the legacy SSE-C reader additionally accepts DARE 1.0 packages meeting
+  §5.5.
+- New writes' seal algorithm is exactly `DAREv2-HMAC-SHA256`.
+- Domain exactly matches the SSE mode.
+- Bucket/object canonicalization is consistent with MinIO `path.Join`
+  semantics.
+- DARE header, cipher ID, payload length, final flag, and AEAD tag MUST be
+  validated.
+- Zero-length plaintext corresponds to a zero-byte DARE stream; only
+  non-empty streams require the final package.
+- Multipart part key uses `HMAC-SHA256(ObjectKey, LE32(partNumber))`.
+- New writes only use `DAREv2-HMAC-SHA256`; legacy `DARE-SHA256` is only
+  permitted for the SSE-C reader under strict legacy format recognition.
+
+### 11.2 Security Invariants
+
+- All random values come from a CSPRNG.
+- The same key/nonce combination MUST NOT be reused.
+- Package plaintext MUST NOT be released before AEAD tag verification
+  completes.
+- Any plaintext key MUST NOT be persisted or logged.
+- Plaintext keys MUST be carried by a secret type that is non-printable,
+  non-serializable, and zeroizes on drop.
+- Corrupt metadata MUST fail closed.
+- Copy/rename/replication MUST NOT break bucket/object cryptographic binding.
+- SSE metadata SHALL NOT serve as sufficient evidence that "on-disk content
+  is encrypted"; tests MUST inspect actual stored bytes.
+- Internal SSE metadata MUST NOT be echoed as user-controllable or
+  client-visible metadata.
+
+### 11.3 MinIO Interoperability Invariants
+
+- The reader MUST accept AES-256-GCM and ChaCha20-Poly1305 sealed ObjectKeys
+  legitimately written by MinIO.
+- KMS-backed SSE-S3 compatibility objects and SSE-KMS objects MUST save and
+  parse the KMS data-key ciphertext and wrapped ObjectKey as separate layers;
+  local/K/V SSE-S3 objects have only the wrapped ObjectKey layer.
+- New KMS-backed writes MUST save KMS key ID/ciphertext as a pair. Only an
+  SSE-S3 sealed-key marker with both KMS fields absent identifies the local
+  K/V shape; an SSE-KMS sealed-key marker always requires the pair. If only
+  one field is present, the metadata is corrupt; if both fields are present,
+  the metadata belongs to the KMS-backed branch and empty values MUST NOT be
+  treated as absent.
+- Provider routing MUST NOT guess from the byte shape of the KMS ciphertext:
+  the historical K/V shape is identified by KMS key ID/ciphertext both
+  absent; when both fields are present they uniformly belong to the
+  KMS-backed branch, including when either value is empty and subsequently
+  rejected as invalid. MinIO static KMS modern binary ciphertext and
+  external KMS ciphertext are both opaque bytes and cannot be reliably
+  distinguished.
+- Only when the key ID in
+  `RUSTFS_MINIO_STATIC_KMS_KEY=<key-id>:<base64-32-byte-key>` exactly
+  matches the object's persisted key ID SHALL the reader unwrap locally
+  using the MinIO static KMS format; on mismatch it MUST delegate to the
+  external KMS. Opportunistic fallback is forbidden. This configuration
+  supports the modern binary AES-256-GCM envelope and the historical Base64
+  JSON `AES-256-GCM-HMAC-SHA-256` / `ChaCha20Poly1305` envelope.
+- The key ID in the static KMS configuration is the sole provider
+  declaration for that ID within the process and MUST NOT duplicate a key ID
+  from the external KMS. After an ID match, if ciphertext, AAD, or tag
+  verification fails, it MUST fail closed; falling back to the external KMS
+  is forbidden—otherwise a key error becomes a provider-probing oracle.
+  Release builds SHALL freeze the configuration on first use; changes
+  require a process restart. The same persisted object MUST NOT drift
+  providers across requests.
+- The static KMS root key MUST be exactly 32 bytes and MUST NOT be all
+  zeros.
+- `RUSTFS_MINIO_STATIC_KMS_KEY` contains a plaintext root key and SHALL only
+  be injected through a protected secret channel; it MUST NOT be written to
+  the repository, logs, errors, audit, or object metadata.
+- The Base64(JSON) representation and decryption context of the KMS context
+  MUST be consistent.
+- The KMS context has no confidentiality and MUST NOT contain secrets or be
+  echoed to clients.
+- Metadata key spelling and casing MUST be compatible with the MinIO
+  persistence format.
+- RustFS writes MUST preserve both the `x-rustfs-internal-*` and
+  `x-minio-internal-*` twin keys; the MinIO key MUST NOT be omitted because
+  the RustFS key is present.
+- Real MinIO single-part, multipart, SSE-C, SSE-S3, and SSE-KMS fixtures
+  MUST be parseable and readable by RustFS.
+- Real four-disk local-KMS SSE-KMS fixtures generated by a fixed RustFS
+  `1.0.0-beta.5` release MUST be readable through the current production
+  bitrot/erasure/GET/KMS decryption chain; tests MUST NOT replace these with
+  fixed-plaintext mocks that ignore the envelope, key ID, or context.
+
+### 11.4 Concurrency and Persistence Invariants
+
+- DARE ciphertext, SSE metadata, part metadata, and actual sizes MUST belong
+  to the same object generation and become visible within the same object
+  commit boundary.
+- "Encrypted" metadata MUST NOT be committed before the ciphertext data;
+  after crash recovery there MUST NOT be a state where metadata claims
+  encryption while the data region is still plaintext or from an old
+  generation.
+- KMS success and key derivation completion do not imply object commit
+  success; failure or cancellation paths MUST clean up temporary data and
+  release all plaintext keys together with the request state.
+- Copy, multipart complete, version writes, and retries MUST be idempotent:
+  a once-committed object SHALL only reference wrapped keys and part
+  boundaries that match its ciphertext.
+- Key rotation or metadata-only rewrap MUST atomically replace the complete
+  envelope while holding the corresponding object write/versioning
+  coordination boundary; readers MUST NOT observe a mix of new KMS
+  ciphertext with old `SealedKey`, or old KMS ciphertext with new
+  `SealedKey`.
+- Concurrent requests MUST NOT share mutable plaintext key buffers, nonce
+  state, or DARE sequence state; each object stream and each multipart part
+  MUST have independent state.
+- PutPart (especially same part number), ListParts, Complete, and Abort for
+  the same upload ID MUST be linearized by a per-upload coordination
+  boundary. Complete MUST pin a part snapshot; only one of Complete and
+  Abort may pass the final commit linearization point.
+- Multipart part data/metadata SHALL only be cleaned up on a best-effort
+  basis after the final generation durable commit; retries MUST be
+  reentrant and MUST NOT delete the last copy referenced by a committed
+  object.
+- Copy/rename/rewrap MUST pin the ciphertext, SSE envelope, and part
+  boundaries of the same source version/generation and commit under the
+  target write lock/commit fence; rewrap uses generation CAS. Cross-generation
+  splicing of envelope and ciphertext is forbidden.
+- SSE commits follow the quorum/atomic commit contract of
+  [erasure-coding.md](erasure-coding.md) and the generation/fencing contract
+  of [unified-object-generation.md](unified-object-generation.md): temporary
+  data and `xl.meta` are synced per the selected `DurabilityMode`, lock
+  loss/epoch is checked before the commit rename, the directory is synced
+  after rename per the durability gate, and the ACK is only sent after the
+  corresponding durable boundary is reached.
+- Cancellation and crash cleanup MUST distinguish pre-commit from
+  post-commit: pre-commit only rolls back this request's tmp generation;
+  post-commit MUST NEVER delete a visible generation. Old data and part
+  metadata cleanup is best-effort only, must be reentrant, and failure MUST
+  NOT turn a committed write into a reported failure.
+
+### 11.5 Performance Invariants
+
+- Each independent request/operation SHALL parse the same SSE metadata at
+  most once, recover the parent key at most once, and unwrap the same
+  `ObjectKey` at most once; within the same request, reuse across ranges,
+  parts, and shards. KMS/provider calls MUST NOT be pushed down to every
+  DARE package, every erasure shard, or every range block.
+- Multipart initiate is responsible for generating and persisting the
+  session envelope; subsequent independent UploadPart requests recover
+  material from that envelope. Plaintext parent keys/`ObjectKey` MUST NOT be
+  cached long-term across requests or across multipart sessions to reduce
+  KMS calls.
+- ObjectKey and sealingKey derivation is constant work per object; partKey
+  derivation is constant work per part.
+- RIO MUST maintain streaming encryption/decryption and MUST NOT buffer the
+  complete object for SSE wrapping.
+- Bulk paths such as ListObjects, when they must recover multiple
+  ObjectKey/ETag keys, SHOULD prefer bulk KMS decrypt; when the provider
+  lacks bulk capability, use bounded concurrency with an explicit batch
+  limit. Serial execution of N remote KMS RTTs is forbidden, as is
+  substituting long-term caching of plaintext keys for batching.
+- Performance optimizations MUST NOT cache or extend the lifetime of
+  plaintext keys, nor reduce work by skipping tag, nonce, final-package, or
+  metadata consistency validation.
+
+### 11.6 rio-v2 Write Format Enablement Gate
+
+Compiling `rio-v2` only means the process has the code capability to read
+and write the MinIO ObjectKey format; it does not mean every node on the
+shared storage has completed the upgrade. To ensure rolling upgrades and
+rollbacks:
+
+- When `RUSTFS_SSE_RIO_V2_WRITE_FORMAT` is unset, or set to
+  `legacy-direct`, the historical Direct format continues to be written; the
+  rio-v2 reader can still read both Direct and MinIO ObjectKey formats.
+- Only after confirming that all nodes that may read the storage pool are
+  running a version that supports the ObjectKey format may
+  `RUSTFS_SSE_RIO_V2_WRITE_FORMAT=minio-object-key` be uniformly set on all
+  write nodes.
+- Unknown configuration values MUST fail closed and MUST NOT silently fall
+  back or be independently enabled by some nodes.
+- `RUSTFS_MINIO_STATIC_KMS_KEY` only controls static KMS unwrapping for
+  compatible reads; it does not change the write format of new objects, nor
+  does it replace the formal KMS service required for SSE-KMS writes.
+- New SSE-S3 writes under `minio-object-key` MUST use the local SSE-S3
+  provider as the parent-key source and persist the historical K/V metadata
+  shape with both KMS fields absent. Enabling this write format MUST NOT make
+  SSE-S3 depend on a configured KMS service. RustFS private local envelopes
+  MUST NOT be placed in the MinIO KMS ciphertext field. The reader SHALL
+  remain compatible with KMS-backed historical SSE-S3 objects by routing
+  their complete persisted KMS pair to KMS.
+- Rolling back from `minio-object-key` to `legacy-direct` only affects
+  subsequent writes; existing ObjectKey objects still require the rio-v2
+  reader, so before rollback it MUST be guaranteed that old nodes will not
+  take over these objects, or a verified migration/rewrap MUST be completed
+  first.
+- The current gate is a deployment-level consistency contract, not automatic
+  peer capability negotiation. If the cluster capability epoch is integrated
+  later, automatic switching SHALL only occur after all live write nodes
+  have confirmed the same epoch; if any node's state is unknown,
+  `legacy-direct` MUST be retained.
+
+## 12. Required Compatibility Test Matrix
+
+Behavior implementations MUST NOT rely solely on encrypt/decrypt round trips
+within the same codebase. The following tests MUST exercise real
+S3/application/storage production call chains and assert on complete bodies,
+exact lengths, typed errors, or physical storage formats:
+
+| Test Category | What MUST Be Proven |
+|---|---|
+| Real MinIO single-part | SSE-C, SSE-S3, and SSE-KMS objects are all completely readable with exact content and length |
+| Real MinIO multipart | Part boundaries, partKey, per-part sequence reset, and complete object assembly are correct |
+| Cipher-suite compatibility | Sealed ObjectKey and object content DARE stream each cover AES-256-GCM/ChaCha20-Poly1305, and cover combinations where seal and content ciphers differ |
+| Metadata physical format | Required MinIO fields are present, Base64 decode lengths are correct, RustFS/MinIO twin keys are simultaneously written |
+| Actual on-disk ciphertext | Stored bytes contain no known plaintext, and "encrypted" is not judged solely from SSE metadata |
+| Path binding | The same wrapped ObjectKey fails to unwrap under a different bucket/object; copy to the target path succeeds |
+| Path edge values | Leading/trailing slashes and legal test keys containing `.` and `..` segments match MinIO `path.Join` results |
+| Range | First/last byte, 64 KiB−1/boundary/+1, exact slices across DARE packages and across multipart parts, Content-Length/Content-Range correct |
+| Copy | Same/cross-bucket CopyObject and UploadPartCopy cover representative source/target SSE-C/S3/KMS combinations, pinned source version, target envelope, and path rebinding |
+| Metadata corruption | Missing fields, illegal Base64, wrong length, unknown algorithm/cipher all fail explicitly |
+| KMS pair consistency | Both key ID/ciphertext present succeeds; both absent routes to the recognized legacy provider; only one absent returns a typed error |
+| Ciphertext corruption and truncation | Corruption of header, payload, tag, or final package never returns unauthenticated or truncated plaintext |
+| Zero-length objects | Plaintext and ciphertext are both zero bytes; not misclassified as truncated due to missing final package |
+| Multipart boundaries | Part number 1, maximum supported part number, empty/minimum part, and cross-64 KiB package boundaries |
+| KMS context | Context absent, explicitly provided, field order variations, and mismatch scenarios match MinIO behavior |
+| ETag/internal metadata | Empty, 16 B, and 48 B paths for sealed ETag, plus at least one metadata value with a distinct `baseKey` domain, match MinIO |
+| Legacy MinIO SSE-C | `DARE-SHA256` historical objects readable, but new writes and corrupted v2 inputs MUST NOT enter the legacy path |
+| Legacy RustFS | Recognizable old objects remain readable; corrupted MinIO format MUST NOT mistakenly enter legacy fallback |
+| Crash/cancellation | Metadata and ciphertext do not mix across generations; temporary files and plaintext material do not leak |
+| Secret non-leakage | Unique sentinel parent/ObjectKey/customer key does not appear in committed metadata, raw payload, tmp/rollback, error/log/audit/notification; secret wrapper drop observably zeroizes |
+| KMS/parse/KDF call counts | Fake KMS, decoder, and KDF counters prove one GET/range recovers the same ObjectKey only once; package/shard count does not increase ObjectKey/sealingKey derivations; multipart range derives partKey at most once per touched part; bulk paths do not serially execute N remote KMS calls |
+| Streaming memory | Constrained reader/peak memory tests prove buffering is O(package/block), not O(object/part) |
+
+Real MinIO fixture tests MUST parse MinIO-generated persisted objects and
+MUST NOT use RustFS writer on-the-fly generation fed to the RustFS reader.
+The fixture corpus MUST be sanitized, pinned to a MinIO release/commit and
+content hash, and included in the default CI gate. Test functions may use
+`#[ignore]` to isolate expensive fixture generation, but MUST be explicitly
+executed via `ignored-only` by a dedicated required workflow covered by the
+relevant source path filter; they MUST NOT rely on developers running them
+manually locally. When the corresponding compatibility branch is removed,
+that gate MUST fail. Until that condition is met, the relevant mode MUST NOT
+be marked as MinIO-interoperable.
+
+Write compatibility MUST be inversely proven by the target MinIO release
+actually HEAD/GET-ing RustFS-generated single-part, multipart, SSE-C,
+SSE-S3, and SSE-KMS objects, or by using cross-implementation fixtures that
+have been verified and frozen by the MinIO reader. Self-write-self-read and
+expected bytes encoded by RustFS itself cannot substitute for this proof.
+
+The test oracle MUST additionally satisfy:
+
+- Twin-key tests SHALL assert identical values for both the RustFS and MinIO
+  prefixes for every SSE suffix in the final committed `xl.meta`, and cover
+  write, remove, and rotate; a write or delete that omits either prefix MUST
+  fail the test.
+- "Actual on-disk ciphertext" tests SHALL, after a real PUT write, read the
+  storage payload bypassing SSE decode, and verify it is valid DARE framing,
+  differs from the plaintext, and can be decrypted by the expected
+  `ObjectKey`; merely searching physical shards for absence of contiguous
+  plaintext does not constitute a proof.
+- Fake KMS MUST record exact AssociatedData, covering bucket entry absent,
+  preexisting, stored/effective context separation, JSON map ordering,
+  mismatch, and same/cross-bucket copy/rename/rewrap; changing
+  insert-if-absent or persisting the effective map MUST fail the test.
+- The crash harness SHALL inject failures after data tmp write, after
+  `xl.meta` tmp write, before/after commit rename, and between the two
+  envelope replacements during rewrap; after restart, only a complete old or
+  complete new generation may be observed. Removing generation commit,
+  lock-loss fence, or CAS MUST fail the test.
+- Corruption/truncation tests SHALL consume package by package and assert
+  specific error categories; first-package authentication failure returns
+  zero plaintext, subsequent package failures return at most the previously
+  fully authenticated packages, and MUST NEVER return `Ok(wrong plaintext)`,
+  silent EOF, or 200 + truncated body.
+- Range tests MUST assert exact byte slices and response range headers;
+  corruption of a touched package MUST error, and corruption of an untouched
+  package MUST NOT pollute a valid range.
+- CopyObject and UploadPartCopy MUST fail variants that "directly carry over
+  old SealedKey/partKey", "use wrong source/target KMS context", and "do not
+  pin source version".
+- Zero-length, AES/ChaCha sealed key, AES/ChaCha content stream, legacy
+  SSE-C, and multipart boundaries MUST each have at least one end-to-end
+  production-path test and MUST NOT be proven only by RIO synthetic unit
+  tests.
+- KMS pair tests MUST distinguish both-present, both-absent, key-ID-only,
+  and ciphertext-only, and MUST NOT reject legitimate legacy shapes with a
+  generic "missing field failure" assertion.
+- Secret sentinel tests MUST cover success, KMS error, cancellation, and
+  crash paths, and capture storage/tmp, error, log, audit, and notification
+  output; zeroize additionally has an observable unit test.
+- Fake KMS, metadata decoder, and KDF counters MUST be attached to
+  production GetObject/range/list/multipart paths; reversing in-request
+  reuse, moving decrypt/KDF into the package/shard loop, or turning bulk KMS
+  into serial calls MUST fail the test.
+
+## 13. One-Sentence Architecture Conclusion
+
+> The SSE layer owns the complete key hierarchy: it selects the local SSE-S3
+> parent key, KMS data key plaintext, or SSE-C customer key according to the
+> SSE mode independently of the selected write format; derives `ObjectKey`
+> and the per-object `sealingKey`; and persists the wrapped `ObjectKey`. RIO
+> only receives the in-memory `ObjectKey` or partKey, performs DARE 2.0 data
+> stream encryption/decryption, and never interprets or creates a
+> `SealedKey`.
@@ -19,19 +19,24 @@
 |---|---|---|
 | admin_auth_test | 3 | ✅ |
 | admin_iam_crud_test | 2 | ✅ |
+| admin_pools_test | 1 | ✅ |
 | admin_timeout_regression_test | 1 |  |
 | anonymous_access_test | 3 | ✅ |
+| api_rate_limit_test | 3 |  |
 | archive_download_integrity_test | 13 |  |
 | bucket_logging_test | 3 |  |
 | bucket_policy_check_test | 1 | ✅ |
 | checksum_upload_test | 7 |  |
 | cluster_concurrency_test | 2 |  |
-| common | 3 |  |
+| cluster_multidrive_pool_test | 2 |  |
+| common | 10 |  |
 | compression_test | 1 |  |
+| connection_cap_test | 2 |  |
 | console_smoke_test | 1 | ✅ |
 | content_encoding_test | 3 | ✅ |
+| copy_object_checksum_test | 3 |  |
 | copy_object_metadata_test | 1 | ✅ |
-| copy_object_version_restore_test | 1 |  |
+| copy_object_version_restore_test | 2 |  |
 | copy_source_invalid_date_test | 1 | ✅ |
 | create_bucket_region_test | 2 | ✅ |
 | degraded_read_eof_regression_test | 3 |  |
@@ -40,6 +45,7 @@
 | delete_objects_versioning_test | 2 | ✅ |
 | existing_object_tag_policy_test | 4 |  |
 | fake_s3_target | 4 | ✅ |
+| fault_proxy | 7 |  |
 | get_codec_streaming_compat_test | 1 |  |
 | head_object_consistency_test | 1 | ✅ |
 | head_object_range_test | 1 | ✅ |
@@ -56,13 +62,13 @@
 | multipart_auth_test | 109 |  |
 | namespace_lock_quorum_test | 2 |  |
 | negative_sigv4_test | 6 | ✅ |
-| notification_webhook_test | 1 | ✅ |
+| notification_webhook_test | 2 | ✅ |
 | object_lambda_test | 16 |  |
 | object_lock | 33 |  |
 | overwrite_cleanup_regression_test | 1 |  |
 | presigned_negative_test | 7 | ✅ |
 | protocols | 16 |  |
-| quota_test | 13 |  |
+| quota_test | 14 |  |
 | reliability_disk_fault_test | 3 |  |
 | reliant | 10 | 4 ✅ |
 | replication_extension_test | 47 | 20 ✅ +27 🌙 |
@@ -75,4 +81,6 @@
 | tls_hot_reload_test | 1 | ✅ |
 | version_id_regression_test | 10 | ✅ |

-**Total listed: 439 tests across 57 modules · PR smoke subset: 112 tests / 27 modules** (25 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-16.
+`notification_webhook_test` also has 1 ignored store-and-forward regression tracked by rustfs#4852; ignored tests are excluded from the active counts above.
+
+**Total listed: 467 tests across 63 modules · PR smoke subset: 114 tests / 28 modules** (26 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-21.
@@ -181,6 +181,7 @@ libc = { workspace = true }
 rand = { workspace = true, features = ["serde"] }
 aes-gcm = { workspace = true, features = ["rand_core"] }
 chacha20poly1305 = { workspace = true }
+zeroize.workspace = true

 # Observability and Metrics
 metrics = { workspace = true }
@@ -16,8 +16,8 @@ use crate::admin::{
    auth::validate_admin_request,
    handlers::audit_runtime_config::{load_server_config_from_store, update_audit_config_and_reload},
    handlers::target_descriptor::{
-        AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, build_enabled_target_kvs,
-        build_json_response, collect_runtime_statuses, extract_supported_target_params,
+        AdminTargetSpec, EndpointKey, RuntimeHealthStatus, TargetEndpointSource, admin_target_spec_from_builtin,
+        build_enabled_target_kvs, build_json_response, collect_runtime_statuses, extract_supported_target_params,
        merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
        target_mutation_block_reason as shared_target_mutation_block_reason,
    },
@@ -192,6 +192,8 @@ struct AuditEndpoint {
    account_id: String,
    service: String,
    status: String,
+    health_state: String,
+    health_reason: String,
    source: TargetEndpointSource,
 }

@@ -221,7 +223,7 @@ async fn authorize_audit_admin_request(req: &S3Request<Body>, action: AdminActio
    validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await
 }

-fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
+fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> S3Result<Option<String>> {
    shared_target_mutation_block_reason(
        audit_target_specs(),
        AUDIT_ROUTE_PREFIX,
@@ -248,16 +250,23 @@ async fn audit_target_operation_block_reason(action: &str) -> Option<String> {
    target_module_disabled_reason("audit", rustfs_config::ENV_AUDIT_ENABLE, is_audit_module_enabled(), action)
 }

-fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<AuditEndpoint> {
-    shared_merge_target_endpoints(audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses)
-        .into_iter()
-        .map(|endpoint| AuditEndpoint {
-            account_id: endpoint.account_id,
-            service: endpoint.service,
-            status: endpoint.status,
-            source: endpoint.source,
-        })
-        .collect()
+fn merge_audit_endpoints(
+    config: &Config,
+    runtime_statuses: HashMap<EndpointKey, RuntimeHealthStatus>,
+) -> S3Result<Vec<AuditEndpoint>> {
+    Ok(
+        shared_merge_target_endpoints(audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses)?
+            .into_iter()
+            .map(|endpoint| AuditEndpoint {
+                account_id: endpoint.account_id,
+                service: endpoint.service,
+                status: endpoint.status,
+                health_state: endpoint.health_state,
+                health_reason: endpoint.health_reason,
+                source: endpoint.source,
+            })
+            .collect(),
+    )
 }

 fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &'a str)> {
@@ -279,7 +288,7 @@ impl Operation for AuditTargetConfig {
            return Err(s3_error!(InvalidRequest, "{reason}"));
        }
        let config_snapshot = load_server_config_from_store().await?;
-        if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name) {
+        if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name)? {
            log_audit_target_operation_blocked!("set_audit_target_config", Some(target_type), Some(target_name), &reason);
            return Err(s3_error!(InvalidRequest, "{reason}"));
        }
@@ -305,12 +314,14 @@ impl Operation for AuditTargetConfig {
        )
        .await?;

-        update_audit_config_and_reload(audit_target_specs(), |config| {
+        let mutation_target_type = target_type.to_lowercase();
+        let mutation_target_name = target_name.to_lowercase();
+        update_audit_config_and_reload(audit_target_specs(), move |config| {
            config
                .0
-                .entry(target_type.to_lowercase())
+                .entry(mutation_target_type.clone())
                .or_default()
-                .insert(target_name.to_lowercase(), kvs.clone());
+                .insert(mutation_target_name.clone(), kvs.clone());
            true
        })
        .await
@@ -344,7 +355,7 @@ impl Operation for ListAuditTargets {
        }

        let config = load_server_config_from_store().await?;
-        let audit_endpoints = merge_audit_endpoints(&config, runtime_statuses);
+        let audit_endpoints = merge_audit_endpoints(&config, runtime_statuses)?;
        let data = serde_json::to_vec(&AuditEndpointsResponse { audit_endpoints }).map_err(|e| {
            log_audit_target_request_failed!("list_audit_targets", "serialize_audit_targets_failed", None, None, e);
            s3_error!(InternalError, "failed to serialize audit targets: {}", e)
@@ -369,19 +380,21 @@ impl Operation for RemoveAuditTarget {
            return Err(s3_error!(InvalidRequest, "{reason}"));
        }
        let config_snapshot = load_server_config_from_store().await?;
-        if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name) {
+        if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name)? {
            log_audit_target_operation_blocked!("remove_audit_target_config", Some(target_type), Some(target_name), &reason);
            return Err(s3_error!(InvalidRequest, "{reason}"));
        }

-        update_audit_config_and_reload(audit_target_specs(), |config| {
+        let mutation_target_type = target_type.to_lowercase();
+        let mutation_target_name = target_name.to_lowercase();
+        update_audit_config_and_reload(audit_target_specs(), move |config| {
            let mut changed = false;
-            if let Some(targets) = config.0.get_mut(&target_type.to_lowercase()) {
-                if targets.remove(&target_name.to_lowercase()).is_some() {
+            if let Some(targets) = config.0.get_mut(&mutation_target_type) {
+                if targets.remove(&mutation_target_name).is_some() {
                    changed = true;
                }
                if targets.is_empty() {
-                    config.0.remove(&target_type.to_lowercase());
+                    config.0.remove(&mutation_target_type);
                }
            }
            changed
@@ -422,6 +435,14 @@ mod tests {
        }])
    }

+    fn online_health() -> RuntimeHealthStatus {
+        RuntimeHealthStatus {
+            status: "online".to_string(),
+            state: "online".to_string(),
+            reason: "reachable".to_string(),
+        }
+    }
+
    fn with_audit_webhook_target_env_cleared<F>(target_name: &str, f: F)
    where
        F: FnOnce(),
@@ -466,10 +487,10 @@ mod tests {
            ],
            || {
                let runtime = HashMap::from([
-                    (("mixed-target".to_string(), "webhook".to_string()), "online".to_string()),
-                    (("env-only".to_string(), "webhook".to_string()), "online".to_string()),
+                    (("mixed-target".to_string(), "webhook".to_string()), online_health()),
+                    (("env-only".to_string(), "webhook".to_string()), online_health()),
                ]);
-                let merged = merge_audit_endpoints(&config, runtime);
+                let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");

                let mixed = merged
                    .iter()
@@ -509,10 +530,10 @@ mod tests {
            ],
            || {
                let runtime = HashMap::from([
-                    (("mixed-kafka".to_string(), "kafka".to_string()), "online".to_string()),
-                    (("env-kafka".to_string(), "kafka".to_string()), "online".to_string()),
+                    (("mixed-kafka".to_string(), "kafka".to_string()), online_health()),
+                    (("env-kafka".to_string(), "kafka".to_string()), online_health()),
                ]);
-                let merged = merge_audit_endpoints(&config, runtime);
+                let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");

                let mixed = merged
                    .iter()
@@ -546,10 +567,10 @@ mod tests {
            ],
            || {
                let runtime = HashMap::from([
-                    (("mixed-amqp".to_string(), "amqp".to_string()), "online".to_string()),
-                    (("env-amqp".to_string(), "amqp".to_string()), "online".to_string()),
+                    (("mixed-amqp".to_string(), "amqp".to_string()), online_health()),
+                    (("env-amqp".to_string(), "amqp".to_string()), online_health()),
                ]);
-                let merged = merge_audit_endpoints(&config, runtime);
+                let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");

                let mixed = merged
                    .iter()
@@ -576,7 +597,8 @@ mod tests {
            ],
            || {
                let config = Config(HashMap::new());
-                let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary");
+                let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary")
+                    .expect("audit target mutation block reason");
                assert!(reason.is_some());
                assert!(reason.unwrap().contains("managed by environment variables"));
            },
@@ -613,7 +635,8 @@ mod tests {
                AUDIT_WEBHOOK_SUB_SYS.to_string(),
                HashMap::from([("primary".to_string(), enabled_kvs("on"))]),
            )]));
-            let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary");
+            let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary")
+                .expect("audit target mutation block reason");
            assert!(reason.is_some());
            assert!(reason.unwrap().contains("both persisted config and environment variables"));
        });
@@ -633,7 +656,7 @@ mod tests {
                ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_MIXED-DISABLED", Some("https://example.com/hook")),
            ],
            || {
-                let merged = merge_audit_endpoints(&config, HashMap::new());
+                let merged = merge_audit_endpoints(&config, HashMap::new()).expect("merge audit endpoints");
                let mixed = merged
                    .iter()
                    .find(|entry| entry.account_id == "mixed-disabled")
@@ -655,7 +678,7 @@ mod tests {
                ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")),
            ],
            || {
-                let merged = merge_audit_endpoints(&config, HashMap::new());
+                let merged = merge_audit_endpoints(&config, HashMap::new()).expect("merge audit endpoints");
                let env_only = merged
                    .iter()
                    .find(|entry| entry.account_id == "env-only")
@@ -767,8 +790,8 @@ mod tests {
                ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARYCASE", Some("https://example.com/hook")),
            ],
            || {
-                let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), "online".to_string())]);
-                let merged = merge_audit_endpoints(&config, runtime);
+                let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), online_health())]);
+                let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");
                let mixed = merged
                    .iter()
                    .find(|entry| entry.account_id == "PrimaryCase" && entry.service == "webhook")
@@ -787,7 +810,11 @@ mod tests {
        )]));

        with_audit_webhook_target_env_cleared("primarycase", || {
-            assert!(audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primarycase").is_none());
+            assert!(
+                audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primarycase")
+                    .expect("audit target mutation block reason")
+                    .is_none()
+            );
        });
    }

@@ -795,7 +822,11 @@ mod tests {
    fn audit_target_mutation_block_reason_allows_runtime_only_target() {
        with_audit_webhook_target_env_cleared("primary", || {
            let config = Config(HashMap::new());
-            assert!(audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary").is_none());
+            assert!(
+                audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary")
+                    .expect("audit target mutation block reason")
+                    .is_none()
+            );
        });
    }

@@ -14,11 +14,15 @@

 use crate::admin::handlers::target_descriptor::AdminTargetSpec;
 use crate::admin::runtime_sources::{AppContext, current_app_context, current_object_store_handle_for_context};
-use crate::admin::storage_api::config::{read_admin_config_without_migrate, save_admin_server_config};
+use crate::admin::storage_api::config::{
+    read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config_no_lock,
+    with_admin_server_config_write_lock,
+};
 use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
 use rustfs_config::DEFAULT_DELIMITER;
 use rustfs_config::server_config::Config;
 use s3s::{S3Result, s3_error};
+use tracing::warn;

 pub(crate) async fn load_server_config_from_store_for_context(context: Option<&AppContext>) -> S3Result<Config> {
    let Some(store) = current_object_store_handle_for_context(context) else {
@@ -51,30 +55,31 @@ pub(crate) async fn apply_audit_runtime_config(specs: &[AdminTargetSpec], config
        match system.get_state().await {
            AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {
                if has_targets {
-                    system
-                        .reload_config(config)
-                        .await
-                        .map_err(|e| s3_error!(InternalError, "failed to reload audit config: {}", e))?;
+                    system.reload_config(config).await.map_err(|_| {
+                        warn!(reason = "reload_failed", "Failed to reload local audit runtime");
+                        s3_error!(InternalError, "failed to reload audit config")
+                    })?;
                } else {
-                    system
-                        .close()
-                        .await
-                        .map_err(|e| s3_error!(InternalError, "failed to stop audit system: {}", e))?;
+                    system.close().await.map_err(|_| {
+                        warn!(reason = "stop_failed", "Failed to stop local audit runtime");
+                        s3_error!(InternalError, "failed to stop audit system")
+                    })?;
                }
            }
            AuditSystemState::Stopped | AuditSystemState::Stopping => {
                if has_targets {
-                    system
-                        .start(config)
-                        .await
-                        .map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
+                    system.start(config).await.map_err(|_| {
+                        warn!(reason = "start_failed", "Failed to start local audit runtime");
+                        s3_error!(InternalError, "failed to start audit system")
+                    })?;
                }
            }
        }
    } else if has_targets {
-        start_global_audit_system(config)
-            .await
-            .map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
+        start_global_audit_system(config).await.map_err(|_| {
+            warn!(reason = "start_failed", "Failed to start global audit runtime");
+            s3_error!(InternalError, "failed to start audit system")
+        })?;
    }

    Ok(())
@@ -86,30 +91,40 @@ async fn update_audit_config_and_reload_for_context<F>(
    mut modifier: F,
 ) -> S3Result<()>
 where
-    F: FnMut(&mut Config) -> bool,
+    F: FnMut(&mut Config) -> bool + Send + 'static,
 {
    let Some(store) = current_object_store_handle_for_context(context) else {
        return Err(s3_error!(InternalError, "server storage not initialized"));
    };

-    let mut config = read_admin_config_without_migrate(store.clone())
-        .await
-        .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
+    let specs = specs.to_vec();
+    let lock_store = store.clone();
+    with_admin_server_config_write_lock(lock_store, move || async move {
+        let mut config = read_admin_config_without_migrate_no_lock(store.clone())
+            .await
+            .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;

-    if !modifier(&mut config) {
-        return Ok(());
-    }
+        if !modifier(&mut config) {
+            return Ok(());
+        }

-    save_admin_server_config(store, &config)
-        .await
-        .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
+        save_admin_server_config_no_lock(store, &config)
+            .await
+            .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;

-    apply_audit_runtime_config(specs, config).await
+        // Keep persistence and runtime publication in one detached, serialized
+        // mutation. Otherwise a cancelled caller or two concurrent updates can
+        // leave the persisted config and active audit generation disagreeing.
+        apply_audit_runtime_config(&specs, config).await
+    })
+    .await
+    .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
+    Ok(())
 }

 pub(crate) async fn update_audit_config_and_reload<F>(specs: &[AdminTargetSpec], modifier: F) -> S3Result<()>
 where
-    F: FnMut(&mut Config) -> bool,
+    F: FnMut(&mut Config) -> bool + Send + 'static,
 {
    let context = current_app_context();
    update_audit_config_and_reload_for_context(context.as_deref(), specs, modifier).await
@@ -121,26 +136,30 @@ pub(crate) async fn set_audit_target_config(
    target_name: &str,
    kvs: rustfs_config::server_config::KVS,
 ) -> S3Result<()> {
-    update_audit_config_and_reload(specs, |config| {
+    let subsystem = subsystem.to_lowercase();
+    let target_name = target_name.to_lowercase();
+    update_audit_config_and_reload(specs, move |config| {
        config
            .0
-            .entry(subsystem.to_lowercase())
+            .entry(subsystem.clone())
            .or_default()
-            .insert(target_name.to_lowercase(), kvs.clone());
+            .insert(target_name.clone(), kvs.clone());
        true
    })
    .await
 }

 pub(crate) async fn remove_audit_target_config(specs: &[AdminTargetSpec], subsystem: &str, target_name: &str) -> S3Result<()> {
-    update_audit_config_and_reload(specs, |config| {
+    let subsystem = subsystem.to_lowercase();
+    let target_name = target_name.to_lowercase();
+    update_audit_config_and_reload(specs, move |config| {
        let mut changed = false;
-        if let Some(targets) = config.0.get_mut(&subsystem.to_lowercase()) {
-            if targets.remove(&target_name.to_lowercase()).is_some() {
+        if let Some(targets) = config.0.get_mut(&subsystem) {
+            if targets.remove(&target_name).is_some() {
                changed = true;
            }
            if targets.is_empty() {
-                config.0.remove(&subsystem.to_lowercase());
+                config.0.remove(&subsystem);
            }
        }
        changed
@@ -13,6 +13,7 @@
 // limitations under the License.

 use crate::admin::auth::validate_admin_request;
+use crate::admin::handlers::supervise_admin_mutation;
 use crate::admin::router::{AdminOperation, Operation, S3Router};
 use crate::admin::runtime_sources::{
    current_app_context, current_object_store_handle_for_context, current_server_config_for_context, publish_server_config,
@@ -20,12 +21,14 @@ use crate::admin::runtime_sources::{
 use crate::admin::service::config::{
    CONFIG_WORKER_RELOAD_FAILURE_STATE, EVENT_CONFIG_WORKER_RELOAD_FAILED, FULL_CONFIG_WORKER_SUBSYSTEMS, LOG_COMPONENT_ADMIN,
    LOG_SUBSYSTEM_CONFIG, PreparedRuntimeConfig, apply_dynamic_config_for_subsystem, is_dynamic_config_subsystem,
-    prepare_server_config, signal_config_snapshot_reload, signal_dynamic_config_reload,
+    preflight_dynamic_config_reload, prepare_server_config, signal_config_snapshot_reload_checked,
+    signal_dynamic_config_reload_checked,
 };
 use crate::admin::storage_api::config::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV};
 use crate::admin::storage_api::config::{
    RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config, read_admin_config_without_migrate,
-    save_admin_config, save_admin_server_config,
+    read_admin_config_without_migrate_no_lock, save_admin_config, save_admin_server_config_no_lock,
+    with_admin_server_config_write_lock,
 };
 use crate::admin::storage_api::contract::list::ListOperations as _;
 use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body};
@@ -47,7 +50,7 @@ use rustfs_config::notify::{
    ENV_NOTIFY_MQTT_QOS, ENV_NOTIFY_MQTT_QUEUE_DIR, ENV_NOTIFY_MQTT_QUEUE_LIMIT, ENV_NOTIFY_MQTT_RECONNECT_INTERVAL,
    ENV_NOTIFY_MQTT_TOPIC, ENV_NOTIFY_MQTT_USERNAME, ENV_NOTIFY_WEBHOOK_AUTH_TOKEN, ENV_NOTIFY_WEBHOOK_CLIENT_CERT,
    ENV_NOTIFY_WEBHOOK_CLIENT_KEY, ENV_NOTIFY_WEBHOOK_ENABLE, ENV_NOTIFY_WEBHOOK_ENDPOINT, ENV_NOTIFY_WEBHOOK_QUEUE_DIR,
-    ENV_NOTIFY_WEBHOOK_QUEUE_LIMIT, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
+    ENV_NOTIFY_WEBHOOK_QUEUE_LIMIT, NOTIFY_MQTT_SUB_SYS, NOTIFY_SUB_SYSTEMS, NOTIFY_WEBHOOK_SUB_SYS,
 };
 use rustfs_config::oidc::{
    ENV_IDENTITY_OPENID_CLAIM_NAME, ENV_IDENTITY_OPENID_CLAIM_PREFIX, ENV_IDENTITY_OPENID_CLIENT_ID,
@@ -726,6 +729,14 @@ async fn load_server_config_from_store() -> S3Result<ServerConfig> {
        .map_err(Into::into)
 }

+async fn load_server_config_from_store_locked() -> S3Result<ServerConfig> {
+    let store = object_store()?;
+    read_admin_config_without_migrate_no_lock(store)
+        .await
+        .map_err(ApiError::from)
+        .map_err(Into::into)
+}
+
 async fn load_active_server_config() -> S3Result<ServerConfig> {
    if let Ok(config) = load_server_config_from_store().await {
        return Ok(config);
@@ -736,9 +747,9 @@ async fn load_active_server_config() -> S3Result<ServerConfig> {
        .ok_or_else(|| s3_error!(InternalError, "server config is not initialized"))
 }

-async fn save_server_config_to_store(config: &ServerConfig) -> S3Result<()> {
+async fn save_server_config_to_store_locked(config: &ServerConfig) -> S3Result<()> {
    let store = object_store()?;
-    save_admin_server_config(store, config)
+    save_admin_server_config_no_lock(store, config)
        .await
        .map_err(ApiError::from)
        .map_err(Into::into)
@@ -1563,20 +1574,144 @@ async fn commit_prepared_config(
 /// Re-apply local mutable worker families after a full-config replacement.
 /// Peers receive one full-snapshot signal after this returns; signaling each
 /// family here as well would recreate audit/scanner targets twice per peer.
-async fn apply_dynamic_subsystems(config: &ServerConfig) {
+fn publish_notify_config_intent(
+    config: &ServerConfig,
+    sub_system: Option<&str>,
+) -> Option<rustfs_notify::NotificationLifecycleTransition> {
+    (sub_system.is_none() || sub_system.is_some_and(|sub_system| NOTIFY_SUB_SYSTEMS.contains(&sub_system)))
+        .then(|| rustfs_notify::ensure_live_events().publish_config(config.clone()))
+}
+
+async fn preflight_notify_config_intent(sub_system: Option<&str>) -> S3Result<()> {
+    if sub_system.is_none() || sub_system.is_some_and(|sub_system| NOTIFY_SUB_SYSTEMS.contains(&sub_system)) {
+        preflight_dynamic_config_reload(sub_system.unwrap_or(NOTIFY_WEBHOOK_SUB_SYS)).await?;
+    }
+    Ok(())
+}
+
+async fn wait_notify_config_intent(transition: Option<rustfs_notify::NotificationLifecycleTransition>) -> S3Result<bool> {
+    let Some(transition) = transition else {
+        return Ok(false);
+    };
+    transition.wait().await.map_err(|err| {
+        warn!(error = %err, "Failed to apply local notification config");
+        s3_error!(InternalError, "failed to apply notification config")
+    })?;
+    Ok(true)
+}
+
+async fn apply_non_notify_dynamic_subsystems(config: &ServerConfig) -> Vec<String> {
+    let mut failures = Vec::new();
    for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS {
-        if let Err(err) = apply_dynamic_config_for_subsystem(config, sub_system).await {
+        if NOTIFY_SUB_SYSTEMS.contains(&sub_system) {
+            continue;
+        }
+        if apply_dynamic_config_for_subsystem(config, sub_system).await.is_err() {
+            failures.push(format!("local {sub_system}"));
            warn!(
                event = EVENT_CONFIG_WORKER_RELOAD_FAILED,
                component = LOG_COMPONENT_ADMIN,
                subsystem = LOG_SUBSYSTEM_CONFIG,
                config_subsystem = sub_system,
                state = CONFIG_WORKER_RELOAD_FAILURE_STATE,
-                error = %err,
+                reason = "apply_failed",
                "Published server config but failed to reload a local worker subsystem"
            );
        }
    }
+    failures
+}
+
+fn finish_config_reconciliation(errors: Vec<String>) -> S3Result<()> {
+    if errors.is_empty() {
+        Ok(())
+    } else {
+        Err(s3_error!(
+            InternalError,
+            "server config persisted but runtime convergence failed: {}",
+            errors.join("; ")
+        ))
+    }
+}
+
+async fn reconcile_targeted_config(
+    config: ServerConfig,
+    sub_system: Option<String>,
+    storage_class_applied: bool,
+    notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
+) -> S3Result<bool> {
+    let mut errors = Vec::new();
+    let notify_applied = notify_transition.is_some();
+    if let Err(err) = wait_notify_config_intent(notify_transition).await {
+        warn!(error = %err, "Local notification config failed to converge");
+        errors.push("local notify".to_string());
+    }
+
+    let config_applied = if notify_applied {
+        if let Some(sub_system) = sub_system.as_deref()
+            && let Err(err) = signal_dynamic_config_reload_checked(sub_system).await
+        {
+            warn!(config_subsystem = sub_system, error = %err, "Peer config reload failed");
+            errors.push(format!("peer {sub_system}"));
+        }
+        true
+    } else if storage_class_applied {
+        if let Err(err) = signal_dynamic_config_reload_checked(STORAGE_CLASS_SUB_SYS).await {
+            warn!(error = %err, "Peer storage-class reload failed");
+            errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}"));
+        }
+        true
+    } else if let Some(sub_system) = sub_system.as_deref()
+        && is_dynamic_config_subsystem(sub_system)
+    {
+        let config_applied = match apply_dynamic_config_for_subsystem(&config, sub_system).await {
+            Ok(applied) => applied,
+            Err(_) => {
+                warn!(config_subsystem = sub_system, reason = "apply_failed", "Local config reload failed");
+                errors.push(format!("local {sub_system}"));
+                false
+            }
+        };
+        if let Err(err) = signal_dynamic_config_reload_checked(sub_system).await {
+            warn!(config_subsystem = sub_system, error = %err, "Peer config reload failed");
+            errors.push(format!("peer {sub_system}"));
+        }
+        config_applied
+    } else {
+        if let Err(err) = signal_config_snapshot_reload_checked().await {
+            warn!(error = %err, "Peer config snapshot reload failed");
+            errors.push("peer config snapshot".to_string());
+        }
+        false
+    };
+
+    finish_config_reconciliation(errors)?;
+    Ok(config_applied)
+}
+
+async fn reconcile_full_config(
+    config: ServerConfig,
+    notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
+) -> S3Result<()> {
+    let mut errors = Vec::new();
+    if let Err(err) = wait_notify_config_intent(notify_transition).await {
+        warn!(error = %err, "Local notification config failed to converge");
+        errors.push("local notify".to_string());
+    }
+    if let Err(err) = signal_dynamic_config_reload_checked(STORAGE_CLASS_SUB_SYS).await {
+        warn!(error = %err, "Peer storage-class reload failed");
+        errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}"));
+    }
+    errors.extend(apply_non_notify_dynamic_subsystems(&config).await);
+    if let Err(err) = signal_dynamic_config_reload_checked(NOTIFY_WEBHOOK_SUB_SYS).await {
+        warn!(error = %err, "Peer notification config reload failed");
+        errors.push("peer notify".to_string());
+    }
+    if let Err(err) = signal_config_snapshot_reload_checked().await {
+        warn!(error = %err, "Peer config snapshot reload failed");
+        errors.push("peer config snapshot".to_string());
+    }
+    finish_config_reconciliation(errors)
 }

 pub struct GetConfigKVHandler {}
@@ -1611,37 +1746,39 @@ impl Operation for SetConfigKVHandler {
        }
        validate_config_directives(&directives)?;

-        let sub_system = config_update_sub_system(&directives)?;
-        let mut config = load_server_config_from_store().await?;
-        apply_set_directives(&mut config, &directives)?;
-        let prepared = prepare_server_config(&config, sub_system).await?;
-        save_server_config_history(&body).await?;
-        let config_applied = if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
-            commit_prepared_config(
-                config.clone(),
-                prepared,
-                save_server_config_to_store(&config),
-                publish_prepared_config_snapshots,
-            )
-            .await?;
-            signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await;
-            true
-        } else {
-            save_server_config_to_store(&config).await?;
-            publish_server_config(config.clone());
-            if let Some(sub_system) = sub_system
-                && is_dynamic_config_subsystem(sub_system)
-            {
-                let config_applied = apply_dynamic_config_for_subsystem(&config, sub_system).await?;
-                if config_applied {
-                    signal_dynamic_config_reload(sub_system).await;
-                }
-                config_applied
-            } else {
-                signal_config_snapshot_reload().await;
-                false
-            }
-        };
+        let sub_system = config_update_sub_system(&directives)?.map(str::to_owned);
+        let transaction_sub_system = sub_system.clone();
+        let config_store = object_store()?;
+        let config_applied = supervise_admin_mutation("config mutation", async move {
+            preflight_notify_config_intent(sub_system.as_deref()).await?;
+            let (config, storage_class_applied, notify_transition) =
+                with_admin_server_config_write_lock(config_store, move || async move {
+                    let sub_system = transaction_sub_system.as_deref();
+                    let mut config = load_server_config_from_store_locked().await?;
+                    apply_set_directives(&mut config, &directives)?;
+                    let prepared = prepare_server_config(&config, sub_system).await?;
+                    save_server_config_history(&body).await?;
+                    if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
+                        commit_prepared_config(
+                            config.clone(),
+                            prepared,
+                            save_server_config_to_store_locked(&config),
+                            publish_prepared_config_snapshots,
+                        )
+                        .await?;
+                    } else {
+                        save_server_config_to_store_locked(&config).await?;
+                        publish_server_config(config.clone());
+                    }
+                    let notify_transition = publish_notify_config_intent(&config, sub_system);
+                    Ok::<_, S3Error>((config, sub_system == Some(STORAGE_CLASS_SUB_SYS), notify_transition))
+                })
+                .await
+                .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
+
+            reconcile_targeted_config(config, sub_system, storage_class_applied, notify_transition).await
+        })
+        .await?;

        success_response(config_applied)
    }
@@ -1660,37 +1797,39 @@ impl Operation for DelConfigKVHandler {
        }
        validate_config_directives(&directives)?;

-        let sub_system = config_update_sub_system(&directives)?;
-        let mut config = load_server_config_from_store().await?;
-        apply_delete_directives(&mut config, &directives);
-        let prepared = prepare_server_config(&config, sub_system).await?;
-        save_server_config_history(&body).await?;
-        let config_applied = if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
-            commit_prepared_config(
-                config.clone(),
-                prepared,
-                save_server_config_to_store(&config),
-                publish_prepared_config_snapshots,
-            )
-            .await?;
-            signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await;
-            true
-        } else {
-            save_server_config_to_store(&config).await?;
-            publish_server_config(config.clone());
-            if let Some(sub_system) = sub_system
-                && is_dynamic_config_subsystem(sub_system)
-            {
-                let config_applied = apply_dynamic_config_for_subsystem(&config, sub_system).await?;
-                if config_applied {
-                    signal_dynamic_config_reload(sub_system).await;
-                }
-                config_applied
-            } else {
-                signal_config_snapshot_reload().await;
-                false
-            }
-        };
+        let sub_system = config_update_sub_system(&directives)?.map(str::to_owned);
+        let transaction_sub_system = sub_system.clone();
+        let config_store = object_store()?;
+        let config_applied = supervise_admin_mutation("config mutation", async move {
+            preflight_notify_config_intent(sub_system.as_deref()).await?;
+            let (config, storage_class_applied, notify_transition) =
+                with_admin_server_config_write_lock(config_store, move || async move {
+                    let sub_system = transaction_sub_system.as_deref();
+                    let mut config = load_server_config_from_store_locked().await?;
+                    apply_delete_directives(&mut config, &directives);
+                    let prepared = prepare_server_config(&config, sub_system).await?;
+                    save_server_config_history(&body).await?;
+                    if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
+                        commit_prepared_config(
+                            config.clone(),
+                            prepared,
+                            save_server_config_to_store_locked(&config),
+                            publish_prepared_config_snapshots,
+                        )
+                        .await?;
+                    } else {
+                        save_server_config_to_store_locked(&config).await?;
+                        publish_server_config(config.clone());
+                    }
+                    let notify_transition = publish_notify_config_intent(&config, sub_system);
+                    Ok::<_, S3Error>((config, sub_system == Some(STORAGE_CLASS_SUB_SYS), notify_transition))
+                })
+                .await
+                .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
+
+            reconcile_targeted_config(config, sub_system, storage_class_applied, notify_transition).await
+        })
+        .await?;

        success_response(config_applied)
    }
@@ -1776,16 +1915,26 @@ impl Operation for RestoreConfigHistoryKVHandler {
        let mut config = ServerConfig::new();
        apply_set_directives(&mut config, &directives)?;
        let prepared = prepare_server_config(&config, None).await?;
-        commit_prepared_config(
-            config.clone(),
-            prepared,
-            save_server_config_to_store(&config),
-            publish_prepared_config_snapshots,
-        )
+        let config_store = object_store()?;
+        supervise_admin_mutation("config mutation", async move {
+            preflight_notify_config_intent(None).await?;
+            let persisted_config = config.clone();
+            let notify_transition = with_admin_server_config_write_lock(config_store, move || async move {
+                commit_prepared_config(
+                    persisted_config.clone(),
+                    prepared,
+                    save_server_config_to_store_locked(&persisted_config),
+                    publish_prepared_config_snapshots,
+                )
+                .await?;
+                Ok::<_, S3Error>(publish_notify_config_intent(&persisted_config, None))
+            })
+            .await
+            .map_err(|err| s3_error!(InternalError, "failed to lock server config restore: {}", err))??;
+
+            reconcile_full_config(config, notify_transition).await
+        })
        .await?;
-        signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await;
-        apply_dynamic_subsystems(&config).await;
-        signal_config_snapshot_reload().await;

        success_response(false)
    }
@@ -1820,17 +1969,27 @@ impl Operation for SetConfigHandler {
        let mut config = ServerConfig::new();
        apply_set_directives(&mut config, &directives)?;
        let prepared = prepare_server_config(&config, None).await?;
-        save_server_config_history(&body).await?;
-        commit_prepared_config(
-            config.clone(),
-            prepared,
-            save_server_config_to_store(&config),
-            publish_prepared_config_snapshots,
-        )
+        let config_store = object_store()?;
+        supervise_admin_mutation("config mutation", async move {
+            preflight_notify_config_intent(None).await?;
+            save_server_config_history(&body).await?;
+            let persisted_config = config.clone();
+            let notify_transition = with_admin_server_config_write_lock(config_store, move || async move {
+                commit_prepared_config(
+                    persisted_config.clone(),
+                    prepared,
+                    save_server_config_to_store_locked(&persisted_config),
+                    publish_prepared_config_snapshots,
+                )
+                .await?;
+                Ok::<_, S3Error>(publish_notify_config_intent(&persisted_config, None))
+            })
+            .await
+            .map_err(|err| s3_error!(InternalError, "failed to lock full server config update: {}", err))??;
+
+            reconcile_full_config(config, notify_transition).await
+        })
        .await?;
-        signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await;
-        apply_dynamic_subsystems(&config).await;
-        signal_config_snapshot_reload().await;

        success_response(false)
    }
@@ -1890,92 +2049,6 @@ mod tests {
        assert_eq!(*events.lock().expect("result events lock"), ["persist"]);
    }

-    #[test]
-    fn storage_config_write_handlers_persist_before_publishing() {
-        const SOURCE: &str = include_str!("config_admin.rs");
-
-        for (handler, next_handler, follow_up) in [
-            (
-                "SetConfigKVHandler",
-                "DelConfigKVHandler",
-                "signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await",
-            ),
-            (
-                "DelConfigKVHandler",
-                "HelpConfigKVHandler",
-                "signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await",
-            ),
-            (
-                "RestoreConfigHistoryKVHandler",
-                "GetConfigHandler",
-                "signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await",
-            ),
-            (
-                "SetConfigHandler",
-                "#[cfg(test)]",
-                "signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await",
-            ),
-        ] {
-            let start_marker = format!("impl Operation for {handler}");
-            let start = SOURCE
-                .find(&start_marker)
-                .unwrap_or_else(|| panic!("missing {handler} implementation"));
-            let tail = &SOURCE[start..];
-            let end = tail
-                .find(next_handler)
-                .unwrap_or_else(|| panic!("missing {next_handler} after {handler}"));
-            let implementation = &tail[..end];
-            let prepared_commit_path = if matches!(handler, "SetConfigKVHandler" | "DelConfigKVHandler") {
-                let branch_start = implementation
-                    .find("if sub_system == Some(STORAGE_CLASS_SUB_SYS)")
-                    .unwrap_or_else(|| panic!("missing storage-class branch in {handler}"));
-                let branch = &implementation[branch_start..];
-                let branch_end = branch
-                    .find("} else {")
-                    .unwrap_or_else(|| panic!("missing non-storage branch in {handler}"));
-                &branch[..branch_end]
-            } else {
-                implementation
-            };
-
-            assert_eq!(prepared_commit_path.matches("commit_prepared_config(").count(), 1, "{handler}");
-            let commit_start = prepared_commit_path.find("commit_prepared_config(").expect("commit call");
-            let commit_end = prepared_commit_path[commit_start..].find(';').expect("commit terminator") + commit_start;
-            let commit_statement = &prepared_commit_path[commit_start..=commit_end];
-            assert!(commit_statement.contains(".await?;"), "{handler} must propagate commit failure");
-
-            let follow_up_start = prepared_commit_path
-                .find(follow_up)
-                .unwrap_or_else(|| panic!("missing follow-up in {handler}"));
-            assert!(follow_up_start > commit_end, "{handler} must run follow-up only after commit");
-            assert!(
-                !prepared_commit_path.contains("publish_server_config("),
-                "{handler} must not publish directly"
-            );
-            assert!(
-                !prepared_commit_path.contains(".publish_storage_class("),
-                "{handler} must not publish directly"
-            );
-
-            if matches!(handler, "RestoreConfigHistoryKVHandler" | "SetConfigHandler") {
-                let worker_start = prepared_commit_path
-                    .find("apply_dynamic_subsystems(&config).await")
-                    .unwrap_or_else(|| panic!("missing local worker apply in {handler}"));
-                let signal_start = prepared_commit_path
-                    .find("signal_config_snapshot_reload().await")
-                    .unwrap_or_else(|| panic!("missing snapshot signal in {handler}"));
-                assert!(
-                    worker_start > follow_up_start,
-                    "{handler} must converge peer parity before local worker apply"
-                );
-                assert!(
-                    signal_start > worker_start,
-                    "{handler} must signal the full snapshot after local worker apply"
-                );
-            }
-        }
-    }
-
    #[test]
    fn tokenize_config_line_handles_quotes_and_escapes() {
        let tokens = tokenize_config_line(r#"identity_openid client_id="console app" client_secret="s3cr\"et" enable=on"#)
@@ -15,13 +15,16 @@
 use crate::admin::{
    auth::validate_admin_request,
    handlers::notify_runtime_access::{get_notification_system, load_notification_config_snapshot},
+    handlers::supervise_admin_mutation,
    handlers::target_descriptor::{
-        AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, build_enabled_target_kvs,
-        build_json_response, collect_runtime_statuses, extract_supported_target_params,
+        AdminTargetSpec, EndpointKey, RuntimeHealthStatus, TargetEndpointSource, admin_target_spec_from_builtin,
+        build_enabled_target_kvs, build_json_response, collect_runtime_statuses, extract_supported_target_params,
        merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
        target_mutation_block_reason as shared_target_mutation_block_reason,
    },
    router::{AdminOperation, Operation, S3Router},
+    runtime_sources::{AppContext, app_context_from_req},
+    service::config::{preflight_dynamic_config_reload_for_context, signal_dynamic_config_reload_checked_for_context},
 };
 use crate::auth::{check_key_valid, get_session_token};
 use crate::server::{
@@ -43,6 +46,45 @@ use std::sync::LazyLock;
 use tracing::{Span, error, info, warn};

 const LOG_COMPONENT_ADMIN_API: &str = "admin_api";
+
+async fn converge_target_mutation_on_cluster(
+    context: Option<&AppContext>,
+    target_type: &str,
+    local_result: Result<(), rustfs_notify::NotificationError>,
+) -> S3Result<()> {
+    let subsystem = notification_target_subsystem(target_type)?;
+    let peer_result = signal_dynamic_config_reload_checked_for_context(context, subsystem).await;
+
+    match (local_result, peer_result) {
+        (Ok(()), Ok(())) => Ok(()),
+        (Err(local), Ok(())) => {
+            warn!(target_type, error = %local, "Local notification runtime failed to converge");
+            Err(s3_error!(InternalError, "local notification runtime failed to converge"))
+        }
+        (Ok(()), Err(peer)) => Err(peer),
+        (Err(local), Err(peer)) => {
+            warn!(target_type, error = %local, "Local notification runtime failed while peer convergence also failed");
+            Err(s3_error!(
+                InternalError,
+                "local notification runtime and peer convergence failed: {}",
+                peer
+            ))
+        }
+    }
+}
+
+fn notification_target_subsystem(target_type: &str) -> S3Result<&'static str> {
+    notification_target_specs()
+        .iter()
+        .find(|spec| spec.subsystem == target_type)
+        .map(|spec| spec.subsystem)
+        .ok_or_else(|| s3_error!(InvalidArgument, "unsupported notification target type: {}", target_type))
+}
+
+async fn preflight_target_mutation_on_cluster(context: Option<&AppContext>, target_type: &str) -> S3Result<()> {
+    preflight_dynamic_config_reload_for_context(context, notification_target_subsystem(target_type)?).await
+}
+
 const LOG_SUBSYSTEM_NOTIFICATION_TARGET: &str = "notification_target";
 const EVENT_ADMIN_REQUEST_REJECTED: &str = "admin_request_rejected";
 const EVENT_ADMIN_REQUEST_FAILED: &str = "admin_request_failed";
@@ -198,11 +240,14 @@ struct NotificationEndpoint {
    account_id: String,
    service: String,
    status: String,
+    health_state: String,
+    health_reason: String,
    source: TargetEndpointSource,
 }

 #[derive(Serialize, Debug)]
 struct NotificationEndpointsResponse {
+    notify_enabled: bool,
    notification_endpoints: Vec<NotificationEndpoint>,
 }

@@ -229,7 +274,7 @@ async fn authorize_notification_admin_request(req: &S3Request<Body>, action: Adm
    validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await
 }

-fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
+fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> S3Result<Option<String>> {
    shared_target_mutation_block_reason(
        notification_target_specs(),
        NOTIFY_ROUTE_PREFIX,
@@ -256,16 +301,23 @@ async fn notification_target_operation_block_reason(action: &str) -> Option<Stri
    target_module_disabled_reason("notify", rustfs_config::ENV_NOTIFY_ENABLE, is_notify_module_enabled(), action)
 }

-fn merge_notification_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<NotificationEndpoint> {
-    shared_merge_target_endpoints(notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses)
-        .into_iter()
-        .map(|endpoint| NotificationEndpoint {
-            account_id: endpoint.account_id,
-            service: endpoint.service,
-            status: endpoint.status,
-            source: endpoint.source,
-        })
-        .collect()
+fn merge_notification_endpoints(
+    config: &Config,
+    runtime_statuses: HashMap<EndpointKey, RuntimeHealthStatus>,
+) -> S3Result<Vec<NotificationEndpoint>> {
+    Ok(
+        shared_merge_target_endpoints(notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses)?
+            .into_iter()
+            .map(|endpoint| NotificationEndpoint {
+                account_id: endpoint.account_id,
+                service: endpoint.service,
+                status: endpoint.status,
+                health_state: endpoint.health_state,
+                health_reason: endpoint.health_reason,
+                source: endpoint.source,
+            })
+            .collect(),
+    )
 }

 fn collect_online_target_arns(region: &str, target_statuses: Vec<(rustfs_targets::arn::TargetID, String)>) -> Vec<String> {
@@ -284,6 +336,7 @@ impl Operation for NotificationTarget {
        let span = Span::current();
        let _enter = span.enter();
        let (target_type, target_name) = extract_target_params(&params)?;
+        let context = app_context_from_req(&req);

        authorize_notification_admin_request(&req, AdminAction::SetBucketTargetAction).await?;
        if let Some(reason) = notification_target_operation_block_reason("managing notification targets from the console").await {
@@ -291,7 +344,7 @@ impl Operation for NotificationTarget {
            return Err(s3_error!(InvalidRequest, "{reason}"));
        }
        let (ns, config_snapshot) = load_notification_config_snapshot().await?;
-        if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) {
+        if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name)? {
            log_notification_target_operation_blocked!("set_target_config", Some(target_type), Some(target_name), &reason);
            return Err(s3_error!(InvalidRequest, "{reason}"));
        }
@@ -320,7 +373,15 @@ impl Operation for NotificationTarget {
        )
        .await?;

-        ns.set_target_config(target_type, target_name, kvs).await.map_err(|e| {
+        let mutation_target_type = target_type.to_owned();
+        let mutation_target_name = target_name.to_owned();
+        supervise_admin_mutation("notification target mutation", async move {
+            preflight_target_mutation_on_cluster(context.as_deref(), &mutation_target_type).await?;
+            let local_result = ns.set_target_config(&mutation_target_type, &mutation_target_name, kvs).await;
+            converge_target_mutation_on_cluster(context.as_deref(), &mutation_target_type, local_result).await
+        })
+        .await
+        .map_err(|e| {
            log_notification_target_request_failed!(
                "set_target_config",
                "set_target_config_failed",
@@ -328,7 +389,7 @@ impl Operation for NotificationTarget {
                Some(target_name),
                e
            );
-            s3_error!(InternalError, "failed to set target config: {}", e)
+            e
        })?;
        log_notification_target_config_updated!("set_target_config", target_type, target_name);

@@ -343,11 +404,29 @@ impl Operation for ListNotificationTargets {
        let span = Span::current();
        let _enter = span.enter();
        authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;
+        refresh_persisted_module_switches_from_store().await.map_err(|err| {
+            warn!(
+                event = EVENT_ADMIN_REQUEST_FAILED,
+                component = LOG_COMPONENT_ADMIN_API,
+                subsystem = LOG_SUBSYSTEM_NOTIFICATION_TARGET,
+                operation = "list_targets",
+                result = "failed",
+                reason = "module_switch_refresh_failed",
+                error = %err,
+                "admin request failed"
+            );
+            s3_error!(InternalError, "failed to refresh notification module state")
+        })?;
+        let notify_enabled = refresh_notify_module_enabled();
        let (ns, config) = load_notification_config_snapshot().await?;
        let runtime_statuses = collect_runtime_statuses(ns.get_target_values().await).await;
-        let notification_endpoints = merge_notification_endpoints(&config, runtime_statuses);
+        let notification_endpoints = merge_notification_endpoints(&config, runtime_statuses)?;

-        let data = serde_json::to_vec(&NotificationEndpointsResponse { notification_endpoints }).map_err(|e| {
+        let data = serde_json::to_vec(&NotificationEndpointsResponse {
+            notify_enabled,
+            notification_endpoints,
+        })
+        .map_err(|e| {
            log_notification_target_request_failed!("list_targets", "serialize_targets_failed", None, None, e);
            s3_error!(InternalError, "failed to serialize targets: {}", e)
        })?;
@@ -388,7 +467,7 @@ impl Operation for ListTargetsArns {
        let target_statuses = collect_runtime_statuses(ns.get_target_values().await)
            .await
            .into_iter()
-            .map(|((account_id, service), status)| (rustfs_targets::arn::TargetID::new(account_id, service), status))
+            .map(|((account_id, service), health)| (rustfs_targets::arn::TargetID::new(account_id, service), health.status))
            .collect();

        let data_target_arn_list = collect_online_target_arns(region.as_str(), target_statuses);
@@ -409,6 +488,7 @@ impl Operation for RemoveNotificationTarget {
        let span = Span::current();
        let _enter = span.enter();
        let (target_type, target_name) = extract_target_params(&params)?;
+        let context = app_context_from_req(&req);

        authorize_notification_admin_request(&req, AdminAction::SetBucketTargetAction).await?;
        if let Some(reason) = notification_target_operation_block_reason("managing notification targets from the console").await {
@@ -416,12 +496,20 @@ impl Operation for RemoveNotificationTarget {
            return Err(s3_error!(InvalidRequest, "{reason}"));
        }
        let (ns, config_snapshot) = load_notification_config_snapshot().await?;
-        if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) {
+        if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name)? {
            log_notification_target_operation_blocked!("remove_target_config", Some(target_type), Some(target_name), &reason);
            return Err(s3_error!(InvalidRequest, "{reason}"));
        }

-        ns.remove_target_config(target_type, target_name).await.map_err(|e| {
+        let mutation_target_type = target_type.to_owned();
+        let mutation_target_name = target_name.to_owned();
+        supervise_admin_mutation("notification target mutation", async move {
+            preflight_target_mutation_on_cluster(context.as_deref(), &mutation_target_type).await?;
+            let local_result = ns.remove_target_config(&mutation_target_type, &mutation_target_name).await;
+            converge_target_mutation_on_cluster(context.as_deref(), &mutation_target_type, local_result).await
+        })
+        .await
+        .map_err(|e| {
            log_notification_target_request_failed!(
                "remove_target_config",
                "remove_target_config_failed",
@@ -429,7 +517,7 @@ impl Operation for RemoveNotificationTarget {
                Some(target_name),
                e
            );
-            s3_error!(InternalError, "failed to remove target config: {}", e)
+            e
        })?;
        log_notification_target_config_updated!("remove_target_config", target_type, target_name);

@@ -464,6 +552,54 @@ mod tests {
        }])
    }

+    fn runtime_health(status: &str) -> RuntimeHealthStatus {
+        let online = status == "online";
+        RuntimeHealthStatus {
+            status: status.to_string(),
+            state: if online { "online" } else { "offline" }.to_string(),
+            reason: if online { "reachable" } else { "unreachable" }.to_string(),
+        }
+    }
+
+    #[test]
+    fn notification_target_subsystem_resolves_admin_route_type() {
+        assert_eq!(
+            notification_target_subsystem(NOTIFY_WEBHOOK_SUB_SYS).expect("webhook subsystem should resolve"),
+            NOTIFY_WEBHOOK_SUB_SYS
+        );
+        assert!(notification_target_subsystem("webhook").is_err());
+    }
+
+    #[test]
+    fn notification_endpoints_response_includes_required_module_state() {
+        let response = NotificationEndpointsResponse {
+            notify_enabled: true,
+            notification_endpoints: vec![NotificationEndpoint {
+                account_id: "primary".to_string(),
+                service: "webhook".to_string(),
+                status: "online".to_string(),
+                health_state: "online".to_string(),
+                health_reason: "reachable".to_string(),
+                source: TargetEndpointSource::Config,
+            }],
+        };
+
+        assert_eq!(
+            serde_json::to_value(response).expect("notification target response should serialize"),
+            serde_json::json!({
+                "notify_enabled": true,
+                "notification_endpoints": [{
+                    "account_id": "primary",
+                    "service": "webhook",
+                    "status": "online",
+                    "health_state": "online",
+                    "health_reason": "reachable",
+                    "source": "config"
+                }]
+            })
+        );
+    }
+
    #[test]
    fn merge_notification_endpoints_keeps_configured_targets_after_runtime_loss() {
        let mut cfg_map = HashMap::new();
@@ -477,8 +613,15 @@ mod tests {
        );
        let config = Config(cfg_map);

-        let runtime = HashMap::from([(("webhook-a".to_string(), "webhook".to_string()), "online".to_string())]);
-        let merged = merge_notification_endpoints(&config, runtime);
+        let runtime = HashMap::from([(
+            ("webhook-a".to_string(), "webhook".to_string()),
+            RuntimeHealthStatus {
+                status: "offline".to_string(),
+                state: "error".to_string(),
+                reason: "timed_out".to_string(),
+            },
+        )]);
+        let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");

        let mqtt = merged
            .iter()
@@ -491,7 +634,9 @@ mod tests {
            .iter()
            .find(|entry| entry.account_id == "webhook-a" && entry.service == "webhook")
            .expect("webhook-a should be present");
-        assert_eq!(webhook.status, "online");
+        assert_eq!(webhook.status, "offline");
+        assert_eq!(webhook.health_state, "error");
+        assert_eq!(webhook.health_reason, "timed_out");
        assert_eq!(webhook.source, TargetEndpointSource::Config);
    }

@@ -504,10 +649,10 @@ mod tests {
        let config = Config(HashMap::from([(NOTIFY_WEBHOOK_SUB_SYS.to_string(), webhook_targets)]));

        let runtime = HashMap::from([
-            (("webhook-enabled".to_string(), "webhook".to_string()), "online".to_string()),
-            (("env-only".to_string(), "mqtt".to_string()), "offline".to_string()),
+            (("webhook-enabled".to_string(), "webhook".to_string()), runtime_health("online")),
+            (("env-only".to_string(), "mqtt".to_string()), runtime_health("offline")),
        ]);
-        let merged = merge_notification_endpoints(&config, runtime);
+        let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");

        let env_only = merged
            .iter()
@@ -546,10 +691,10 @@ mod tests {
            ],
            || {
                let runtime = HashMap::from([
-                    (("mixed-target".to_string(), "webhook".to_string()), "online".to_string()),
-                    (("env-only".to_string(), "webhook".to_string()), "online".to_string()),
+                    (("mixed-target".to_string(), "webhook".to_string()), runtime_health("online")),
+                    (("env-only".to_string(), "webhook".to_string()), runtime_health("online")),
                ]);
-                let merged = merge_notification_endpoints(&config, runtime);
+                let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");

                let mixed = merged
                    .iter()
@@ -589,10 +734,10 @@ mod tests {
            ],
            || {
                let runtime = HashMap::from([
-                    (("mixed-kafka".to_string(), "kafka".to_string()), "online".to_string()),
-                    (("env-kafka".to_string(), "kafka".to_string()), "online".to_string()),
+                    (("mixed-kafka".to_string(), "kafka".to_string()), runtime_health("online")),
+                    (("env-kafka".to_string(), "kafka".to_string()), runtime_health("online")),
                ]);
-                let merged = merge_notification_endpoints(&config, runtime);
+                let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");

                let mixed = merged
                    .iter()
@@ -626,10 +771,10 @@ mod tests {
            ],
            || {
                let runtime = HashMap::from([
-                    (("mixed-amqp".to_string(), "amqp".to_string()), "online".to_string()),
-                    (("env-amqp".to_string(), "amqp".to_string()), "online".to_string()),
+                    (("mixed-amqp".to_string(), "amqp".to_string()), runtime_health("online")),
+                    (("env-amqp".to_string(), "amqp".to_string()), runtime_health("online")),
                ]);
-                let merged = merge_notification_endpoints(&config, runtime);
+                let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");

                let mixed = merged
                    .iter()
@@ -656,7 +801,8 @@ mod tests {
            ],
            || {
                let config = Config(HashMap::new());
-                let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary");
+                let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary")
+                    .expect("target mutation block reason");
                assert!(reason.is_some());
                assert!(reason.unwrap().contains("managed by environment variables"));
            },
@@ -696,7 +842,8 @@ mod tests {
                NOTIFY_WEBHOOK_SUB_SYS.to_string(),
                HashMap::from([("primary".to_string(), enabled_kvs("on"))]),
            )]));
-            let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary");
+            let reason =
+                target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary").expect("target mutation block reason");
            assert!(reason.is_some());
            assert!(reason.unwrap().contains("both persisted config and environment variables"));
        });
@@ -709,7 +856,11 @@ mod tests {
            NOTIFY_WEBHOOK_SUB_SYS.to_string(),
            HashMap::from([(target_name.to_string(), enabled_kvs("on"))]),
        )]));
-        assert!(target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, target_name).is_none());
+        assert!(
+            target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, target_name)
+                .expect("target mutation block reason")
+                .is_none()
+        );
    }

    #[test]
@@ -726,7 +877,7 @@ mod tests {
                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_MIXED-DISABLED", Some("https://example.com/hook")),
            ],
            || {
-                let merged = merge_notification_endpoints(&config, HashMap::new());
+                let merged = merge_notification_endpoints(&config, HashMap::new()).expect("merge notification endpoints");
                let mixed = merged
                    .iter()
                    .find(|entry| entry.account_id == "mixed-disabled")
@@ -748,7 +899,7 @@ mod tests {
                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")),
            ],
            || {
-                let merged = merge_notification_endpoints(&config, HashMap::new());
+                let merged = merge_notification_endpoints(&config, HashMap::new()).expect("merge notification endpoints");
                let env_only = merged
                    .iter()
                    .find(|entry| entry.account_id == "env-only")
@@ -797,8 +948,8 @@ mod tests {
                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARYCASE", Some("https://example.com/hook")),
            ],
            || {
-                let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), "online".to_string())]);
-                let merged = merge_notification_endpoints(&config, runtime);
+                let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), runtime_health("online"))]);
+                let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
                let mixed = merged
                    .iter()
                    .find(|entry| entry.account_id == "PrimaryCase" && entry.service == "webhook")
@@ -835,7 +986,11 @@ mod tests {
                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARYCASE", None::<&str>),
            ],
            || {
-                assert!(target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primarycase").is_none());
+                assert!(
+                    target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primarycase")
+                        .expect("target mutation block reason")
+                        .is_none()
+                );
            },
        );
    }
@@ -864,6 +1019,22 @@ mod tests {
            list_block.contains("authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;"),
            "notification target list should require GetBucketTargetAction"
        );
+        let authorize_index = list_block
+            .find("authorize_notification_admin_request")
+            .expect("target list should authorize the request");
+        let refresh_index = list_block
+            .find("refresh_persisted_module_switches_from_store().await.map_err")
+            .expect("target list should fail when persisted module state cannot be refreshed");
+        let effective_state_index = list_block
+            .find("let notify_enabled = refresh_notify_module_enabled();")
+            .expect("target list should resolve the effective env and persisted module state");
+        let load_index = list_block
+            .find("load_notification_config_snapshot().await?")
+            .expect("target list should load notification config");
+        assert!(
+            authorize_index < refresh_index && refresh_index < effective_state_index && effective_state_index < load_index,
+            "target list should authorize, refresh persisted state, resolve effective state, then load targets"
+        );
        assert!(
            arns_block.contains("authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;"),
            "notification target arn listing should require GetBucketTargetAction"
@@ -486,6 +486,8 @@ mod tests {
            account_id: "primary".to_string(),
            service: "webhook".to_string(),
            status: "offline".to_string(),
+            health_state: "offline".to_string(),
+            health_reason: "not_loaded_in_runtime".to_string(),
            source: PluginInstanceSource::Config,
            enabled: true,
            config: HashMap::from([("endpoint".to_string(), "https://example.test/webhook".to_string())]),
@@ -63,6 +63,19 @@ pub mod user_iam;
 pub mod user_lifecycle;
 pub mod user_policy_binding;

+pub(crate) async fn supervise_admin_mutation<T>(
+    operation: &'static str,
+    mutation: impl std::future::Future<Output = s3s::S3Result<T>> + Send + 'static,
+) -> s3s::S3Result<T>
+where
+    T: Send + 'static,
+{
+    tokio::spawn(mutation).await.map_err(|err| {
+        let outcome = if err.is_cancelled() { "cancelled" } else { "panicked" };
+        s3s::s3_error!(InternalError, "{} task {}", operation, outcome)
+    })?
+}
+
 #[cfg(test)]
 mod tests {
    use super::*;
@@ -114,6 +127,44 @@ mod tests {
        // Test passes if we reach this point without panicking
    }

+    #[tokio::test]
+    async fn supervised_admin_mutation_survives_waiter_cancellation() {
+        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
+        let (completed_tx, completed_rx) = tokio::sync::oneshot::channel();
+
+        let waiter = tokio::spawn(async move {
+            supervise_admin_mutation("test mutation", async move {
+                let _ = started_tx.send(());
+                let _ = release_rx.await;
+                let _ = completed_tx.send(());
+                Ok(())
+            })
+            .await
+        });
+
+        started_rx.await.expect("mutation started");
+        waiter.abort();
+        release_tx.send(()).expect("release mutation");
+        tokio::time::timeout(std::time::Duration::from_secs(1), completed_rx)
+            .await
+            .expect("detached mutation should complete")
+            .expect("completion signal");
+    }
+
+    #[tokio::test]
+    async fn supervised_admin_mutation_does_not_expose_panic_payload() {
+        let error = supervise_admin_mutation::<()>("test mutation", async {
+            panic!("do-not-expose-payload");
+        })
+        .await
+        .expect_err("panicking mutation should fail");
+
+        let rendered = error.to_string();
+        assert!(rendered.contains("panicked"));
+        assert!(!rendered.contains("do-not-expose-payload"));
+    }
+
    // Note: Testing the actual async handler implementations requires:
    // 1. S3Request setup with proper headers, URI, and credentials
    // 2. Global object store initialization
@@ -12,26 +12,31 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use crate::admin::runtime_sources::default_admin_usecase;
+use crate::admin::runtime_sources::{AppContext, app_context_from_req, default_admin_usecase};
+use crate::admin::service::config::{
+    preflight_dynamic_config_reload_for_context, signal_dynamic_config_reload_checked_for_context,
+};
 use crate::admin::{
    auth::validate_admin_request,
+    handlers::supervise_admin_mutation,
    router::{AdminOperation, Operation, S3Router},
 };
 use crate::auth::{check_key_valid, get_session_token};
 use crate::server::{
-    ADMIN_PREFIX, ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches, RemoteAddr, current_module_switch_snapshot,
-    init_event_notifier, refresh_audit_module_enabled, refresh_notify_module_enabled,
-    refresh_persisted_module_switches_from_store, save_persisted_module_switches_to_store, shutdown_event_notifier,
-    start_audit_system, stop_audit_system, validate_module_switch_update,
+    ADMIN_PREFIX, MODULE_SWITCHES_SIGNAL_SUBSYSTEM, ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches,
+    RemoteAddr, apply_audit_module_switch_for_context, current_module_switch_snapshot, mark_event_notifier_reconciled,
+    mark_event_notifier_unreconciled, refresh_audit_module_enabled, refresh_notify_module_enabled,
+    refresh_persisted_module_switches_from, refresh_persisted_module_switches_from_store, save_persisted_module_switches_to,
+    validate_module_switch_update,
 };
 use http::{HeaderMap, StatusCode};
 use hyper::Method;
 use matchit::Params;
-use rustfs_audit::AuditError;
 use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
 use rustfs_policy::policy::action::{Action, AdminAction};
 use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
 use serde::{Deserialize, Serialize};
+use std::sync::Arc;

 pub fn register_module_switch_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
    r.insert(
@@ -139,6 +144,74 @@ async fn refresh_module_switch_snapshot() -> S3Result<ModuleSwitchSnapshot> {
    Ok(current_module_switch_snapshot())
 }

+async fn apply_module_switch_update(context: Arc<AppContext>, switches: PersistedModuleSwitches) -> S3Result<()> {
+    preflight_dynamic_config_reload_for_context(Some(context.as_ref()), MODULE_SWITCHES_SIGNAL_SUBSYSTEM).await?;
+    let store = context.object_store();
+    mark_event_notifier_unreconciled();
+    let notification_system = rustfs_notify::ensure_live_events();
+    if switches.notify_enabled {
+        notification_system
+            .reload_persisted_config_from_store(store.clone())
+            .await
+            .map_err(|err| {
+                tracing::warn!(error = %err, "Failed to load notification config for module switch update");
+                s3_error!(InternalError, "failed to load notification config")
+            })?;
+    }
+
+    let transition_system = notification_system.clone();
+    let notify_transition = save_persisted_module_switches_to(store.clone(), switches, move || {
+        let enabled = refresh_notify_module_enabled();
+        transition_system.publish_targets_enabled(enabled, None)
+    })
+    .await
+    .map_err(|err| {
+        tracing::warn!(error = %err, "Failed to save module switches");
+        s3_error!(InternalError, "failed to save module switches")
+    })?;
+
+    let mut failures = Vec::new();
+    let mut notify_converged = true;
+    if let Err(err) = notify_transition.wait().await {
+        tracing::warn!(error = %err, "Local notification runtime failed to apply module switch update");
+        notify_converged = false;
+        failures.push("local notify");
+    }
+    if !switches.notify_enabled
+        && let Err(err) = notification_system.reload_persisted_config_from_store(store).await
+    {
+        tracing::warn!(error = %err, "Local notification config cache failed to reload after module disable");
+        notify_converged = false;
+        failures.push("local notify config cache");
+    }
+    if notify_converged && notification_system.runtime_lifecycle_is_converged() {
+        mark_event_notifier_reconciled();
+    } else if notify_converged {
+        failures.push("local notify convergence");
+    }
+
+    if apply_audit_module_switch_for_context(Some(context.as_ref())).await.is_err() {
+        tracing::warn!(reason = "apply_failed", "Local audit runtime failed to apply module switch update");
+        failures.push("local audit");
+    }
+    if let Err(err) =
+        signal_dynamic_config_reload_checked_for_context(Some(context.as_ref()), MODULE_SWITCHES_SIGNAL_SUBSYSTEM).await
+    {
+        tracing::warn!(error = %err, "Peer nodes failed to apply module switch update");
+        failures.push("peer module switches");
+    }
+
+    if failures.is_empty() {
+        Ok(())
+    } else {
+        Err(s3_error!(
+            InternalError,
+            "module switches persisted but runtime convergence failed: {}",
+            failures.join("; ")
+        ))
+    }
+}
+
 pub struct GetModuleSwitchesHandler {}

 #[async_trait::async_trait]
@@ -156,7 +229,9 @@ pub struct UpdateModuleSwitchesHandler {}
 impl Operation for UpdateModuleSwitchesHandler {
    async fn call(&self, mut req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
        authorize_module_switch_request(&req, AdminAction::ConfigUpdateAdminAction).await?;
-        refresh_persisted_module_switches_from_store()
+        let context = app_context_from_req(&req).ok_or_else(|| s3_error!(InternalError, "storage layer not initialized"))?;
+        let store = context.object_store();
+        refresh_persisted_module_switches_from(store.clone())
            .await
            .map_err(|e| s3_error!(InternalError, "failed to reload persisted module switches: {}", e))?;

@@ -183,28 +258,7 @@ impl Operation for UpdateModuleSwitchesHandler {
            return Err(s3_error!(InvalidRequest, "{err}"));
        }

-        save_persisted_module_switches_to_store(switches)
-            .await
-            .map_err(|e| s3_error!(InternalError, "failed to save module switches: {}", e))?;
-
-        // Apply the new effective values immediately on this node so the console
-        // response reflects the runtime state after to write completes.
-        if refresh_notify_module_enabled() {
-            init_event_notifier().await;
-        } else {
-            shutdown_event_notifier().await;
-        }
-
-        if refresh_audit_module_enabled() {
-            match start_audit_system().await {
-                Ok(()) | Err(AuditError::AlreadyInitialized) => {}
-                Err(e) => return Err(s3_error!(InternalError, "failed to apply audit module switch: {}", e)),
-            }
-        } else {
-            stop_audit_system()
-                .await
-                .map_err(|e| s3_error!(InternalError, "failed to stop audit module after switch update: {}", e))?;
-        }
+        supervise_admin_mutation("module switch update", apply_module_switch_update(context, switches)).await?;

        let snapshot = current_module_switch_snapshot();
        build_response(StatusCode::OK, &ModuleSwitchesResponse::from(snapshot), req.headers.get("x-request-id"))
@@ -233,6 +287,12 @@ mod tests {
            put_block.contains("authorize_module_switch_request(&req, AdminAction::ConfigUpdateAdminAction).await?;"),
            "module switch PUT should require ConfigUpdateAdminAction"
        );
+        assert!(
+            put_block.contains(
+                "supervise_admin_mutation(\"module switch update\", apply_module_switch_update(context, switches)).await?;"
+            ),
+            "module switch PUT must delegate the complete mutation to its supervisor"
+        );
    }

    #[test]
@@ -12,31 +12,33 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use crate::server::init_event_notifier;
+use crate::server::{init_event_notifier, is_event_notifier_reconciled};
 use rustfs_config::server_config::Config;
 use s3s::{S3Result, s3_error};
 use std::sync::Arc;
-use tokio::sync::Mutex;
-
-static NOTIFICATION_SYSTEM_INIT_LOCK: Mutex<()> = Mutex::const_new(());

 pub(crate) async fn get_notification_system() -> S3Result<Arc<rustfs_notify::NotificationSystem>> {
-    if let Some(system) = rustfs_notify::notification_system() {
+    if is_event_notifier_reconciled()
+        && let Some(system) = rustfs_notify::notification_system()
+        && system.runtime_lifecycle_is_converged()
+    {
        return Ok(system);
    }

-    let _guard = NOTIFICATION_SYSTEM_INIT_LOCK.lock().await;
-    if let Some(system) = rustfs_notify::notification_system() {
-        return Ok(system);
+    init_event_notifier()
+        .await
+        .map_err(|err| s3_error!(InternalError, "failed to reconcile notification runtime: {}", err))?;
+    let system =
+        rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized"))?;
+    if !system.runtime_lifecycle_is_converged() {
+        return Err(s3_error!(InternalError, "latest notification lifecycle generation has not converged"));
    }
-
-    init_event_notifier().await;
-    rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized"))
+    Ok(system)
 }

 pub(crate) async fn load_notification_config_snapshot() -> S3Result<(Arc<rustfs_notify::NotificationSystem>, Config)> {
    let system = get_notification_system().await?;
-    let config = system.config.read().await.clone();
+    let config = system.config_snapshot().await;
    Ok((system, config))
 }

@@ -19,7 +19,10 @@ use crate::admin::runtime_sources::{
    current_server_config_for_context,
 };
 use crate::admin::service::federated_identity::DefaultFederatedSessionBinding;
-use crate::admin::storage_api::config::{read_admin_config_without_migrate, save_admin_server_config};
+use crate::admin::storage_api::config::{
+    read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config_no_lock,
+    with_admin_server_config_write_lock,
+};
 use crate::auth::{check_key_valid, get_session_token};
 use crate::server::{ADMIN_PREFIX, CONSOLE_PREFIX, MINIO_ADMIN_PREFIX, RemoteAddr};
 use http::StatusCode;
@@ -368,13 +371,20 @@ impl Operation for PutOidcConfigHandler {
        if is_env_managed_provider(provider_id) {
            return Err(s3_error!(AccessDenied, "provider is managed by environment variables"));
        }
+        let provider_id = provider_id.to_owned();

        let request: OidcConfigUpsertRequest = parse_json_body(&mut req).await?;
-        let mut config = load_server_config_from_store().await?;
-        let existing_secret = persisted_provider_secret(&config, provider_id);
-        let provider_config = build_provider_config_from_upsert(provider_id, request, existing_secret)?;
-        upsert_persisted_provider_config(&mut config, &provider_config);
-        save_server_config_to_store(&config).await?;
+        let store = oidc_config_store()?;
+        let lock_store = store.clone();
+        with_admin_server_config_write_lock(lock_store, move || async move {
+            let mut config = load_server_config_from_store_locked(store.clone()).await?;
+            let existing_secret = persisted_provider_secret(&config, &provider_id);
+            let provider_config = build_provider_config_from_upsert(&provider_id, request, existing_secret)?;
+            upsert_persisted_provider_config(&mut config, &provider_config);
+            save_server_config_to_store_locked(store, &config).await
+        })
+        .await
+        .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;

        json_response(
            StatusCode::OK,
@@ -403,10 +413,17 @@ impl Operation for DeleteOidcConfigHandler {
        if is_env_managed_provider(provider_id) {
            return Err(s3_error!(AccessDenied, "provider is managed by environment variables"));
        }
+        let provider_id = provider_id.to_owned();

-        let mut config = load_server_config_from_store().await?;
-        delete_persisted_provider_config(&mut config, provider_id)?;
-        save_server_config_to_store(&config).await?;
+        let store = oidc_config_store()?;
+        let lock_store = store.clone();
+        with_admin_server_config_write_lock(lock_store, move || async move {
+            let mut config = load_server_config_from_store_locked(store.clone()).await?;
+            delete_persisted_provider_config(&mut config, &provider_id)?;
+            save_server_config_to_store_locked(store, &config).await
+        })
+        .await
+        .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;

        json_response(
            StatusCode::OK,
@@ -881,23 +898,32 @@ fn json_response<T: Serialize>(status: StatusCode, payload: &T) -> S3Result<S3Re
 }

 async fn load_server_config_from_store() -> S3Result<ServerConfig> {
-    let context = current_app_context();
-    let Some(store) = current_object_store_handle_for_context(context.as_deref()) else {
-        return Err(s3_error!(InternalError, "storage layer not initialized"));
-    };
+    let store = oidc_config_store()?;

    read_admin_config_without_migrate(store)
        .await
        .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}")))
 }

-async fn save_server_config_to_store(config: &ServerConfig) -> S3Result<()> {
+fn oidc_config_store() -> S3Result<std::sync::Arc<crate::admin::storage_api::runtime::ECStore>> {
    let context = current_app_context();
-    let Some(store) = current_object_store_handle_for_context(context.as_deref()) else {
-        return Err(s3_error!(InternalError, "storage layer not initialized"));
-    };
+    current_object_store_handle_for_context(context.as_deref())
+        .ok_or_else(|| s3_error!(InternalError, "storage layer not initialized"))
+}

-    save_admin_server_config(store, config)
+async fn load_server_config_from_store_locked(
+    store: std::sync::Arc<crate::admin::storage_api::runtime::ECStore>,
+) -> S3Result<ServerConfig> {
+    read_admin_config_without_migrate_no_lock(store)
+        .await
+        .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}")))
+}
+
+async fn save_server_config_to_store_locked(
+    store: std::sync::Arc<crate::admin::storage_api::runtime::ECStore>,
+    config: &ServerConfig,
+) -> S3Result<()> {
+    save_admin_server_config_no_lock(store, config)
        .await
        .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to save server config: {e}")))
 }
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
唐小鸭	d69d682193	Refactor code structure for improved readability and maintainability	2026-07-24 14:01:40 +08:00
唐小鸭	5ea9a1fd8f	fix(sse): separate SSE-S3 and KMS key providers	2026-07-22 21:05:23 +08:00
houseme	a044d11443	chore(deps): refresh cargo dependencies (#5132 ) Update selected workspace dependencies and lockfile entries. Keep async-nats and rcgen on explicit feature sets while preserving the RustFS targets and TLS test surfaces. Co-authored-by: heihutu <heihutu@gmail.com>	2026-07-22 21:02:59 +08:00
houseme	e1e6a8b020	fix(tier): gate exact remote version consumption (#5126 ) * fix(tier): gate exact remote version consumption Reject non-empty remote tier versions before transitioned GET and remote delete backend I/O when the tier backend does not support exact version operations. Co-Authored-By: heihutu <heihutu@gmail.com> * test(tier): split mock remote version validation fault Separate one-shot mock remote version validation failures from persistent unsupported-backend behavior so cleanup durability tests can still verify exact-version recovery while #1358 fail-closed gate tests keep asserting no backend I/O. Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>	2026-07-22 11:52:56 +00:00
houseme	5cfe4ccc7d	fix(tier): include persisted refs in tier proof (#5128 ) Extend tier mutation reference proof to cover persisted delete journal, transition transaction, and free-version references with one-pass target matching. Co-authored-by: heihutu <heihutu@gmail.com>	2026-07-22 11:31:03 +00:00
houseme	df2db15ce8	fix(tier): persist unknown upload outcomes (#5127 ) Advance transition transactions to UploadOutcomeUnknown before remote tier PUT so response-loss windows can be recovered through provider-authoritative probing. Co-authored-by: heihutu <heihutu@gmail.com>	2026-07-22 19:14:47 +08:00
houseme	0321e9350d	fix(tier): retain prepared intents after abort failure (#5129 ) Keep coordinator Prepared intents durable when peer abort recovery fails, then make reload retry abort or commit based on the persisted tier config digest. Co-authored-by: heihutu <heihutu@gmail.com>	2026-07-22 19:12:55 +08:00
houseme	cd9a2eecb1	test(tier): fix uring test build fake peer (#5124 ) Co-authored-by: heihutu <heihutu@gmail.com>	2026-07-22 10:03:50 +00:00
houseme	8b09634e62	fix(tier): fan out coordinator mutations to peers (#5122 ) Co-authored-by: heihutu <heihutu@gmail.com>	2026-07-22 17:02:57 +08:00
houseme	1ede77b1c1	fix(ilm): recover unknown transition uploads by probing tier (#5120 ) * fix(ilm): recover unknown transition uploads by probing tier Co-Authored-By: heihutu <heihutu@gmail.com> * style(ilm): format transition recovery assertions Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>	2026-07-22 17:02:54 +08:00
houseme	666e251b78	test(tier): cover prepared intent scan cleanup (#5121 ) Co-authored-by: heihutu <heihutu@gmail.com>	2026-07-22 17:02:44 +08:00
houseme	65ba138c27	test(tier): cover mixed-version committed replay (#5119 ) Co-authored-by: heihutu <heihutu@gmail.com>	2026-07-22 17:02:32 +08:00
Zhengchao An	92ae19b340	fix(targets): isolate invalid target instances instead of failing the whole subsystem (#5118 ) A single instance with a malformed `enable` value (e.g. the typo `enable` instead of `enabled`/`on`) made every configured notification/audit target of every type fail to load. `create_targets_from_config_with_store_mode` collected instance configs through the strict `try_collect_target_configs`, whose `.collect::<Result<Vec, _>>()` short-circuits on the first error and drops all siblings; the surrounding `?` then aborted the loop over every plugin type. This contradicted the function's own documented contract ("Creation is fault-isolated per instance") and was inconsistent: target construction failures were already isolated into the `failures` accumulator, only config collection failures were fatal. Add `collect_target_config_results[_from_env]`, which returns the enabled `(id, config)` pairs plus a per-instance failure summary, and use it in the create path. Per-instance collection errors now flow into the same `failures`/`creation_failures` accumulator as construction failures: healthy siblings and other target types still load, while the malformed instance is surfaced so the notify lifecycle stays non-converged/retryable and an Admin write cannot report a false success. Audit likewise keeps logging through its valid targets instead of going fully Stopped. Follow-up to #5088.	2026-07-22 16:32:16 +08:00
houseme	4607c3be53	fix(tier): recover coordinator mutation intents (#5114 ) Persist coordinator tier mutation intents before config CAS, recover prepared coordinator intents whose candidate digest already matches the saved config, and clean finished coordinator records after local publish. Refs rustfs/backlog#1357 Co-authored-by: heihutu <heihutu@gmail.com>	2026-07-22 07:35:29 +00:00
cxymds	e0bac66941	fix(targets): unify runtime health snapshots (#5110 ) * fix(targets): unify runtime health snapshots * fix(targets): stabilize health snapshot merge * fix(admin): import runtime health test type	2026-07-22 06:50:36 +00:00
houseme	31dc78eab0	feat(tier): probe transition candidates from providers (#5112 ) * feat(tier): add transition candidate probe contract Add a fail-closed WarmBackend probe contract for provider-authoritative transition candidate state. Default providers report Unsupported, while the shared mock backend can now model missing, unversioned, and exact-version candidates for follow-up recovery tests. This is a forward-compatible foundation for #1352/#1358 recovery work and does not change production cleanup behavior. Co-Authored-By: heihutu <heihutu@gmail.com> * feat(tier): probe transition candidates from providers Implement provider-authoritative transition candidate probing for S3-family warm backends by querying ListObjectVersions with exact-key filtering and fail-closed classification for delete markers, multiple versions, truncation, and unknown versioning state. This keeps non-S3 providers on the default Unsupported probe result and forwards MinIO, RustFS, and R2 through the S3 probe implementation. Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>	2026-07-22 06:13:32 +00:00
Zhengchao An	daca7294c7	fix(ecstore): stop logging not-found listing quorum miss at error (#5111 ) A metacache listing that misses quorum purely because the volume or path is absent on a quorum of drives (every drive reports VolumeNotFound or FileNotFound) is a benign, expected outcome: list_path_raw already returns VolumeNotFound/FileNotFound and lets the caller decide how to react. The common trigger is a startup race where the system bucket (.rustfs.sys) is not yet created on every drive when an early reader such as the IAM config loader lists config/iam/. Logging that at error prints a scary message during normal boot and, in #5076, misled a user into blaming it for unrelated upload failures. Add is_benign_not_found_listing_failure() and demote the pure not-found case to debug (state = "quorum_not_found"), keeping error for listings that failed for a real reason (I/O, timeout, corruption). Return value and control flow are unchanged. Add a unit test for the classifier. Refs: #5076	2026-07-22 05:26:09 +00:00
cxymds	1655f3192e	fix(notify): unify runtime lifecycle coordination (#5088 ) * fix(notify): unify runtime lifecycle coordination * fix(notify): repair lifecycle convergence checks * fix(admin): expose effective notify state (#5097)	2026-07-22 05:01:15 +00:00
houseme	0adb3c5ea1	fix(tier): gate inexact provider versions (#5109 ) Bind provider exact GET and DELETE capability to tier operation leases and reject non-empty remote versions before committing transition metadata when the provider cannot address exact versions. Co-authored-by: heihutu <heihutu@gmail.com>	2026-07-22 12:24:05 +08:00
houseme	68e156a5c5	fix(tier): retain unproven transition candidates (#5108 ) Keep LocalCommitStarted transition transaction records retained when the local source cannot prove the commit, so recovery does not classify the record as failed or delete the remote candidate without cleanup proof. Co-authored-by: heihutu <heihutu@gmail.com>	2026-07-22 04:03:33 +00:00
houseme	3f60cc743e	fix(tier): replay committed mutation intents (#5107 ) Keep recovered tier mutation blocks installed until the local publish transition has atomically established draining for the affected tiers, so old-generation leases cannot slip in after peer commit replay and before local publish. Delay committed intent cleanup until local publish succeeds. If peer replay succeeds but local publish fails, the durable committed intent remains available for retry and the runtime block stays fail-closed. Co-authored-by: heihutu <heihutu@gmail.com>	2026-07-22 03:06:55 +00:00
houseme	35af4a611f	fix(tier): fail closed legacy mutations without etag (#5106 ) * fix: fail closed legacy tier mutations without etag Reject non-add tier config mutations when the loaded durable snapshot has no current config ETag. This keeps legacy JSON/no-ETag paths from being saved as creation-style updates before coordinator intent state can bind an old config revision. Co-Authored-By: heihutu <heihutu@gmail.com> * fix(tier): allow empty clear without config etag Allow empty tier clears to initialize the binary config under the namespace coordinator lock while keeping non-empty legacy config mutations fail-closed when no current ETag exists. Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>	2026-07-22 02:02:00 +00:00