Merge branch 'main' into overtrue/activate-group-e2e

* test(kms): bound Vault failover progress wait

* ci(nightly): honor manual dispatch ref

* test(kms): preserve Vault worker failures

* test(kms): validate Vault circuit recovery

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=9f767b37ed8b1c82da62ea441462d75487785c8086e56f08fb6f6cd89c6e2e52
-sha256-linux=fbdaf42b220958d4b1e8880e0f8b5a7992d38e21051bb60596dd4538424757d6
+sha256-darwin=f832043fcca8c0b616c5d820a3a652da7544298ef5812a8668a3a9a3e4607b8b
+sha256-linux=93b94adb110b86a41d0b7313909e0bf53cb1515e2d08e8f105652b29b249990f

.github/workflows/nightly-gnu.yml

@@ -39,11 +39,10 @@ jobs:
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -89,11 +88,10 @@ jobs:
       # either casing.
       NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -178,11 +176,10 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup

crates/e2e_test/src/group_delete_test.rs

@@ -14,7 +14,7 @@
 
 //! E2E tests for group management (fixes #2028).
 
-use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_put, init_logging};
+use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::{Client, Config};
 use tracing::info;
@@ -83,7 +83,6 @@ async fn update_group_members_rejects_invalid_new_group_names() -> Result<(), Bo
 
 /// Test that deleting a group with members fails, and deleting an empty group succeeds.
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "requires awscurl and spawns a real RustFS server"]
 async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
@@ -91,29 +90,58 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
     env.start_rustfs_server(vec![]).await?;
 
     // 1. Create a user
-    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey=testuser1", env.url);
     let user_body = serde_json::json!({
         "secretKey": "testuser1secret",
         "status": "enabled"
     });
-    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/add-user?accessKey=testuser1",
+        Some(user_body.to_string()),
+    )
+    .await?;
     info!("Created testuser1");
 
     // 2. Create a group with testuser1 as a member
-    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
     let add_member_body = serde_json::json!({
         "group": "testgroup",
         "members": ["testuser1"],
         "isRemove": false,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(add_member_body.to_string()),
+    )
+    .await?;
     info!("Added testuser1 to testgroup");
 
     // 3. Attempt to delete the group while it still has members — should fail
-    let delete_group_url = format!("{}/rustfs/admin/v3/group/testgroup", env.url);
-    let delete_result = awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await;
-    assert!(delete_result.is_err(), "deleting a non-empty group should fail");
+    let (delete_status, delete_body) = admin_request(
+        &env.url,
+        http::Method::DELETE,
+        "/rustfs/admin/v3/group/testgroup",
+        None,
+        &env.access_key,
+        &env.secret_key,
+    )
+    .await?;
+    assert_eq!(
+        delete_status,
+        reqwest::StatusCode::BAD_REQUEST,
+        "deleting a non-empty group must return HTTP 400, body: {delete_body}"
+    );
+    assert!(
+        delete_body.contains("<Code>InvalidRequest</Code>"),
+        "deleting a non-empty group must return InvalidRequest, body: {delete_body}"
+    );
+    assert!(
+        delete_body.contains("<Message>group is not empty</Message>"),
+        "deleting a non-empty group returned an unexpected message: {delete_body}"
+    );
     info!("Delete of non-empty group correctly rejected");
 
     // 4. Remove the member from the group
@@ -123,17 +151,42 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
         "isRemove": true,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &remove_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(remove_member_body.to_string()),
+    )
+    .await?;
     info!("Removed testuser1 from testgroup");
 
     // 5. Delete the now-empty group — should succeed
-    awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
+    admin_ok(&env, http::Method::DELETE, "/rustfs/admin/v3/group/testgroup", None).await?;
     info!("Deleted empty testgroup successfully");
 
     // 6. Verify the group no longer exists
-    let get_group_url = format!("{}/rustfs/admin/v3/group?group=testgroup", env.url);
-    let get_result = awscurl_get(&get_group_url, &env.access_key, &env.secret_key).await;
-    assert!(get_result.is_err(), "group should no longer exist after deletion");
+    let (get_status, get_body) = admin_request(
+        &env.url,
+        http::Method::GET,
+        "/rustfs/admin/v3/group?group=testgroup",
+        None,
+        &env.access_key,
+        &env.secret_key,
+    )
+    .await?;
+    assert_eq!(
+        get_status,
+        reqwest::StatusCode::NOT_FOUND,
+        "a deleted group must return HTTP 404, body: {get_body}"
+    );
+    assert!(
+        get_body.contains("<Code>NoSuchResource</Code>"),
+        "a deleted group must return NoSuchResource, body: {get_body}"
+    );
+    assert!(
+        get_body.contains("<Message>group &apos;testgroup&apos; does not exist</Message>"),
+        "a deleted group returned an unexpected message: {get_body}"
+    );
     info!("Confirmed testgroup no longer exists");
 
     Ok(())
@@ -142,7 +195,6 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
 /// Test that a user with only group membership (no explicit user policy) gets group policies
 /// and can perform actions allowed by the group (regression test for #2028.1).
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "requires awscurl and spawns a real RustFS server"]
 async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
@@ -160,39 +212,56 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
         "Statement": [{
             "Effect": "Allow",
             "Action": ["s3:ListAllMyBuckets"],
-            "Resource": ["*"]
+            "Resource": ["arn:aws:s3:::*"]
         }]
     });
-    let add_policy_url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
-    awscurl_put(&add_policy_url, &policy_doc.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"),
+        Some(policy_doc.to_string()),
+    )
+    .await?;
     info!("Created canned policy {}", policy_name);
 
     // 2. Create user with no explicit policy
-    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
     let user_body = serde_json::json!({
         "secretKey": user_secret,
         "status": "enabled"
     });
-    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
+        Some(user_body.to_string()),
+    )
+    .await?;
     info!("Created user {} with no explicit policy", user_name);
 
     // 3. Add user to group (creates group with this member; user_group_memberships must be updated)
-    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
     let add_member_body = serde_json::json!({
         "group": group_name,
         "members": [user_name],
         "isRemove": false,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(add_member_body.to_string()),
+    )
+    .await?;
     info!("Added {} to group {}", user_name, group_name);
 
     // 4. Attach policy to group
-    let set_policy_url = format!(
-        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=true",
-        env.url, policy_name, group_name
-    );
-    awscurl_put(&set_policy_url, "", &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={group_name}&isGroup=true"),
+        Some(String::new()),
+    )
+    .await?;
     info!("Attached policy {} to group {}", policy_name, group_name);
 
     // 5. User with only group (no user policy) should be able to list buckets
@@ -209,7 +278,6 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
 /// Test that after deleting a user who was the only member of a group, the group can be deleted
 /// (regression test for #2028.2: delete group uses backend membership, not stale cache).
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "requires awscurl and spawns a real RustFS server"]
 async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
@@ -221,33 +289,47 @@ async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::erro
     let group_name = "soledeletegroup";
 
     // 1. Create user
-    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
     let user_body = serde_json::json!({
         "secretKey": user_secret,
         "status": "enabled"
     });
-    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
+        Some(user_body.to_string()),
+    )
+    .await?;
     info!("Created user {}", user_name);
 
     // 2. Add user to group
-    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
     let add_member_body = serde_json::json!({
         "group": group_name,
         "members": [user_name],
         "isRemove": false,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(add_member_body.to_string()),
+    )
+    .await?;
     info!("Added {} to group {}", user_name, group_name);
 
     // 3. Delete the user (backend and cache update so group membership becomes empty)
-    let remove_user_url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, user_name);
-    awscurl_delete(&remove_user_url, &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::DELETE,
+        &format!("/rustfs/admin/v3/remove-user?accessKey={user_name}"),
+        None,
+    )
+    .await?;
     info!("Deleted user {}", user_name);
 
     // 4. Deleting the group should succeed (backend has empty members; no stale cache)
-    let delete_group_url = format!("{}/rustfs/admin/v3/group/{}", env.url, group_name);
-    awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
+    admin_ok(&env, http::Method::DELETE, &format!("/rustfs/admin/v3/group/{group_name}"), None).await?;
     info!("Deleted group {} after user was removed", group_name);
 
     Ok(())

crates/filemeta/src/metacache.rs

@@ -21,7 +21,6 @@ use arc_swap::ArcSwapOption;
 use rmp::Marker;
 use serde::{Deserialize, Serialize};
 use std::cmp::Ordering;
-use std::collections::{HashMap, HashSet};
 use std::str::from_utf8;
 use std::{
     fmt::Debug,
@@ -38,13 +37,8 @@ use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 use tokio::spawn;
 use tokio::sync::Mutex;
 use tracing::{debug, warn};
-use uuid::Uuid;
 
 const SLASH_SEPARATOR: &str = "/";
-pub const MAX_META_CACHE_HEAL_CANDIDATES: usize = 1024;
-/// Keep truncation continuations bounded while still giving the scanner a
-/// safe object-level retry for versions that did not fit in the candidate set.
-pub const MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS: usize = 64;
 
 #[derive(Clone, Debug, Default)]
 pub struct MetadataResolutionParams {
@@ -72,50 +66,6 @@ pub struct MetaCacheEntry {
     pub reusable: bool,
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Hash)]
-pub enum MetaCacheHealCandidateKind {
-    Object,
-    DeleteMarker,
-    UnversionedObject,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Hash)]
-pub struct MetaCacheHealCandidate {
-    pub object: String,
-    pub version_id: Option<Uuid>,
-    pub kind: MetaCacheHealCandidateKind,
-    /// Number of raw disk entries that carried this validated version.
-    pub replica_count: usize,
-}
-
-impl MetaCacheHealCandidate {
-    pub fn validated_version(&self) -> Option<Uuid> {
-        match self.kind {
-            MetaCacheHealCandidateKind::Object | MetaCacheHealCandidateKind::DeleteMarker => self.version_id,
-            MetaCacheHealCandidateKind::UnversionedObject => None,
-        }
-    }
-
-    pub fn is_unversioned(&self) -> bool {
-        self.kind == MetaCacheHealCandidateKind::UnversionedObject
-    }
-}
-
-#[derive(Clone, Debug, Default, PartialEq, Eq)]
-pub struct MetaCacheHealDiscovery {
-    pub candidates: Vec<MetaCacheHealCandidate>,
-    pub unverified_count: usize,
-    pub truncated: bool,
-    /// Object names whose validated version set exceeded the candidate cap.
-    /// The scanner retries these names without a version and with destructive
-    /// healing disabled; this is an explicit bounded continuation, not a
-    /// version claim.
-    pub truncated_objects: Vec<String>,
-    /// Validated candidates beyond the main cap, retained with exact version
-    /// identities so callers never fall back to a latest-version request.
-    pub truncated_candidates: Vec<MetaCacheHealCandidate>,
-}
-
 impl MetaCacheEntry {
     pub fn marshal_msg(&self) -> Result<Vec<u8>> {
         let mut wr = Vec::new();
@@ -420,185 +370,6 @@ impl MetaCacheEntries {
         })
     }
 
-    /// Discover validated object/delete-marker versions and safe unversioned
-    /// inspection candidates in the raw entries without applying read quorum.
-    /// This is intentionally separate from [`Self::resolve`]: a sub-quorum
-    /// version is a valid heal target even though it must not participate in
-    /// normal reads or writes.
-    ///
-    /// The validated list is bounded and deduplicated by object, version id,
-    /// and metadata kind; each candidate retains the number of raw disk
-    /// entries that carried it so callers can classify sub-quorum versions.
-    /// Entries whose xl.meta cannot be decoded are counted separately for
-    /// discovery accounting; they never become versionless destructive heal
-    /// requests and do not consume the validated quota. An
-    /// [`MetaCacheHealCandidateKind::UnversionedObject`] is always consumed by
-    /// a non-destructive scanner request.
-    pub fn discover_heal_candidates(&self, bucket: &str, max_candidates: usize) -> MetaCacheHealDiscovery {
-        let limit = max_candidates.min(MAX_META_CACHE_HEAL_CANDIDATES);
-        if limit == 0 || bucket.is_empty() {
-            return MetaCacheHealDiscovery::default();
-        }
-
-        let mut discovery = MetaCacheHealDiscovery {
-            candidates: Vec::<MetaCacheHealCandidate>::with_capacity(limit.min(self.0.len())),
-            unverified_count: 0,
-            truncated: false,
-            truncated_objects: Vec::with_capacity(MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS.min(limit)),
-            truncated_candidates: Vec::new(),
-        };
-        let mut seen: HashMap<(String, Option<Uuid>, MetaCacheHealCandidateKind), usize> =
-            HashMap::with_capacity(limit.min(self.0.len()));
-
-        for entry in self.0.iter().flatten() {
-            if !valid_heal_candidate_name(bucket, entry) {
-                continue;
-            }
-
-            let meta = match FileMeta::load(&entry.metadata) {
-                Ok(meta) => meta,
-                Err(_) => {
-                    discovery.unverified_count = discovery.unverified_count.saturating_add(1);
-                    continue;
-                }
-            };
-            let mut entry_seen = HashSet::new();
-
-            for shallow in meta.versions {
-                let version = match shallow.parse_version_meta() {
-                    Ok(version) if version.valid() => version,
-                    Ok(_) | Err(_) => {
-                        discovery.unverified_count = discovery.unverified_count.saturating_add(1);
-                        continue;
-                    }
-                };
-                if version.free_version() {
-                    continue;
-                }
-
-                let payload_header = version.header();
-                if normalize_version_id(shallow.header.version_id) != normalize_version_id(payload_header.version_id)
-                    || shallow.header.version_type != payload_header.version_type
-                {
-                    discovery.unverified_count = discovery.unverified_count.saturating_add(1);
-                    continue;
-                }
-
-                let (kind, version_id) = match version.version_type {
-                    VersionType::Object
-                        if version.object.is_some() && version.delete_marker.is_none() && version.legacy_object.is_none() =>
-                    {
-                        match version.object.as_ref().and_then(|object| object.version_id) {
-                            Some(id) if !id.is_nil() => (MetaCacheHealCandidateKind::Object, Some(id)),
-                            Some(_) | None => (MetaCacheHealCandidateKind::UnversionedObject, None),
-                        }
-                    }
-                    VersionType::Delete
-                        if version.delete_marker.is_some() && version.object.is_none() && version.legacy_object.is_none() =>
-                    {
-                        let Some(id) = version.delete_marker.as_ref().and_then(|marker| marker.version_id) else {
-                            discovery.unverified_count = discovery.unverified_count.saturating_add(1);
-                            continue;
-                        };
-                        if id.is_nil() {
-                            discovery.unverified_count = discovery.unverified_count.saturating_add(1);
-                            continue;
-                        }
-                        (MetaCacheHealCandidateKind::DeleteMarker, Some(id))
-                    }
-                    VersionType::Legacy
-                        if version.legacy_object.is_some() && version.object.is_none() && version.delete_marker.is_none() =>
-                    {
-                        let Some(legacy) = version.legacy_object.as_ref() else {
-                            continue;
-                        };
-                        if legacy.version_id.is_empty() {
-                            (MetaCacheHealCandidateKind::UnversionedObject, None)
-                        } else {
-                            let Ok(id) = Uuid::parse_str(&legacy.version_id) else {
-                                discovery.unverified_count = discovery.unverified_count.saturating_add(1);
-                                continue;
-                            };
-                            if id.is_nil() {
-                                discovery.unverified_count = discovery.unverified_count.saturating_add(1);
-                                continue;
-                            }
-                            (MetaCacheHealCandidateKind::Object, Some(id))
-                        }
-                    }
-                    _ => {
-                        discovery.unverified_count = discovery.unverified_count.saturating_add(1);
-                        continue;
-                    }
-                };
-
-                if normalize_version_id(payload_header.version_id) != version_id {
-                    discovery.unverified_count = discovery.unverified_count.saturating_add(1);
-                    continue;
-                }
-
-                // `all_parts=true` is the trust-boundary check for versioned
-                // candidates. A null/legacy object may still need the old
-                // non-destructive inspection fallback when its part arrays
-                // are parseable but incomplete; never use that fallback for
-                // a candidate carrying a real version id.
-                let file_info = match version.clone().into_fileinfo(bucket, &entry.name, true) {
-                    Ok(file_info) => file_info,
-                    Err(_) if version_id.is_none() && matches!(kind, MetaCacheHealCandidateKind::UnversionedObject) => {
-                        discovery.unverified_count = discovery.unverified_count.saturating_add(1);
-                        match version.into_fileinfo(bucket, &entry.name, false) {
-                            Ok(file_info) => file_info,
-                            Err(_) => continue,
-                        }
-                    }
-                    Err(_) => {
-                        discovery.unverified_count = discovery.unverified_count.saturating_add(1);
-                        continue;
-                    }
-                };
-                if file_info.volume != bucket || file_info.name != entry.name {
-                    discovery.unverified_count = discovery.unverified_count.saturating_add(1);
-                    continue;
-                }
-
-                let candidate = MetaCacheHealCandidate {
-                    object: entry.name.clone(),
-                    version_id,
-                    kind,
-                    replica_count: 1,
-                };
-                let key = (candidate.object.clone(), candidate.version_id, candidate.kind.clone());
-                if entry_seen.contains(&key) {
-                    continue;
-                }
-                if let Some(index) = seen.get(&key).copied() {
-                    entry_seen.insert(key);
-                    discovery.candidates[index].replica_count = discovery.candidates[index].replica_count.saturating_add(1);
-                } else if discovery.candidates.len() >= limit {
-                    // Keep the validated candidate list bounded, but retain a
-                    // bounded object-level continuation so the scanner cannot
-                    // silently lose every version of a busy object.
-                    discovery.truncated = true;
-                    if discovery.truncated_objects.len() < MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS
-                        && !discovery.truncated_objects.iter().any(|object| object == &candidate.object)
-                    {
-                        discovery.truncated_objects.push(candidate.object.clone());
-                    }
-                    if discovery.truncated_objects.iter().any(|object| object == &candidate.object) {
-                        discovery.truncated_candidates.push(candidate);
-                    }
-                    continue;
-                } else {
-                    entry_seen.insert(key.clone());
-                    seen.insert(key, discovery.candidates.len());
-                    discovery.candidates.push(candidate);
-                }
-            }
-        }
-
-        discovery
-    }
-
     fn resolve_inner(&self, mut params: MetadataResolutionParams, enforce_write_quorum: bool) -> Option<MetaCacheEntry> {
         if self.0.is_empty() {
             debug!(
@@ -775,33 +546,6 @@ impl MetaCacheEntries {
     }
 }
 
-fn valid_heal_candidate_name(bucket: &str, entry: &MetaCacheEntry) -> bool {
-    if bucket.is_empty()
-        || entry.name.is_empty()
-        || entry.is_dir()
-        || (cfg!(windows) && entry.name.contains('\\'))
-        || entry.name.chars().any(char::is_control)
-    {
-        return false;
-    }
-
-    // Validate raw key components without normalizing them. The scanner maps
-    // accepted keys to filesystem paths later, so dot components and empty
-    // internal components must be rejected before that boundary. A final
-    // empty component is retained for valid keys ending in '/'.
-    let mut components = entry.name.split('/').peekable();
-    while let Some(component) = components.next() {
-        if component == "." || component == ".." || (component.is_empty() && components.peek().is_some()) {
-            return false;
-        }
-    }
-    true
-}
-
-fn normalize_version_id(version_id: Option<Uuid>) -> Option<Uuid> {
-    version_id.filter(|id| !id.is_nil())
-}
-
 #[derive(Debug, Default)]
 pub struct MetaCacheEntriesSortedResult {
     pub entries: Option<MetaCacheEntriesSorted>,
@@ -1247,7 +991,7 @@ impl<T: Clone + Debug + Send + Sync + 'static> Cache<T> {
 mod tests {
     use super::*;
     use crate::test_data::create_real_xlmeta;
-    use crate::{FileMetaVersion, MetaDeleteMarker, MetaObjectV1, MetaObjectV1Erasure, MetaObjectV1Stat, TRANSITION_COMPLETE};
+    use crate::{FileMetaVersion, MetaDeleteMarker, TRANSITION_COMPLETE};
     use std::collections::HashMap;
     use std::io::Cursor;
     use std::sync::{
@@ -1848,381 +1592,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn discover_heal_candidates_keeps_sub_quorum_versions_and_deduplicates() {
-        let now = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
-        let entries = MetaCacheEntries(vec![
-            Some(metacache_entry_single_version(1, now, "one")),
-            Some(metacache_entry_single_version(2, now, "two")),
-            Some(metacache_entry_single_version(2, now, "two")),
-            Some(metacache_entry_single_version(3, now, "three")),
-        ]);
-
-        let discovery = entries.discover_heal_candidates("bucket", 16);
-        let ids: std::collections::HashSet<Uuid> = discovery
-            .candidates
-            .iter()
-            .filter_map(|candidate| candidate.version_id)
-            .collect();
-        assert_eq!(
-            ids,
-            [Uuid::from_u128(1), Uuid::from_u128(2), Uuid::from_u128(3)]
-                .into_iter()
-                .collect()
-        );
-        assert_eq!(discovery.candidates.len(), 3, "duplicate tied versions must be emitted once");
-        assert_eq!(
-            discovery
-                .candidates
-                .iter()
-                .find(|candidate| candidate.version_id == Some(Uuid::from_u128(2)))
-                .expect("duplicate version should be discovered")
-                .replica_count,
-            2
-        );
-    }
-
-    #[test]
-    fn discover_heal_candidates_does_not_count_duplicate_versions_within_one_entry() {
-        let now = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
-        let mut meta = FileMeta::load(&metacache_entry_single_version(1, now, "duplicate").metadata)
-            .expect("duplicate fixture should decode");
-        meta.versions.push(meta.versions[0].clone());
-        let entry = MetaCacheEntry {
-            name: "object".to_string(),
-            metadata: meta.marshal_msg().expect("duplicate metadata should marshal"),
-            cached: Some(meta),
-            reusable: false,
-        };
-
-        let discovery = MetaCacheEntries(vec![Some(entry)]).discover_heal_candidates("bucket", 16);
-        let candidate = discovery
-            .candidates
-            .iter()
-            .find(|candidate| candidate.version_id == Some(Uuid::from_u128(1)))
-            .expect("duplicate fixture should be discovered");
-        assert_eq!(candidate.replica_count, 1);
-    }
-
-    #[test]
-    fn discover_heal_candidates_covers_divergent_quorum_boundaries_n2_n4_n6() {
-        let now = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
-
-        for (disk_count, quorum) in [(2usize, 1usize), (4, 2), (6, 3)] {
-            let target_id = Uuid::from_u128(0x1000 + disk_count as u128);
-            for target_replicas in [quorum.saturating_sub(1), quorum, quorum + 1] {
-                let entries = (0..disk_count)
-                    .map(|disk| {
-                        let version_id = if disk < target_replicas {
-                            target_id
-                        } else {
-                            Uuid::from_u128(0x2000 + disk as u128)
-                        };
-                        Some(metacache_entry_single_version(version_id.as_u128(), now, "divergent"))
-                    })
-                    .collect();
-                let discovery = MetaCacheEntries(entries).discover_heal_candidates("bucket", 32);
-                let target = discovery
-                    .candidates
-                    .iter()
-                    .find(|candidate| candidate.version_id == Some(target_id));
-                assert_eq!(target.is_some(), target_replicas > 0, "N={disk_count}, replicas={target_replicas}");
-                if let Some(target) = target {
-                    assert_eq!(target.replica_count, target_replicas);
-                }
-            }
-        }
-    }
-
-    #[test]
-    fn discover_heal_candidates_separates_delete_markers_and_preserves_unversioned_objects() {
-        let mut marker_meta = FileMeta::new();
-        marker_meta
-            .add_version(FileInfo {
-                volume: "bucket".to_string(),
-                name: "object".to_string(),
-                version_id: Some(Uuid::from_u128(99)),
-                deleted: true,
-                mod_time: Some(OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp")),
-                ..Default::default()
-            })
-            .expect("delete marker should be added");
-        let marker = MetaCacheEntry {
-            name: "object".to_string(),
-            metadata: marker_meta.marshal_msg().expect("delete marker metadata should marshal"),
-            cached: Some(marker_meta),
-            reusable: false,
-        };
-
-        let unversioned_entry = metacache_entry_with_mod_time(
-            OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"),
-            "unversioned",
-        );
-        let discovery = MetaCacheEntries(vec![Some(marker), Some(unversioned_entry)]).discover_heal_candidates("bucket", 16);
-        assert!(discovery.candidates.iter().any(|candidate| {
-            candidate.kind == MetaCacheHealCandidateKind::DeleteMarker && candidate.version_id == Some(Uuid::from_u128(99))
-        }));
-        assert!(discovery.candidates.iter().any(|candidate| {
-            candidate.kind == MetaCacheHealCandidateKind::UnversionedObject && candidate.version_id.is_none()
-        }));
-    }
-
-    #[test]
-    fn discover_heal_candidates_rejects_delete_markers_without_ids() {
-        let mut marker_meta = FileMeta::new();
-        marker_meta
-            .add_version(FileInfo {
-                volume: "bucket".to_string(),
-                name: "object".to_string(),
-                deleted: true,
-                mod_time: Some(OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp")),
-                ..Default::default()
-            })
-            .expect("nil delete marker should be added");
-        let discovery = MetaCacheEntries(vec![Some(MetaCacheEntry {
-            name: "object".to_string(),
-            metadata: marker_meta.marshal_msg().expect("nil marker metadata should marshal"),
-            cached: Some(marker_meta),
-            reusable: false,
-        })])
-        .discover_heal_candidates("bucket", 16);
-
-        assert!(
-            !discovery
-                .candidates
-                .iter()
-                .any(|candidate| candidate.kind == MetaCacheHealCandidateKind::DeleteMarker)
-        );
-        assert!(discovery.unverified_count >= 1);
-    }
-
-    #[test]
-    fn discover_heal_candidates_skips_free_versions() {
-        let object_id = Uuid::from_u128(100);
-        let free_id = Uuid::from_u128(101);
-        let mut meta = FileMeta::new();
-        meta.add_version(FileInfo {
-            volume: "bucket".to_string(),
-            name: "object".to_string(),
-            version_id: Some(object_id),
-            transition_status: TRANSITION_COMPLETE.to_string(),
-            transitioned_objname: "remote/object".to_string(),
-            transition_version_id: Some(Uuid::from_u128(102)),
-            transition_tier: "WARM".to_string(),
-            mod_time: Some(OffsetDateTime::now_utc()),
-            ..Default::default()
-        })
-        .expect("transitioned object should be added");
-        let mut delete = FileInfo {
-            volume: "bucket".to_string(),
-            name: "object".to_string(),
-            version_id: Some(object_id),
-            mod_time: Some(OffsetDateTime::now_utc()),
-            ..Default::default()
-        };
-        delete.set_tier_free_version_id(&free_id.to_string());
-        meta.delete_version(&delete).expect("free version should be persisted");
-
-        let discovery = MetaCacheEntries(vec![Some(MetaCacheEntry {
-            name: "object".to_string(),
-            metadata: meta.marshal_msg().expect("free version metadata should marshal"),
-            cached: Some(meta),
-            reusable: false,
-        })])
-        .discover_heal_candidates("bucket", 16);
-        assert!(discovery.candidates.is_empty());
-    }
-
-    #[test]
-    fn discover_heal_candidates_preserves_unversioned_legacy_object() {
-        let legacy = MetaObjectV1 {
-            version: "1.0.1".to_string(),
-            format: "xl".to_string(),
-            stat: MetaObjectV1Stat {
-                size: 1,
-                mod_time: Some(OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp")),
-                name: "object".to_string(),
-                ..Default::default()
-            },
-            erasure: MetaObjectV1Erasure {
-                data_blocks: 4,
-                parity_blocks: 2,
-                index: 1,
-                distribution: vec![1, 2, 3, 4, 5, 6],
-                ..Default::default()
-            },
-            ..Default::default()
-        };
-        let version = FileMetaVersion {
-            version_type: VersionType::Legacy,
-            legacy_object: Some(legacy),
-            ..Default::default()
-        };
-        let mut meta = FileMeta::new();
-        meta.versions
-            .push(FileMetaShallowVersion::try_from(version).expect("legacy metadata should marshal"));
-        let discovery = MetaCacheEntries(vec![Some(MetaCacheEntry {
-            name: "object".to_string(),
-            metadata: meta.marshal_msg().expect("legacy metadata should marshal"),
-            cached: Some(meta),
-            reusable: false,
-        })])
-        .discover_heal_candidates("bucket", 16);
-        assert!(discovery.candidates.iter().any(|candidate| {
-            candidate.kind == MetaCacheHealCandidateKind::UnversionedObject && candidate.version_id.is_none()
-        }));
-    }
-
-    #[test]
-    fn discover_heal_candidates_rejects_nil_and_malformed_metadata_and_is_bounded() {
-        let now = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
-        let mut nil = metacache_entry_single_version(1, now, "nil");
-        let mut nil_meta = FileMeta::load(&nil.metadata).expect("nil fixture should decode");
-        let mut nil_version = nil_meta.versions[0]
-            .parse_version_meta()
-            .expect("nil fixture version should decode");
-        nil_version.object.as_mut().expect("object fixture").version_id = Some(Uuid::nil());
-        nil_meta.versions[0] = FileMetaShallowVersion::try_from(nil_version).expect("nil fixture should marshal");
-        nil.metadata = nil_meta.marshal_msg().expect("nil fixture metadata should marshal");
-
-        let mut mismatched = metacache_entry_single_version(2, now, "mismatched");
-        let mut mismatched_meta = FileMeta::load(&mismatched.metadata).expect("mismatched fixture should decode");
-        mismatched_meta.versions[0].header.version_id = Some(Uuid::from_u128(200));
-        mismatched.metadata = mismatched_meta.marshal_msg().expect("mismatched metadata should marshal");
-
-        let mut short_parts = metacache_entry_single_version(3, now, "short-parts");
-        let mut short_parts_meta = FileMeta::load(&short_parts.metadata).expect("short-parts fixture should decode");
-        let mut short_parts_version = short_parts_meta.versions[0]
-            .parse_version_meta()
-            .expect("short-parts fixture version should decode");
-        let object = short_parts_version.object.as_mut().expect("object fixture");
-        object.part_numbers = vec![1];
-        object.part_actual_sizes = vec![1];
-        object.part_sizes.clear();
-        short_parts_meta.versions[0] = FileMetaShallowVersion::try_from(short_parts_version).expect("short-parts should marshal");
-        short_parts.metadata = short_parts_meta.marshal_msg().expect("short-parts metadata should marshal");
-
-        let mut short_unversioned = metacache_entry_with_mod_time(now, "short-unversioned");
-        let mut short_unversioned_meta =
-            FileMeta::load(&short_unversioned.metadata).expect("short-unversioned fixture should decode");
-        let mut short_unversioned_version = short_unversioned_meta.versions[0]
-            .parse_version_meta()
-            .expect("short-unversioned version should decode");
-        let unversioned_object = short_unversioned_version.object.as_mut().expect("unversioned object fixture");
-        unversioned_object.part_numbers = vec![1];
-        unversioned_object.part_actual_sizes = vec![1];
-        unversioned_object.part_sizes.clear();
-        short_unversioned_meta.versions[0] =
-            FileMetaShallowVersion::try_from(short_unversioned_version).expect("short-unversioned should marshal");
-        short_unversioned.metadata = short_unversioned_meta
-            .marshal_msg()
-            .expect("short-unversioned metadata should marshal");
-
-        let mut malformed = nil.clone();
-        malformed.name = "malformed".to_string();
-        malformed.metadata = vec![1, 2, 3];
-
-        let entries = MetaCacheEntries(
-            std::iter::once(Some(nil))
-                .chain(std::iter::once(Some(mismatched)))
-                .chain(std::iter::once(Some(short_parts)))
-                .chain(std::iter::once(Some(short_unversioned)))
-                .chain(std::iter::once(Some(malformed)))
-                .chain((0..32).map(|id| Some(metacache_entry_single_version(id + 10, now, "bounded"))))
-                .collect(),
-        );
-        let discovery = entries.discover_heal_candidates("bucket", 5);
-        assert!(discovery.candidates.len() <= 5);
-        assert!(discovery.truncated, "bounded discovery must expose dropped candidates");
-        assert!(
-            discovery
-                .truncated_candidates
-                .iter()
-                .all(|candidate| candidate.version_id.is_some()),
-            "overflow candidates must retain exact version identities"
-        );
-        assert!(
-            discovery.truncated_objects.iter().any(|object| object == "object"),
-            "bounded discovery must expose an object-level safe continuation"
-        );
-        assert!(
-            !discovery
-                .candidates
-                .iter()
-                .any(|candidate| candidate.version_id == Some(Uuid::nil()))
-        );
-        assert!(
-            !discovery
-                .candidates
-                .iter()
-                .any(|candidate| candidate.version_id == Some(Uuid::from_u128(2)))
-        );
-        assert!(
-            !discovery
-                .candidates
-                .iter()
-                .any(|candidate| candidate.version_id == Some(Uuid::from_u128(3)))
-        );
-        assert!(discovery.candidates.iter().any(|candidate| {
-            candidate.kind == MetaCacheHealCandidateKind::UnversionedObject && candidate.version_id.is_none()
-        }));
-        assert!(
-            discovery.unverified_count >= 1,
-            "malformed and rejected metadata must remain observable during discovery"
-        );
-
-        for invalid_name in [
-            "../object",
-            "./object",
-            "object/../other",
-            "object//name",
-            "object\u{0001}name",
-            "object\0name",
-        ] {
-            let mut entry = metacache_entry_single_version(400, now, invalid_name);
-            entry.name = invalid_name.to_string();
-            let discovery = MetaCacheEntries(vec![Some(entry)]).discover_heal_candidates("bucket", 5);
-            assert!(
-                discovery.candidates.is_empty(),
-                "invalid key should not become a heal candidate: {invalid_name:?}"
-            );
-        }
-
-        #[cfg(windows)]
-        {
-            let mut entry = metacache_entry_single_version(400, now, "object\\name");
-            entry.name = "object\\name".to_string();
-            assert!(
-                MetaCacheEntries(vec![Some(entry)])
-                    .discover_heal_candidates("bucket", 5)
-                    .candidates
-                    .is_empty(),
-                "backslash is a path separator on Windows"
-            );
-        }
-
-        #[cfg(not(windows))]
-        {
-            let mut entry = metacache_entry_single_version(400, now, "object\\name");
-            entry.name = "object\\name".to_string();
-            assert_eq!(
-                MetaCacheEntries(vec![Some(entry)])
-                    .discover_heal_candidates("bucket", 5)
-                    .candidates
-                    .len(),
-                1,
-                "backslash is object-key data on Unix"
-            );
-        }
-
-        for valid_name in ["trailing/", "prefix/object"] {
-            let mut entry = metacache_entry_single_version(401, now, valid_name);
-            entry.name = valid_name.to_string();
-            let discovery = MetaCacheEntries(vec![Some(entry)]).discover_heal_candidates("bucket", 5);
-            assert_eq!(discovery.candidates.len(), 1, "raw S3 key should remain opaque: {valid_name:?}");
-        }
-    }
-
     #[test]
     fn resolve_rejects_partial_latest_and_returns_committed_previous_metadata() {
         let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");

crates/kms/tests/vault_ha_failover_live.rs

@@ -16,14 +16,14 @@
 //!
 //! `scripts/test/vault_ha_kms_live.sh` owns the official Vault containers and
 //! kills the active node while this test continuously decrypts through a
-//! surviving standby. KV2 and Transit requests must remain successful, use a
-//! bounded number of attempts, and leave the circuit and in-flight gauges at
-//! zero after a new leader is elected.
+//! surviving standby. KV2 and Transit must recover after the bounded circuit
+//! interval, use a bounded number of attempts, and leave the circuit and
+//! in-flight gauges at zero after a new leader is elected.
 
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
-use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
 use metrics_util::MetricKind;
@@ -43,6 +43,11 @@ const OPERATION_ATTEMPTS: &str = "rustfs_kms_backend_operation_attempts";
 const IN_FLIGHT: &str = "rustfs_kms_backend_in_flight";
 const CIRCUIT_OPEN: &str = "rustfs_kms_backend_circuit_open";
 const MAX_ATTEMPTS: u32 = 10;
+const ATTEMPT_TIMEOUT: Duration = Duration::from_secs(2);
+const HEALTHY_PROGRESS_TIMEOUT: Duration = Duration::from_secs(20);
+// The circuit remains open for 30s after five failed attempts.
+const POST_FAILOVER_PROGRESS_TIMEOUT: Duration = Duration::from_secs(35);
+const FAILOVER_ERROR_POLL_INTERVAL: Duration = Duration::from_millis(100);
 
 type MetricEntry = (
     metrics_util::CompositeKey,
@@ -64,7 +69,7 @@ fn config(backend: KmsBackend, backend_config: BackendConfig) -> KmsConfig {
         backend,
         backend_config,
         allow_insecure_dev_defaults: true,
-        timeout: Duration::from_secs(2),
+        timeout: ATTEMPT_TIMEOUT,
         retry_attempts: MAX_ATTEMPTS,
         enable_cache: false,
         ..KmsConfig::default()
@@ -164,14 +169,31 @@ fn retryable_failures(snapshot: &[MetricEntry], operation: &str) -> u64 {
         .sum()
 }
 
-async fn wait_for_count(counter: &AtomicU64, minimum: u64, description: &str) {
-    tokio::time::timeout(Duration::from_secs(20), async {
+async fn wait_for_count(
+    counter: &AtomicU64,
+    failure: &Mutex<Option<String>>,
+    minimum: u64,
+    description: &str,
+    timeout: Duration,
+) {
+    tokio::time::timeout(timeout, async {
         while counter.load(Ordering::SeqCst) < minimum {
+            if let Some(error) = failure.lock().expect("decrypt failure lock poisoned").as_ref() {
+                panic!(
+                    "{description} worker failed after {} successful decrypts: {error}",
+                    counter.load(Ordering::SeqCst)
+                );
+            }
             tokio::time::sleep(Duration::from_millis(25)).await;
         }
     })
     .await
-    .unwrap_or_else(|_| panic!("timed out waiting for {description}"));
+    .unwrap_or_else(|_| {
+        panic!(
+            "timed out after {timeout:?} waiting for {description}: completed {}, expected {minimum}",
+            counter.load(Ordering::SeqCst)
+        )
+    });
 }
 
 async fn wait_for_file(path: &Path, description: &str) {
@@ -189,7 +211,8 @@ async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
     request: DecryptRequest,
     expected: Vec<u8>,
     completed: Arc<AtomicU64>,
-    failed: Arc<AtomicBool>,
+    allow_failover_errors: Arc<AtomicBool>,
+    failure: Arc<Mutex<Option<String>>>,
     stop: CancellationToken,
 ) {
     while !stop.is_cancelled() {
@@ -197,8 +220,18 @@ async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
             Ok(response) if response.plaintext == expected => {
                 completed.fetch_add(1, Ordering::SeqCst);
             }
-            Ok(_) | Err(_) => {
-                failed.store(true, Ordering::SeqCst);
+            Ok(_) => {
+                *failure.lock().expect("decrypt failure lock poisoned") =
+                    Some("decrypt returned unexpected plaintext".to_string());
+                return;
+            }
+            Err(rustfs_kms::KmsError::BackendError { .. } | rustfs_kms::KmsError::OperationTimedOut { .. })
+                if allow_failover_errors.load(Ordering::SeqCst) =>
+            {
+                tokio::time::sleep(FAILOVER_ERROR_POLL_INTERVAL).await;
+            }
+            Err(error) => {
+                *failure.lock().expect("decrypt failure lock poisoned") = Some(error.to_string());
                 return;
             }
         }
@@ -296,7 +329,9 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
     );
 
     let stop = CancellationToken::new();
-    let failed = Arc::new(AtomicBool::new(false));
+    let allow_failover_errors = Arc::new(AtomicBool::new(false));
+    let kv2_failure = Arc::new(Mutex::new(None));
+    let transit_failure = Arc::new(Mutex::new(None));
     let kv2_completed = Arc::new(AtomicU64::new(0));
     let transit_completed = Arc::new(AtomicU64::new(0));
     let kv2_worker = tokio::spawn(decrypt_loop(
@@ -304,7 +339,8 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
         kv2_request,
         kv2_data_key.plaintext_key,
         Arc::clone(&kv2_completed),
-        Arc::clone(&failed),
+        Arc::clone(&allow_failover_errors),
+        Arc::clone(&kv2_failure),
         stop.clone(),
     ));
     let transit_worker = tokio::spawn(decrypt_loop(
@@ -312,12 +348,21 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
         transit_request,
         transit_data_key.plaintext_key,
         Arc::clone(&transit_completed),
-        Arc::clone(&failed),
+        Arc::clone(&allow_failover_errors),
+        Arc::clone(&transit_failure),
         stop.clone(),
     ));
 
-    wait_for_count(&kv2_completed, 2, "two healthy KV2 decrypts").await;
-    wait_for_count(&transit_completed, 2, "two healthy Transit decrypts").await;
+    wait_for_count(&kv2_completed, &kv2_failure, 2, "two healthy KV2 decrypts", HEALTHY_PROGRESS_TIMEOUT).await;
+    wait_for_count(
+        &transit_completed,
+        &transit_failure,
+        2,
+        "two healthy Transit decrypts",
+        HEALTHY_PROGRESS_TIMEOUT,
+    )
+    .await;
+    allow_failover_errors.store(true, Ordering::SeqCst);
     std::fs::write(&marker, b"ready").expect("publish failover readiness marker");
 
     wait_for_file(&elected, "the replacement Vault leader").await;
@@ -326,18 +371,39 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
 
     let kv2_after_election = kv2_completed.load(Ordering::SeqCst) + 2;
     let transit_after_election = transit_completed.load(Ordering::SeqCst) + 2;
-    wait_for_count(&kv2_completed, kv2_after_election, "post-failover KV2 decrypts").await;
-    wait_for_count(&transit_completed, transit_after_election, "post-failover Transit decrypts").await;
+    wait_for_count(
+        &kv2_completed,
+        &kv2_failure,
+        kv2_after_election,
+        "post-failover KV2 decrypts",
+        POST_FAILOVER_PROGRESS_TIMEOUT,
+    )
+    .await;
+    wait_for_count(
+        &transit_completed,
+        &transit_failure,
+        transit_after_election,
+        "post-failover Transit decrypts",
+        POST_FAILOVER_PROGRESS_TIMEOUT,
+    )
+    .await;
 
     stop.cancel();
     kv2_worker.await.expect("KV2 decrypt worker must join");
     transit_worker.await.expect("Transit decrypt worker must join");
-    assert!(!failed.load(Ordering::SeqCst), "no decrypt may fail or return different plaintext");
+    assert!(
+        kv2_failure.lock().expect("KV2 failure lock poisoned").is_none(),
+        "no KV2 decrypt may fail or return different plaintext"
+    );
+    assert!(
+        transit_failure.lock().expect("Transit failure lock poisoned").is_none(),
+        "no Transit decrypt may fail or return different plaintext"
+    );
 }
 
 #[test]
 #[ignore = "requires a real three-node Vault Raft cluster; run scripts/test/vault_ha_kms_live.sh"]
-fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() {
+fn vault_raft_leader_failure_recovers_kv2_and_transit_decrypts() {
     let recorder = DebuggingRecorder::new();
     let snapshotter = recorder.snapshotter();
     metrics::with_local_recorder(&recorder, || {
@@ -349,11 +415,6 @@ fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() {
     });
     let snapshot = snapshotter.snapshot().into_vec();
 
-    assert_eq!(
-        counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "circuit_open")]),
-        0,
-        "a bounded leader election must not open the circuit"
-    );
     assert_eq!(
         counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]),
         0,

crates/scanner/src/scanner_folder.rs

@@ -43,10 +43,7 @@ use rustfs_common::metrics::{
     UpdateCurrentPathFn, current_path_updater, global_metrics,
 };
 use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit, trace_subscriber_count};
-use rustfs_filemeta::{
-    MAX_META_CACHE_HEAL_CANDIDATES, MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS, MetaCacheEntries, MetaCacheEntry,
-    MetaCacheHealCandidateKind,
-};
+use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
 use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
 use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, VersioningConfiguration};
 use time::OffsetDateTime;
@@ -99,11 +96,6 @@ const METRIC_SCANNER_EXCESS_OBJECT_VERSION_SIZE_TOTAL: &str = "rustfs_scanner_ex
 const METRIC_SCANNER_EXCESS_FOLDERS_TOTAL: &str = "rustfs_scanner_excess_folders_total";
 const METRIC_SCANNER_PENDING_HEAL_PRUNE_TOTAL: &str = "rustfs_scanner_pending_heal_prune_total";
 const METRIC_SCANNER_PENDING_HEAL_MALFORMED_TOTAL: &str = "rustfs_scanner_pending_heal_malformed_total";
-const METRIC_SCANNER_HEAL_DISCOVERY_CANDIDATES_TOTAL: &str = "rustfs_scanner_heal_discovery_candidates_total";
-const METRIC_SCANNER_HEAL_DISCOVERY_SUB_QUORUM_TOTAL: &str = "rustfs_scanner_heal_discovery_sub_quorum_total";
-const METRIC_SCANNER_HEAL_DISCOVERY_UNVERIFIED_TOTAL: &str = "rustfs_scanner_heal_discovery_unverified_total";
-const METRIC_SCANNER_HEAL_DISCOVERY_QUEUED_TOTAL: &str = "rustfs_scanner_heal_discovery_queued_total";
-const METRIC_SCANNER_HEAL_DISCOVERY_TRUNCATED_TOTAL: &str = "rustfs_scanner_heal_discovery_truncated_total";
 const MAX_PENDING_SCANNER_HEAL_RETRIES_PER_BUCKET: usize = 128;
 
 // --- scanner excess alerts as S3 notification events (rustfs/backlog#1868) --
@@ -891,7 +883,7 @@ impl FolderScanner {
         object: Option<String>,
         version_id: Option<String>,
         request: HealChannelRequest,
-    ) -> Result<HealAdmissionResult, ScannerError> {
+    ) -> Result<(), ScannerError> {
         let candidate_type = pending_scanner_heal_candidate_type(kind);
         let priority = request.priority;
         let scan_mode = request.scan_mode.unwrap_or(self.scan_mode);
@@ -919,7 +911,7 @@ impl FolderScanner {
                     error = %err,
                     "Scanner deferred heal request after channel error"
                 );
-                return Ok(HealAdmissionResult::Full);
+                return Ok(());
             }
         };
         self.update_pending_scanner_heal_after_admission(
@@ -931,7 +923,7 @@ impl FolderScanner {
             result,
         );
         if result.is_admitted() {
-            return Ok(result);
+            return Ok(());
         }
 
         record_high_priority_heal_escalation(candidate_type, priority, result);
@@ -952,7 +944,7 @@ impl FolderScanner {
             state = "high_priority_not_admitted",
             "Scanner high-priority heal admission failed"
         );
-        Ok(result)
+        Ok(())
     }
 
     pub fn set_heal_object_select(&mut self, prob: u32) {
@@ -1744,7 +1736,14 @@ impl FolderScanner {
                 break;
             }
 
-            let mut previous_bucket = String::new();
+            let mut resolver = MetadataResolutionParams {
+                dir_quorum: self.disks_quorum,
+                obj_quorum: self.disks_quorum,
+                bucket: "".to_string(),
+                strict: false,
+                ..Default::default()
+            };
+
             for name in abandoned_children {
                 if !self.should_heal().await {
                     break;
@@ -1752,7 +1751,7 @@ impl FolderScanner {
 
                 let (bucket, prefix) = path2_bucket_object(name.as_str());
 
-                if bucket != previous_bucket {
+                if bucket != resolver.bucket {
                     self.send_required_scanner_heal_request(
                         PendingScannerHealKind::Bucket,
                         bucket.clone(),
@@ -1761,9 +1760,10 @@ impl FolderScanner {
                         build_bucket_heal_request(bucket.clone(), HealChannelPriority::High),
                     )
                     .await?;
-                    previous_bucket = bucket.clone();
                 }
 
+                resolver.bucket = bucket.clone();
+
                 let child_ctx = ctx.child_token();
 
                 let (agreed_tx, mut agreed_rx) = mpsc::channel::<String>(1);
@@ -1880,8 +1880,6 @@ impl FolderScanner {
                 let mut agreed_closed = false;
                 let mut partial_closed = false;
                 let mut finished_closed = false;
-                let mut seen_heal_candidates: HashSet<(String, Option<String>, MetaCacheHealCandidateKind)> = HashSet::new();
-                let mut seen_truncated_objects: HashSet<String> = HashSet::new();
 
                 loop {
                     if agreed_closed && partial_closed && finished_closed {
@@ -1906,112 +1904,65 @@ impl FolderScanner {
                                 break;
                             }
 
-                            let discovery = entries.discover_heal_candidates(&bucket, MAX_META_CACHE_HEAL_CANDIDATES);
-                            counter!(METRIC_SCANNER_HEAL_DISCOVERY_CANDIDATES_TOTAL)
-                                .increment(u64::try_from(discovery.candidates.len()).unwrap_or(u64::MAX));
-                            counter!(METRIC_SCANNER_HEAL_DISCOVERY_SUB_QUORUM_TOTAL).increment(
-                                u64::try_from(
-                                    discovery
-                                        .candidates
-                                        .iter()
-                                        .filter(|candidate| candidate.replica_count < disks_quorum)
-                                        .count(),
-                                )
-                                .unwrap_or(u64::MAX),
-                            );
-                            counter!(METRIC_SCANNER_HEAL_DISCOVERY_UNVERIFIED_TOTAL).increment(
-                                u64::try_from(discovery.unverified_count).unwrap_or(u64::MAX),
-                            );
-                            if discovery.truncated {
-                                counter!(METRIC_SCANNER_HEAL_DISCOVERY_TRUNCATED_TOTAL).increment(1);
+                            let Some(entry) = resolve_object_heal_entry(&entries, resolver.clone()) else {
+                                continue;
+                            };
+
+                            (self.update_current_path)(&entry.name).await;
+
+                            if entry.is_dir() {
+                                continue;
                             }
 
-                            for candidate in discovery.candidates {
-                                let sub_quorum_candidate = candidate.replica_count < disks_quorum;
-                                let version_id = candidate.validated_version().map(|id| id.to_string());
-                                let identity = (candidate.object.clone(), version_id.clone(), candidate.kind.clone());
-                                if seen_heal_candidates.len() >= MAX_META_CACHE_HEAL_CANDIDATES
-                                    && !seen_heal_candidates.contains(&identity)
-                                {
-                                    continue;
-                                }
-                                if !seen_heal_candidates.insert(identity) {
-                                    continue;
-                                }
-                                let request = if candidate.is_unversioned() {
-                                    build_non_destructive_object_heal_request(
-                                        bucket.clone(),
-                                        candidate.object.clone(),
-                                        self.scan_mode,
-                                        HealChannelPriority::High,
-                                    )
-                                } else {
-                                    build_object_heal_request(
-                                        bucket.clone(),
-                                        candidate.object.clone(),
-                                        version_id.clone(),
-                                        self.scan_mode,
-                                        HealChannelPriority::High,
-                                    )
-                                };
-                                (self.update_current_path)(&candidate.object).await;
-                                let admission = self.send_required_scanner_heal_request(
-                                    PendingScannerHealKind::Object,
-                                    bucket.clone(),
-                                    Some(candidate.object.clone()),
-                                    version_id.clone(),
-                                    request,
-                                )
-                                .await?;
-                                if admission.is_admitted() {
-                                    counter!(METRIC_SCANNER_HEAL_DISCOVERY_QUEUED_TOTAL).increment(1);
-                                } else if sub_quorum_candidate {
-                                    self.mark_pending_scanner_heal_reason(
-                                        PendingScannerHealKind::Object,
-                                        &bucket,
-                                        Some(&candidate.object),
-                                        version_id.as_deref(),
-                                        "sub_quorum_metadata",
+                            let fivs = match entry.file_info_versions(&bucket) {
+                                Ok(fivs) => fivs,
+                                Err(e) => {
+                                    error!(
+                                        target: "rustfs::scanner::folder",
+                                        event = EVENT_SCANNER_FOLDER_STATE,
+                                        component = LOG_COMPONENT_SCANNER,
+                                        subsystem = LOG_SUBSYSTEM_FOLDER,
+                                        bucket = %bucket,
+                                        entry = %entry.name,
+                                        state = "file_info_versions_failed",
+                                        error = %e,
+                                        "Scanner list_path_raw failed to resolve file versions"
                                     );
-                                }
-                                found_objects = true;
-                            }
-
-                            // Candidates beyond the main cap remain exact
-                            // version requests; never downgrade them to a
-                            // latest-version (version_id=None) heal.
-                            for candidate in discovery.truncated_candidates {
-                                let version_id = candidate.validated_version().map(|id| id.to_string());
-                                let identity = (candidate.object.clone(), version_id.clone(), candidate.kind.clone());
-                                if seen_truncated_objects.len() >= MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS
-                                    && !seen_truncated_objects.contains(&candidate.object)
-                                {
-                                    continue;
-                                }
-                                seen_truncated_objects.insert(candidate.object.clone());
-                                if !seen_heal_candidates.insert(identity) {
-                                    continue;
-                                }
-                                let request = build_object_heal_request(
-                                    bucket.clone(),
-                                    candidate.object.clone(),
-                                    version_id.clone(),
-                                    self.scan_mode,
-                                    HealChannelPriority::High,
-                                );
-                                (self.update_current_path)(&candidate.object).await;
-                                let admission = self
-                                    .send_required_scanner_heal_request(
+                                    self.send_required_scanner_heal_request(
                                         PendingScannerHealKind::Object,
                                         bucket.clone(),
-                                        Some(candidate.object.clone()),
-                                        version_id,
-                                        request,
+                                        Some(entry.name.clone()),
+                                        None,
+                                        build_object_heal_request(
+                                            bucket.clone(),
+                                            entry.name.clone(),
+                                            None,
+                                            self.scan_mode,
+                                            HealChannelPriority::High,
+                                        ),
                                     )
                                     .await?;
-                                if admission.is_admitted() {
-                                    counter!(METRIC_SCANNER_HEAL_DISCOVERY_QUEUED_TOTAL).increment(1);
+                                    found_objects = true;
+                                    continue;
                                 }
+                            };
+
+                            for fiv in fivs.versions {
+                                let version_id = fiv.version_id.and_then(|v| if v.is_nil() { None } else { Some(v.to_string()) });
+                                self.send_required_scanner_heal_request(
+                                    PendingScannerHealKind::Object,
+                                    bucket.clone(),
+                                    Some(entry.name.clone()),
+                                    version_id.clone(),
+                                    build_object_heal_request(
+                                        bucket.clone(),
+                                        entry.name.clone(),
+                                        version_id,
+                                        self.scan_mode,
+                                        HealChannelPriority::High,
+                                    ),
+                                )
+                                .await?;
                                 found_objects = true;
                             }
 

crates/scanner/src/scanner_folder/item_actions.rs

@@ -13,8 +13,6 @@
 // limitations under the License.
 /// Per-object scan actions: ScannerItem, the get-size failure policy, and the heal/ILM admission helpers.
 use super::*;
-#[cfg(test)]
-use rustfs_filemeta::MetadataResolutionParams;
 
 /// Cached folder information for scanning
 #[derive(Clone, Debug)]
@@ -90,22 +88,6 @@ pub(super) fn build_object_heal_request(
     }
 }
 
-/// Build the versionless inspection request used when discovery cannot prove
-/// a destructive version identity (for example an unversioned object or a
-/// bounded candidate overflow). The explicit flag is the fail-closed safety
-/// boundary; callers must not reconstruct it with the destructive default.
-pub(super) fn build_non_destructive_object_heal_request(
-    bucket: String,
-    object: String,
-    scan_mode: HealScanMode,
-    priority: HealChannelPriority,
-) -> HealChannelRequest {
-    let mut request = build_object_heal_request(bucket, object, None, scan_mode, priority);
-    request.remove_corrupted = Some(false);
-    request
-}
-
-#[cfg(test)]
 pub(super) fn resolve_object_heal_entry(
     entries: &MetaCacheEntries,
     resolver: MetadataResolutionParams,

crates/scanner/src/scanner_folder/ledger.rs

@@ -105,29 +105,6 @@ impl FolderScanner {
         }
     }
 
-    /// Preserve the discovery reason when a candidate could not be admitted
-    /// immediately. The existing string field is intentionally reused so the
-    /// scanner's map-encoded cache schema stays backward compatible.
-    pub(super) fn mark_pending_scanner_heal_reason(
-        &mut self,
-        kind: PendingScannerHealKind,
-        bucket: &str,
-        object: Option<&str>,
-        version_id: Option<&str>,
-        reason: &str,
-    ) {
-        if let Some(entry) = self
-            .new_cache
-            .info
-            .pending_heals
-            .iter_mut()
-            .find(|entry| pending_scanner_heal_matches(entry, kind, bucket, object, version_id))
-        {
-            entry.last_admission_reason = reason.to_string();
-            self.sync_pending_heals();
-        }
-    }
-
     pub(super) fn prune_pending_scanner_heals(&mut self) {
         let now = Self::now_secs();
         let before_expiry = self.new_cache.info.pending_heals.len();
@@ -328,22 +305,13 @@ pub(super) fn build_pending_scanner_heal_request(entry: &PendingScannerHeal) ->
     match entry.kind {
         PendingScannerHealKind::Bucket => Some(build_bucket_heal_request(entry.bucket.clone(), HealChannelPriority::High)),
         PendingScannerHealKind::Object => entry.object.as_ref().map(|object| {
-            if entry.version_id.is_none() {
-                build_non_destructive_object_heal_request(
-                    entry.bucket.clone(),
-                    object.clone(),
-                    entry.scan_mode,
-                    HealChannelPriority::High,
-                )
-            } else {
-                build_object_heal_request(
-                    entry.bucket.clone(),
-                    object.clone(),
-                    entry.version_id.clone(),
-                    entry.scan_mode,
-                    HealChannelPriority::High,
-                )
-            }
+            build_object_heal_request(
+                entry.bucket.clone(),
+                object.clone(),
+                entry.version_id.clone(),
+                entry.scan_mode,
+                HealChannelPriority::High,
+            )
         }),
     }
 }

crates/scanner/src/scanner_folder/tests.rs

@@ -17,7 +17,7 @@ use crate::SCANNER_SLEEPER;
 use super::*;
 use crate::storage_api::VersionPurgeStatusType;
 use crate::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, TierStats, new_disk, storageclass};
-use rustfs_filemeta::{FileInfo, FileMeta, MetadataResolutionParams};
+use rustfs_filemeta::{FileInfo, FileMeta};
 use std::io::Write;
 #[cfg(unix)]
 use std::os::unix::fs::{PermissionsExt, symlink};
@@ -982,21 +982,6 @@ fn test_build_object_heal_request_omits_nil_version_id() {
     assert_eq!(request.recreate_missing, Some(false));
 }
 
-#[test]
-fn test_build_non_destructive_object_heal_request_disables_removal() {
-    let request = build_non_destructive_object_heal_request(
-        "bucket".to_string(),
-        "path/to/object".to_string(),
-        HealScanMode::Deep,
-        HealChannelPriority::High,
-    );
-
-    assert_eq!(request.object_version_id, None);
-    assert_eq!(request.remove_corrupted, Some(false));
-    assert_eq!(request.recreate_missing, Some(false));
-    assert_eq!(request.source, HealRequestSource::Scanner);
-}
-
 #[test]
 fn test_build_bucket_heal_request_disables_recreate_for_scanner() {
     let request = build_bucket_heal_request("bucket".to_string(), HealChannelPriority::Low);
@@ -1136,42 +1121,6 @@ fn test_pending_heal_reconstructs_object_request_with_version() {
     assert_eq!(request.source, HealRequestSource::Scanner);
 }
 
-#[test]
-fn test_pending_heal_reconstructs_unversioned_request_without_removal() {
-    let pending = pending_heal(PendingScannerHealKind::Object, "bucket", Some("object"), None, 1, 1);
-
-    let request = build_pending_scanner_heal_request(&pending).expect("unversioned object request should rebuild");
-
-    assert!(request.object_version_id.is_none());
-    assert_eq!(request.remove_corrupted, Some(false));
-    assert_eq!(request.recreate_missing, Some(false));
-}
-
-#[tokio::test]
-async fn test_pending_heal_reason_preserves_sub_quorum_discovery() {
-    let (mut scanner, temp_dir) = build_test_scanner().await;
-    let _guard = TestGuard::new(u64::MAX, usize::MAX, &mut scanner, temp_dir);
-
-    scanner.update_pending_scanner_heal_after_admission(
-        PendingScannerHealKind::Object,
-        "bucket",
-        Some("object"),
-        Some("version-a"),
-        HealScanMode::Deep,
-        HealAdmissionResult::Full,
-    );
-    scanner.mark_pending_scanner_heal_reason(
-        PendingScannerHealKind::Object,
-        "bucket",
-        Some("object"),
-        Some("version-a"),
-        "sub_quorum_metadata",
-    );
-
-    assert_eq!(scanner.new_cache.info.pending_heals.len(), 1);
-    assert_eq!(scanner.new_cache.info.pending_heals[0].last_admission_reason, "sub_quorum_metadata");
-}
-
 #[test]
 fn test_pending_heal_retry_candidates_respect_cap_and_order() {
     let pending: Vec<PendingScannerHeal> = (0..(MAX_PENDING_SCANNER_HEAL_RETRIES_PER_BUCKET + 2))
@@ -1374,20 +1323,6 @@ fn metadata_for_object(bucket: &str, object: &str) -> Vec<u8> {
     meta.marshal_msg().expect("test metadata should marshal")
 }
 
-fn metadata_for_object_version(bucket: &str, object: &str, version_id: Option<Uuid>) -> Vec<u8> {
-    let mut file_info = FileInfo::new(object, 4, 2);
-    file_info.volume = bucket.to_string();
-    file_info.name = object.to_string();
-    file_info.version_id = version_id;
-    file_info.versioned = version_id.is_some();
-    file_info.mod_time = Some(OffsetDateTime::now_utc());
-    file_info.size = 1;
-
-    let mut meta = FileMeta::new();
-    meta.add_version(file_info).expect("test metadata version should be accepted");
-    meta.marshal_msg().expect("test metadata should marshal")
-}
-
 async fn write_test_object_metadata(root: &std::path::Path, bucket: &str, object: &str) {
     write_test_object_metadata_bytes(root, bucket, object, &metadata_for_object(bucket, object)).await;
 }
@@ -1791,21 +1726,12 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
     let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
     let heal_starts = Arc::new(AtomicUsize::new(0));
     let heal_starts_clone = heal_starts.clone();
-    let healed_versions = Arc::new(Mutex::new(Vec::<Option<String>>::new()));
-    let healed_versions_clone = healed_versions.clone();
     let mut heal_rx =
         rustfs_common::heal_channel::init_heal_channel().expect("heal channel should initialize once for scanner tests");
     let _heal_responder = tokio::spawn(async move {
         while let Some(command) = heal_rx.recv().await {
-            if let rustfs_common::heal_channel::HealChannelCommand::Start {
-                request, response_tx, ..
-            } = command
-            {
+            if let rustfs_common::heal_channel::HealChannelCommand::Start { response_tx, .. } = command {
                 heal_starts_clone.fetch_add(1, Ordering::Relaxed);
-                healed_versions_clone
-                    .lock()
-                    .expect("heal version capture lock should not be poisoned")
-                    .push(request.object_version_id);
                 let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
             }
         }
@@ -1813,18 +1739,13 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
 
     let bucket = "src-archive";
     let object = "snapshots/37b3f20d941e2f5e6d99114d9bb2f3e67a8a2e5c9c4c5a1b0d6e7f8091a2b3c4";
-    let orphan_version = Uuid::from_u128(0x1934);
-    let shared_version = Uuid::from_u128(0x1935);
-    let orphan_metadata = metadata_for_object_version(bucket, object, Some(orphan_version));
-    let shared_metadata = metadata_for_object_version(bucket, object, Some(shared_version));
-    write_test_object_metadata_bytes(&temp_dir, bucket, object, &orphan_metadata).await;
-    let mut expected_metadata = vec![(temp_dir.join(bucket).join(object).join("xl.meta"), orphan_metadata.clone())];
+    let metadata = metadata_for_object(bucket, object);
+    write_test_object_metadata_bytes(&temp_dir, bucket, object, &metadata).await;
 
     let mut disks = vec![scanner.local_disk.clone()];
     for disk_name in ["disk2", "disk3", "disk4"] {
         let disk_root = temp_dir.join(disk_name);
-        write_test_object_metadata_bytes(&disk_root, bucket, object, &shared_metadata).await;
-        expected_metadata.push((disk_root.join(bucket).join(object).join("xl.meta"), shared_metadata.clone()));
+        write_test_object_metadata_bytes(&disk_root, bucket, object, &metadata).await;
         let endpoint = Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("failed to create extra disk endpoint");
         let disk = new_disk(
             &endpoint,
@@ -1873,29 +1794,8 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
         .new_cache
         .checked_flatten(bucket)
         .expect("healed cache must contain canonical child links");
-    // The fixture intentionally exposes two divergent version histories, so
-    // the scanner keeps both logical versions visible while discovering heals.
-    assert_eq!(root.objects, 2);
+    assert_eq!(root.objects, 1);
     assert!(heal_starts.load(Ordering::Relaxed) > 0, "test must execute the heal child-link path");
-    let orphan_version_text = orphan_version.to_string();
-    assert!(
-        healed_versions
-            .lock()
-            .expect("heal version capture lock should not be poisoned")
-            .iter()
-            .any(|version| version.as_deref() == Some(orphan_version_text.as_str())),
-        "sub-quorum orphan version must be submitted as an exact heal candidate"
-    );
-    for (path, expected) in expected_metadata {
-        assert_eq!(
-            tokio::fs::read(&path)
-                .await
-                .expect("scanner discovery must not delete metadata"),
-            expected,
-            "scanner discovery must not modify candidate metadata: {}",
-            path.display()
-        );
-    }
 }
 
 #[tokio::test]

docs/testing/e2e-suite-inventory.md

@@ -52,7 +52,7 @@
 | fault_proxy | 7 |  |
 | get_codec_streaming_compat_test | 1 |  |
 | get_stream_failure_observability_test | 1 |  |
-| group_delete_test | 1 |  |
+| group_delete_test | 4 |  |
 | head_object_consistency_test | 1 | ✅ |
 | head_object_range_test | 1 | ✅ |
 | heal_erasure_disk_rebuild_test | 4 | 🌙 |
@@ -99,4 +99,4 @@
 | tls_hot_reload_test | 1 | ✅ |
 | version_id_regression_test | 10 | ✅ |
 
-**Total listed: 575 tests across 82 modules · PR smoke: 163 tests / 36 modules · merge/main full: 453 tests / 73 modules · nightly replication: 55 tests · nightly cluster faults: 28 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-23.
+**Total listed: 578 tests across 82 modules · PR smoke: 163 tests / 36 modules · merge/main full: 456 tests / 73 modules · nightly replication: 55 tests · nightly cluster faults: 28 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-23.

scripts/test/vault_ha_kms_live.sh

@@ -241,7 +241,7 @@ env \
   RUSTFS_TEST_VAULT_FAILOVER_MARKER="$MARKER" \
   RUSTFS_TEST_VAULT_OLD_LEADER="$OLD_LEADER" \
   cargo test -p rustfs-kms --test vault_ha_failover_live \
-    vault_raft_leader_failure_preserves_kv2_and_transit_decrypts -- \
+    vault_raft_leader_failure_recovers_kv2_and_transit_decrypts -- \
     --ignored --nocapture --test-threads=1 &
 TEST_PID=$!