Merge branch 'main' into overtrue/report-security-coverage-ratchet

* test(kms): bound Vault failover progress wait

* ci(nightly): honor manual dispatch ref

* test(kms): preserve Vault worker failures

* test(kms): validate Vault circuit recovery

.config/coverage-baselines.toml

@@ -0,0 +1,20 @@
+# Report-only calibration baseline from https://github.com/rustfs/rustfs/actions/runs/29394996173.
+# Update counts only with a linked coverage run and a reviewed explanation.
+phase = "report-only"
+allowed_drop_percentage_points = 1.0
+
+[crates."crates/iam"]
+covered = 5149
+count = 8131
+
+[crates."crates/kms"]
+covered = 2950
+count = 4200
+
+[crates."crates/policy"]
+covered = 4636
+count = 5464
+
+[crates."crates/crypto"]
+covered = 469
+count = 494

.config/make/tests.mak

@@ -36,6 +36,7 @@ script-tests: ## Run shell script tests
 	./scripts/test_manual_transition_runbooks.sh
 	./scripts/check_embedded_secrets.sh --self-test
 	python3 ./scripts/check_test_wiring.py --self-test
+	python3 ./scripts/check_security_coverage.py --self-test
 	python3 ./scripts/check_scheduled_validation_freshness.py --self-test
 	python3 ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh

.github/workflows/coverage.yml

@@ -12,14 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Weekly workspace line-coverage baseline (backlog#1153 infra-5).
+# Workspace line-coverage baseline and security-crate calibration
+# (backlog#1153 infra-5/infra-6).
 #
-# NON-BLOCKING by design: this workflow only runs on schedule and manual
-# dispatch, so it never attaches a status to a PR and must never be made a
-# required check. It exists to give coverage a visible baseline and trend
-# (per-crate table in the job summary, lcov artifact kept 90 days) — the
-# per-crate ratchet for the security-critical crates builds on it later
-# (backlog#1153 infra-6, report-only first per the ci-11 ladder).
+# NON-BLOCKING by design: the weekly job gives coverage a visible baseline and
+# trend, while relevant pull requests run a report-only security-crate
+# comparison. Neither job is a required check during calibration.
 #
 # Measurement scope matches the PR test gate (ci.yml "Run tests"):
 # `--workspace --exclude e2e_test` with the `ci` nextest profile. Doctests are
@@ -31,6 +29,17 @@
 name: coverage
 
 on:
+  pull_request:
+    branches: [main]
+    paths:
+      - "crates/iam/**"
+      - "crates/kms/**"
+      - "crates/policy/**"
+      - "crates/crypto/**"
+      - ".config/coverage-baselines.toml"
+      - "scripts/coverage_per_crate.py"
+      - "scripts/check_security_coverage.py"
+      - ".github/workflows/coverage.yml"
   workflow_dispatch:
   schedule:
     # 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00),
@@ -39,6 +48,10 @@ on:
     # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
     - cron: "43 7 * * 0"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'schedule' }}
+
 # Only alert-on-failure needs more than read access; it declares its own
 # job-level `issues: write`.
 permissions:
@@ -46,12 +59,13 @@ permissions:
 
 jobs:
   coverage:
-    name: Workspace coverage (weekly)
+    name: Workspace line coverage
     runs-on: sm-standard-4
     # The instrumented build cannot reuse the regular CI cache (different
-    # RUSTFLAGS), so a cold week rebuilds the workspace before running the
-    # full suite; give it double the test job's 60-minute budget.
-    timeout-minutes: 120
+    # RUSTFLAGS), so a cold run rebuilds the workspace before running the
+    # full suite. Exact-head run 32573798257 needed 119m42s including reports
+    # and artifact upload, so keep a bounded 30-minute publication margin.
+    timeout-minutes: 150
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       # Match the PR gate's nextest semantics (ci.yml runs `--profile ci`):
@@ -91,7 +105,9 @@ jobs:
           cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
 
       - name: Write per-crate summary
-        run: python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
+        run: |
+          python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
+          python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
 
       - name: Upload coverage artifact
         if: always()

.github/workflows/nightly-gnu.yml

@@ -39,11 +39,10 @@ jobs:
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -89,11 +88,10 @@ jobs:
       # either casing.
       NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -178,11 +176,10 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout main branch
+      - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup

crates/e2e_test/src/cluster_concurrency_test.rs

@@ -15,14 +15,12 @@
 use crate::common::RustFSTestClusterEnvironment;
 use aws_sdk_s3::Client;
 use aws_sdk_s3::error::SdkError;
-use aws_sdk_s3::types::{CorsConfiguration, CorsRule};
 use bytes::Bytes;
 use std::sync::Arc;
 use tokio::sync::Barrier;
 use tracing::{info, warn};
 
 const BUCKET: &str = "conditional-put-race-bucket";
-const BUCKET_METADATA_RELOAD_BUCKET: &str = "bucket-metadata-reload-barrier";
 
 async fn cleanup_object(client: &Client, key: &str) {
     if let Err(e) = client.delete_object().bucket(BUCKET).key(key).send().await {
@@ -30,16 +28,6 @@ async fn cleanup_object(client: &Client, key: &str) {
     }
 }
 
-async fn assert_bucket_cors_missing(client: &Client) {
-    let result = client.get_bucket_cors().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await;
-    match result {
-        Err(SdkError::ServiceError(error)) => {
-            assert_eq!(error.err().meta().code(), Some("NoSuchCORSConfiguration"));
-        }
-        result => panic!("expected the peer to report a missing CORS configuration: {result:?}"),
-    }
-}
-
 async fn conditional_put(
     client: &Client,
     key: &str,
@@ -248,48 +236,3 @@ async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::
     cleanup_object(&client, test_key).await;
     Ok(())
 }
-
-#[tokio::test]
-async fn test_bucket_cors_write_is_visible_on_peer_before_response() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    crate::common::init_logging();
-
-    let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
-    cluster.start().await?;
-    cluster.create_test_bucket(BUCKET_METADATA_RELOAD_BUCKET).await?;
-
-    let writer = cluster.create_s3_client(0)?;
-    let reader = cluster.create_s3_client(1)?;
-    assert_bucket_cors_missing(&reader).await;
-
-    let rule = CorsRule::builder()
-        .allowed_methods("GET")
-        .allowed_origins("https://example.com")
-        .build()?;
-    let configuration = CorsConfiguration::builder().cors_rules(rule).build()?;
-
-    writer
-        .put_bucket_cors()
-        .bucket(BUCKET_METADATA_RELOAD_BUCKET)
-        .cors_configuration(configuration)
-        .send()
-        .await?;
-
-    let response = reader.get_bucket_cors().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await?;
-    let rules = response.cors_rules();
-    assert_eq!(
-        rules.len(),
-        1,
-        "peer should observe the committed CORS rule before the write response returns"
-    );
-    assert_eq!(rules[0].allowed_methods(), ["GET"]);
-    assert_eq!(rules[0].allowed_origins(), ["https://example.com"]);
-
-    writer
-        .delete_bucket_cors()
-        .bucket(BUCKET_METADATA_RELOAD_BUCKET)
-        .send()
-        .await?;
-    assert_bucket_cors_missing(&reader).await;
-    writer.delete_bucket().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await?;
-    Ok(())
-}

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -85,7 +85,6 @@ const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
 const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
 const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
 const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024;
-const BUCKET_METADATA_RELOAD_TIMEOUT: Duration = Duration::from_secs(5);
 
 /// Error for a peer that reported `success = false` without an `error_info` payload.
 ///
@@ -1329,38 +1328,27 @@ impl PeerRestClient {
     }
 
     pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
-        let result = tokio::time::timeout(BUCKET_METADATA_RELOAD_TIMEOUT, async {
-            let result = self.load_bucket_metadata_once(bucket, scanner_maintenance_change).await;
-            if let Err(err) = &result
-                && Self::is_network_like_error(err)
-            {
-                self.prepare_retry().await;
-                return self.load_bucket_metadata_once(bucket, scanner_maintenance_change).await;
+        self.finalize_result(
+            async {
+                let mut client = self.get_client().await?;
+                let mut request = Request::new(LoadBucketMetadataRequest {
+                    bucket: bucket.to_string(),
+                    scanner_maintenance_change,
+                });
+                set_tonic_mutation_body_digest(&mut request)?;
+
+                let response = client.load_bucket_metadata(request).await?.into_inner();
+                if !response.success {
+                    if let Some(msg) = response.error_info {
+                        return Err(Error::other(msg));
+                    }
+                    return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket)));
+                }
+                Ok(())
             }
-            result
-        })
+            .await,
+        )
         .await
-        .unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
-        self.finalize_result(result).await
-    }
-
-    async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
-        let mut client = self.get_client().await?;
-        let mut request = Request::new(LoadBucketMetadataRequest {
-            bucket: bucket.to_string(),
-            scanner_maintenance_change,
-        });
-        set_tonic_mutation_body_digest(&mut request)?;
-        request.set_timeout(BUCKET_METADATA_RELOAD_TIMEOUT);
-
-        let response = client.load_bucket_metadata(request).await?.into_inner();
-        if !response.success {
-            if let Some(msg) = response.error_info {
-                return Err(Error::other(msg));
-            }
-            return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket)));
-        }
-        Ok(())
     }
 
     pub async fn delete_bucket_metadata(&self, bucket: &str) -> Result<()> {

crates/kms/tests/vault_ha_failover_live.rs

@@ -16,14 +16,14 @@
 //!
 //! `scripts/test/vault_ha_kms_live.sh` owns the official Vault containers and
 //! kills the active node while this test continuously decrypts through a
-//! surviving standby. KV2 and Transit requests must remain successful, use a
-//! bounded number of attempts, and leave the circuit and in-flight gauges at
-//! zero after a new leader is elected.
+//! surviving standby. KV2 and Transit must recover after the bounded circuit
+//! interval, use a bounded number of attempts, and leave the circuit and
+//! in-flight gauges at zero after a new leader is elected.
 
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
-use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
 use metrics_util::MetricKind;
@@ -43,6 +43,11 @@ const OPERATION_ATTEMPTS: &str = "rustfs_kms_backend_operation_attempts";
 const IN_FLIGHT: &str = "rustfs_kms_backend_in_flight";
 const CIRCUIT_OPEN: &str = "rustfs_kms_backend_circuit_open";
 const MAX_ATTEMPTS: u32 = 10;
+const ATTEMPT_TIMEOUT: Duration = Duration::from_secs(2);
+const HEALTHY_PROGRESS_TIMEOUT: Duration = Duration::from_secs(20);
+// The circuit remains open for 30s after five failed attempts.
+const POST_FAILOVER_PROGRESS_TIMEOUT: Duration = Duration::from_secs(35);
+const FAILOVER_ERROR_POLL_INTERVAL: Duration = Duration::from_millis(100);
 
 type MetricEntry = (
     metrics_util::CompositeKey,
@@ -64,7 +69,7 @@ fn config(backend: KmsBackend, backend_config: BackendConfig) -> KmsConfig {
         backend,
         backend_config,
         allow_insecure_dev_defaults: true,
-        timeout: Duration::from_secs(2),
+        timeout: ATTEMPT_TIMEOUT,
         retry_attempts: MAX_ATTEMPTS,
         enable_cache: false,
         ..KmsConfig::default()
@@ -164,14 +169,31 @@ fn retryable_failures(snapshot: &[MetricEntry], operation: &str) -> u64 {
         .sum()
 }
 
-async fn wait_for_count(counter: &AtomicU64, minimum: u64, description: &str) {
-    tokio::time::timeout(Duration::from_secs(20), async {
+async fn wait_for_count(
+    counter: &AtomicU64,
+    failure: &Mutex<Option<String>>,
+    minimum: u64,
+    description: &str,
+    timeout: Duration,
+) {
+    tokio::time::timeout(timeout, async {
         while counter.load(Ordering::SeqCst) < minimum {
+            if let Some(error) = failure.lock().expect("decrypt failure lock poisoned").as_ref() {
+                panic!(
+                    "{description} worker failed after {} successful decrypts: {error}",
+                    counter.load(Ordering::SeqCst)
+                );
+            }
             tokio::time::sleep(Duration::from_millis(25)).await;
         }
     })
     .await
-    .unwrap_or_else(|_| panic!("timed out waiting for {description}"));
+    .unwrap_or_else(|_| {
+        panic!(
+            "timed out after {timeout:?} waiting for {description}: completed {}, expected {minimum}",
+            counter.load(Ordering::SeqCst)
+        )
+    });
 }
 
 async fn wait_for_file(path: &Path, description: &str) {
@@ -189,7 +211,8 @@ async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
     request: DecryptRequest,
     expected: Vec<u8>,
     completed: Arc<AtomicU64>,
-    failed: Arc<AtomicBool>,
+    allow_failover_errors: Arc<AtomicBool>,
+    failure: Arc<Mutex<Option<String>>>,
     stop: CancellationToken,
 ) {
     while !stop.is_cancelled() {
@@ -197,8 +220,18 @@ async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
             Ok(response) if response.plaintext == expected => {
                 completed.fetch_add(1, Ordering::SeqCst);
             }
-            Ok(_) | Err(_) => {
-                failed.store(true, Ordering::SeqCst);
+            Ok(_) => {
+                *failure.lock().expect("decrypt failure lock poisoned") =
+                    Some("decrypt returned unexpected plaintext".to_string());
+                return;
+            }
+            Err(rustfs_kms::KmsError::BackendError { .. } | rustfs_kms::KmsError::OperationTimedOut { .. })
+                if allow_failover_errors.load(Ordering::SeqCst) =>
+            {
+                tokio::time::sleep(FAILOVER_ERROR_POLL_INTERVAL).await;
+            }
+            Err(error) => {
+                *failure.lock().expect("decrypt failure lock poisoned") = Some(error.to_string());
                 return;
             }
         }
@@ -296,7 +329,9 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
     );
 
     let stop = CancellationToken::new();
-    let failed = Arc::new(AtomicBool::new(false));
+    let allow_failover_errors = Arc::new(AtomicBool::new(false));
+    let kv2_failure = Arc::new(Mutex::new(None));
+    let transit_failure = Arc::new(Mutex::new(None));
     let kv2_completed = Arc::new(AtomicU64::new(0));
     let transit_completed = Arc::new(AtomicU64::new(0));
     let kv2_worker = tokio::spawn(decrypt_loop(
@@ -304,7 +339,8 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
         kv2_request,
         kv2_data_key.plaintext_key,
         Arc::clone(&kv2_completed),
-        Arc::clone(&failed),
+        Arc::clone(&allow_failover_errors),
+        Arc::clone(&kv2_failure),
         stop.clone(),
     ));
     let transit_worker = tokio::spawn(decrypt_loop(
@@ -312,12 +348,21 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
         transit_request,
         transit_data_key.plaintext_key,
         Arc::clone(&transit_completed),
-        Arc::clone(&failed),
+        Arc::clone(&allow_failover_errors),
+        Arc::clone(&transit_failure),
         stop.clone(),
     ));
 
-    wait_for_count(&kv2_completed, 2, "two healthy KV2 decrypts").await;
-    wait_for_count(&transit_completed, 2, "two healthy Transit decrypts").await;
+    wait_for_count(&kv2_completed, &kv2_failure, 2, "two healthy KV2 decrypts", HEALTHY_PROGRESS_TIMEOUT).await;
+    wait_for_count(
+        &transit_completed,
+        &transit_failure,
+        2,
+        "two healthy Transit decrypts",
+        HEALTHY_PROGRESS_TIMEOUT,
+    )
+    .await;
+    allow_failover_errors.store(true, Ordering::SeqCst);
     std::fs::write(&marker, b"ready").expect("publish failover readiness marker");
 
     wait_for_file(&elected, "the replacement Vault leader").await;
@@ -326,18 +371,39 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
 
     let kv2_after_election = kv2_completed.load(Ordering::SeqCst) + 2;
     let transit_after_election = transit_completed.load(Ordering::SeqCst) + 2;
-    wait_for_count(&kv2_completed, kv2_after_election, "post-failover KV2 decrypts").await;
-    wait_for_count(&transit_completed, transit_after_election, "post-failover Transit decrypts").await;
+    wait_for_count(
+        &kv2_completed,
+        &kv2_failure,
+        kv2_after_election,
+        "post-failover KV2 decrypts",
+        POST_FAILOVER_PROGRESS_TIMEOUT,
+    )
+    .await;
+    wait_for_count(
+        &transit_completed,
+        &transit_failure,
+        transit_after_election,
+        "post-failover Transit decrypts",
+        POST_FAILOVER_PROGRESS_TIMEOUT,
+    )
+    .await;
 
     stop.cancel();
     kv2_worker.await.expect("KV2 decrypt worker must join");
     transit_worker.await.expect("Transit decrypt worker must join");
-    assert!(!failed.load(Ordering::SeqCst), "no decrypt may fail or return different plaintext");
+    assert!(
+        kv2_failure.lock().expect("KV2 failure lock poisoned").is_none(),
+        "no KV2 decrypt may fail or return different plaintext"
+    );
+    assert!(
+        transit_failure.lock().expect("Transit failure lock poisoned").is_none(),
+        "no Transit decrypt may fail or return different plaintext"
+    );
 }
 
 #[test]
 #[ignore = "requires a real three-node Vault Raft cluster; run scripts/test/vault_ha_kms_live.sh"]
-fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() {
+fn vault_raft_leader_failure_recovers_kv2_and_transit_decrypts() {
     let recorder = DebuggingRecorder::new();
     let snapshotter = recorder.snapshotter();
     metrics::with_local_recorder(&recorder, || {
@@ -349,11 +415,6 @@ fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() {
     });
     let snapshot = snapshotter.snapshot().into_vec();
 
-    assert_eq!(
-        counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "circuit_open")]),
-        0,
-        "a bounded leader election must not open the circuit"
-    );
     assert_eq!(
         counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]),
         0,

docs/testing/README.md

@@ -158,10 +158,11 @@ added by backlog#1153 infra-4.
 
 ## Coverage
 
-Line coverage is measured **weekly, not per-PR**, and is non-blocking: it
-exists for visibility and trend, never as a required check. Per-crate ratchets
-for the security-critical crates (iam / kms / policy / crypto) build on this
-baseline later (backlog#1153 infra-6, report-only first).
+Workspace line coverage is measured weekly. Pull requests that touch iam, kms,
+policy, or crypto also run a non-required, report-only comparison against
+`.config/coverage-baselines.toml`. During calibration, a regression is recorded
+in the job summary without failing the job; missing or malformed coverage
+evidence still fails closed (backlog#1153 infra-6).
 
 - **CI**: `.github/workflows/coverage.yml` runs every Sunday and on manual
   dispatch: `cargo llvm-cov nextest --workspace --exclude e2e_test` under the
@@ -174,6 +175,12 @@ baseline later (backlog#1153 infra-6, report-only first).
   plus the full suite). It prints the same per-crate table via
   `scripts/coverage_per_crate.py` and writes `target/llvm-cov/lcov.info` and
   `coverage.json`.
+- **Security-critical ratchet**: relevant pull requests compare iam / kms /
+  policy / crypto line coverage with the versioned baseline. Drops greater than
+  the configured one-percentage-point calibration threshold are marked
+  `REGRESSION (report-only)`. The weekly summary runs the same comparison so
+  calibration continues even when no relevant pull request is open. Baseline
+  changes require a linked coverage run and a reviewed explanation.
 - **Trend comparison**: each run's job summary is the weekly per-crate
   snapshot — open two runs from the Actions history (workflow "coverage") and
   compare their tables. For line-level diffs, download the two runs'

rustfs/src/app/bucket_usecase.rs

@@ -513,15 +513,13 @@ fn sr_bucket_meta_item(bucket: String, item_type: &str) -> SRBucketMeta {
     }
 }
 
-async fn notify_bucket_metadata_reload(
+fn notify_bucket_metadata_reload(
     bucket: String,
     operation: &'static str,
     request_context: Option<request_context::RequestContext>,
     scanner_maintenance_change: bool,
 ) {
     record_local_scanner_maintenance_reload(&bucket, scanner_maintenance_change);
-    // Keep reload detached across request cancellation, but wait before a healthy peer can serve the previous config.
-    let (completed_tx, completed_rx) = tokio::sync::oneshot::channel();
     spawn_background_with_context(request_context, async move {
         if let Some(notification_sys) = current_notification_system() {
             let result = if scanner_maintenance_change {
@@ -533,9 +531,7 @@ async fn notify_bucket_metadata_reload(
                 warn!(bucket = %bucket, error = %err, "failed to notify peers after {operation}");
             }
         }
-        let _ = completed_tx.send(());
     });
-    let _ = completed_rx.await;
 }
 
 fn record_local_scanner_maintenance_reload(bucket: &str, scanner_maintenance_change: bool) {
@@ -1480,7 +1476,7 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
-        notify_bucket_metadata_reload(bucket.clone(), "delete bucket encryption", request_context, false).await;
+        notify_bucket_metadata_reload(bucket.clone(), "delete bucket encryption", request_context, false);
 
         let item = sr_bucket_meta_item(bucket.clone(), "sse-config");
         if let Err(err) = site_replication_bucket_meta_hook(item).await {
@@ -1512,7 +1508,7 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
-        notify_bucket_metadata_reload(bucket.clone(), "delete bucket cors", request_context, false).await;
+        notify_bucket_metadata_reload(bucket.clone(), "delete bucket cors", request_context, false);
 
         let item = sr_bucket_meta_item(bucket.clone(), "cors-config");
         if let Err(err) = site_replication_bucket_meta_hook(item).await {
@@ -1544,7 +1540,7 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
-        notify_bucket_metadata_reload(bucket.clone(), "delete bucket lifecycle", request_context, true).await;
+        notify_bucket_metadata_reload(bucket.clone(), "delete bucket lifecycle", request_context, true);
 
         let item = sr_bucket_meta_item(bucket.clone(), "lc-config");
         if let Err(err) = site_replication_bucket_meta_hook(item).await {
@@ -1576,7 +1572,7 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
-        notify_bucket_metadata_reload(bucket.clone(), "delete bucket policy", request_context, false).await;
+        notify_bucket_metadata_reload(bucket.clone(), "delete bucket policy", request_context, false);
 
         let item = sr_bucket_meta_item(bucket.clone(), "policy");
         if let Err(err) = site_replication_bucket_meta_hook(item).await {
@@ -1634,7 +1630,7 @@ impl DefaultBucketUsecase {
         }
         drop(targets_guard);
 
-        notify_bucket_metadata_reload(bucket.clone(), "delete bucket replication", request_context, true).await;
+        notify_bucket_metadata_reload(bucket.clone(), "delete bucket replication", request_context, true);
 
         let item = sr_bucket_meta_item(bucket.clone(), "replication-config");
         if let Err(err) = site_replication_bucket_meta_hook(item).await {
@@ -1659,7 +1655,7 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
-        notify_bucket_metadata_reload(bucket.clone(), "delete bucket tagging", request_context, false).await;
+        notify_bucket_metadata_reload(bucket.clone(), "delete bucket tagging", request_context, false);
 
         let item = sr_bucket_meta_item(bucket.clone(), "tags");
         if let Err(err) = site_replication_bucket_meta_hook(item).await {
@@ -1692,7 +1688,7 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
-        notify_bucket_metadata_reload(bucket.clone(), "delete public access block", request_context, false).await;
+        notify_bucket_metadata_reload(bucket.clone(), "delete public access block", request_context, false);
 
         Ok(S3Response::with_status(DeletePublicAccessBlockOutput::default(), StatusCode::NO_CONTENT))
     }
@@ -2147,7 +2143,7 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
-        notify_bucket_metadata_reload(bucket.clone(), "put bucket encryption", request_context, false).await;
+        notify_bucket_metadata_reload(bucket.clone(), "put bucket encryption", request_context, false);
 
         let mut item = sr_bucket_meta_item(bucket.clone(), "sse-config");
         item.sse_config = Some(
@@ -2226,7 +2222,7 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
-        notify_bucket_metadata_reload(bucket.clone(), "put bucket lifecycle", request_context, true).await;
+        notify_bucket_metadata_reload(bucket.clone(), "put bucket lifecycle", request_context, true);
 
         let mut item = sr_bucket_meta_item(bucket.clone(), "lc-config");
         item.expiry_lc_config =
@@ -2311,7 +2307,7 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
-        notify_bucket_metadata_reload(bucket.clone(), "put bucket notification", request_context, false).await;
+        notify_bucket_metadata_reload(bucket.clone(), "put bucket notification", request_context, false);
 
         let region = resolve_notification_region(self.global_region(), request_region);
         let notify = current_notify_interface_for_context(self.context.as_deref());
@@ -2416,7 +2412,7 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
-        notify_bucket_metadata_reload(bucket.clone(), "put bucket policy", request_context, false).await;
+        notify_bucket_metadata_reload(bucket.clone(), "put bucket policy", request_context, false);
 
         let mut item = sr_bucket_meta_item(bucket.clone(), "policy");
         item.policy = Some(serde_json::from_str(&policy).map_err(|e| s3_error!(InvalidArgument, "parse policy failed {:?}", e))?);
@@ -2451,7 +2447,7 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
-        notify_bucket_metadata_reload(bucket.clone(), "put bucket cors", request_context, false).await;
+        notify_bucket_metadata_reload(bucket.clone(), "put bucket cors", request_context, false);
 
         let mut item = sr_bucket_meta_item(bucket.clone(), "cors-config");
         item.cors =
@@ -2495,7 +2491,7 @@ impl DefaultBucketUsecase {
             .map_err(ApiError::from)?;
         drop(targets_guard);
 
-        notify_bucket_metadata_reload(bucket.clone(), "put bucket replication", request_context, true).await;
+        notify_bucket_metadata_reload(bucket.clone(), "put bucket replication", request_context, true);
 
         let mut item = sr_bucket_meta_item(bucket.clone(), "replication-config");
         item.replication_config = Some(
@@ -2535,7 +2531,7 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
-        notify_bucket_metadata_reload(bucket.clone(), "put public access block", request_context, false).await;
+        notify_bucket_metadata_reload(bucket.clone(), "put public access block", request_context, false);
 
         Ok(S3Response::new(PutPublicAccessBlockOutput::default()))
     }
@@ -2564,7 +2560,7 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
-        notify_bucket_metadata_reload(bucket.clone(), "put bucket tagging", request_context, false).await;
+        notify_bucket_metadata_reload(bucket.clone(), "put bucket tagging", request_context, false);
 
         let mut item = sr_bucket_meta_item(bucket.clone(), "tags");
         item.tags = Some(serialize_config(&tagging).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?);
@@ -2597,7 +2593,7 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
-        notify_bucket_metadata_reload(bucket.clone(), "put bucket versioning", request_context, false).await;
+        notify_bucket_metadata_reload(bucket.clone(), "put bucket versioning", request_context, false);
 
         let mut item = sr_bucket_meta_item(bucket.clone(), "version-config");
         item.versioning = Some(
@@ -3048,7 +3044,7 @@ mod tests {
                 "{method} should identify the bucket metadata operation in reload logs"
             );
             let expected_reload = format!(
-                "notify_bucket_metadata_reload(bucket.clone(), \"{operation}\", request_context, {scanner_maintenance_change}).await;"
+                "notify_bucket_metadata_reload(bucket.clone(), \"{operation}\", request_context, {scanner_maintenance_change});"
             );
             assert!(
                 body.contains(&expected_reload),

scripts/check_security_coverage.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Compare security-critical crate line coverage with the report-only baseline."""
+
+import argparse
+import json
+import math
+import os
+import sys
+import tempfile
+import tomllib
+from pathlib import Path
+
+from coverage_per_crate import fmt_pct, load_coverage
+
+
+SECURITY_CRATES = ("crates/iam", "crates/kms", "crates/policy", "crates/crypto")
+
+
+def load_baselines(path: str) -> tuple[float, dict[str, tuple[int, int]]]:
+    with open(path, "rb") as fh:
+        config = tomllib.load(fh)
+
+    if config.get("phase") != "report-only":
+        raise ValueError("coverage baseline phase must be report-only")
+
+    raw_allowed_drop = config["allowed_drop_percentage_points"]
+    if isinstance(raw_allowed_drop, bool) or not isinstance(raw_allowed_drop, (int, float)):
+        raise ValueError("allowed_drop_percentage_points must be a number")
+    allowed_drop = float(raw_allowed_drop)
+    if not math.isfinite(allowed_drop) or allowed_drop < 0:
+        raise ValueError("allowed_drop_percentage_points must be finite and non-negative")
+
+    baselines: dict[str, tuple[int, int]] = {}
+    for crate, values in config["crates"].items():
+        covered = values["covered"]
+        count = values["count"]
+        if type(covered) is not int or type(count) is not int:
+            raise ValueError(f"invalid baseline for {crate}: covered and count must be integers")
+        if covered < 0 or count <= 0 or covered > count:
+            raise ValueError(f"invalid baseline for {crate}: {covered}/{count}")
+        baselines[crate] = (covered, count)
+    missing = [crate for crate in SECURITY_CRATES if crate not in baselines]
+    unexpected = sorted(set(baselines).difference(SECURITY_CRATES))
+    if missing or unexpected:
+        raise ValueError(f"coverage baseline crate set mismatch: missing={missing}, unexpected={unexpected}")
+    return allowed_drop, baselines
+
+
+def compare(
+    current: dict[str, list[int]],
+    baselines: dict[str, tuple[int, int]],
+    allowed_drop: float,
+) -> list[tuple[str, int, int, int, int, float, bool]]:
+    rows = []
+    for crate, (baseline_covered, baseline_count) in baselines.items():
+        if crate not in current:
+            raise ValueError(f"coverage report is missing {crate}")
+        covered, count = current[crate]
+        if type(covered) is not int or type(count) is not int:
+            raise ValueError(f"invalid coverage for {crate}: covered and count must be integers")
+        if covered < 0 or count <= 0 or covered > count:
+            raise ValueError(f"invalid coverage for {crate}: {covered}/{count}")
+        current_pct = 100.0 * covered / count
+        baseline_pct = 100.0 * baseline_covered / baseline_count
+        delta = current_pct - baseline_pct
+        rows.append((crate, covered, count, baseline_covered, baseline_count, delta, delta < -allowed_drop))
+    return rows
+
+
+def print_report(rows: list[tuple[str, int, int, int, int, float, bool]], allowed_drop: float) -> None:
+    print("## Security-critical coverage ratchet (report-only)")
+    print()
+    print(f"Calibration threshold: a drop greater than {allowed_drop:.2f} percentage points is reported as a regression.")
+    print()
+    print("| Crate | Current | Baseline | Delta | Status |")
+    print("|---|---:|---:|---:|---|")
+    for crate, covered, count, baseline_covered, baseline_count, delta, regressed in rows:
+        status = "REGRESSION (report-only)" if regressed else "OK"
+        print(
+            f"| `{crate}` | {fmt_pct(covered, count)} ({covered}/{count}) "
+            f"| {fmt_pct(baseline_covered, baseline_count)} ({baseline_covered}/{baseline_count}) "
+            f"| {delta:+.2f} pp | {status} |"
+        )
+    print()
+    print("This calibration phase records regressions without failing the job; malformed or incomplete evidence still fails closed.")
+
+
+def self_test() -> None:
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        coverage = root / "coverage.json"
+        baseline = root / "baseline.toml"
+        coverage_data = {
+            "data": [
+                {
+                    "files": [
+                        {
+                            "filename": str(root / "crates/iam/src/lib.rs"),
+                            "summary": {"lines": {"covered": 80, "count": 100}},
+                        },
+                        {
+                            "filename": str(root / "crates/kms/src/lib.rs"),
+                            "summary": {"lines": {"covered": 90, "count": 100}},
+                        },
+                        {
+                            "filename": str(root / "crates/policy/src/lib.rs"),
+                            "summary": {"lines": {"covered": 90, "count": 100}},
+                        },
+                        {
+                            "filename": str(root / "crates/crypto/src/lib.rs"),
+                            "summary": {"lines": {"covered": 90, "count": 100}},
+                        },
+                    ],
+                    "totals": {"lines": {"covered": 350, "count": 400}},
+                }
+            ]
+        }
+        coverage.write_text(json.dumps(coverage_data), encoding="utf-8")
+        baseline_text = """phase = "report-only"
+allowed_drop_percentage_points = 1.0
+[crates."crates/iam"]
+covered = 90
+count = 100
+[crates."crates/kms"]
+covered = 85
+count = 100
+[crates."crates/policy"]
+covered = 90
+count = 100
+[crates."crates/crypto"]
+covered = 90
+count = 100
+"""
+        baseline.write_text(baseline_text, encoding="utf-8")
+        current, _ = load_coverage(str(coverage), str(root))
+        allowed_drop, baselines = load_baselines(str(baseline))
+        rows = compare(current, baselines, allowed_drop)
+        assert [row[-1] for row in rows] == [True, False, False, False]
+        try:
+            compare({"crates/iam": current["crates/iam"]}, baselines, allowed_drop)
+        except ValueError as error:
+            assert str(error) == "coverage report is missing crates/kms"
+        else:
+            raise AssertionError("missing crate must fail closed")
+        try:
+            compare({**current, "crates/iam": [101, 100]}, baselines, allowed_drop)
+        except ValueError as error:
+            assert str(error) == "invalid coverage for crates/iam: 101/100"
+        else:
+            raise AssertionError("invalid coverage must fail closed")
+        for invalid_threshold in ("true", '"1.0"', "nan", "inf", "-inf"):
+            baseline.write_text(
+                baseline_text.replace("allowed_drop_percentage_points = 1.0", f"allowed_drop_percentage_points = {invalid_threshold}"),
+                encoding="utf-8",
+            )
+            try:
+                load_baselines(str(baseline))
+            except ValueError:
+                pass
+            else:
+                raise AssertionError(f"non-finite threshold {invalid_threshold} must fail closed")
+        for field, invalid_values in (
+            ("covered", ("true", '"90"', "90.0", "90.5")),
+            ("count", ("true", '"100"', "100.0", "100.5")),
+        ):
+            for invalid_value in invalid_values:
+                baseline.write_text(
+                    baseline_text.replace(f"{field} = {90 if field == 'covered' else 100}", f"{field} = {invalid_value}", 1),
+                    encoding="utf-8",
+                )
+                try:
+                    load_baselines(str(baseline))
+                except ValueError:
+                    pass
+                else:
+                    raise AssertionError(f"non-integer baseline {field} {invalid_value} must fail closed")
+        for covered, count in (
+            (True, 100),
+            (80, True),
+            (80.0, 100),
+            (80, 100.0),
+            (float("nan"), 100),
+            (80, float("inf")),
+        ):
+            try:
+                compare({**current, "crates/iam": [covered, count]}, baselines, allowed_drop)
+            except ValueError:
+                pass
+            else:
+                raise AssertionError(f"invalid aggregate coverage {covered}/{count} must fail closed")
+        lines = coverage_data["data"][0]["files"][0]["summary"]["lines"]
+        for field, invalid_values in (
+            ("covered", (True, "80", 80.0, 80.5, float("nan"), float("inf"), float("-inf"))),
+            ("count", (True, "100", 100.0, 100.5, float("nan"), float("inf"), float("-inf"))),
+        ):
+            original = lines[field]
+            for invalid_value in invalid_values:
+                lines[field] = invalid_value
+                coverage.write_text(json.dumps(coverage_data), encoding="utf-8")
+                try:
+                    load_coverage(str(coverage), str(root))
+                except ValueError:
+                    pass
+                else:
+                    raise AssertionError(f"invalid raw coverage {field} {invalid_value} must fail closed")
+            lines[field] = original
+        baseline.write_text(
+            baseline_text.replace(
+                '[crates."crates/crypto"]\ncovered = 90\ncount = 100\n',
+                "",
+            ),
+            encoding="utf-8",
+        )
+        try:
+            load_baselines(str(baseline))
+        except ValueError:
+            pass
+        else:
+            raise AssertionError("missing security-crate baseline must fail closed")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("coverage_json", nargs="?")
+    parser.add_argument("--baseline", default=".config/coverage-baselines.toml")
+    parser.add_argument("--repo-root", default=os.getcwd())
+    parser.add_argument("--self-test", action="store_true")
+    args = parser.parse_args()
+
+    if args.self_test:
+        self_test()
+        print("security coverage self-test passed")
+        return 0
+    if not args.coverage_json:
+        parser.error("coverage_json is required unless --self-test is used")
+
+    try:
+        current, _ = load_coverage(args.coverage_json, os.path.abspath(args.repo_root))
+        allowed_drop, baselines = load_baselines(args.baseline)
+        rows = compare(current, baselines, allowed_drop)
+    except (OSError, ValueError, KeyError, IndexError, json.JSONDecodeError, tomllib.TOMLDecodeError) as error:
+        print(f"error: {error}", file=sys.stderr)
+        return 1
+
+    print_report(rows, allowed_drop)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/coverage_per_crate.py

@@ -47,6 +47,31 @@ def fmt_pct(covered: int, count: int) -> str:
     return f"{100.0 * covered / count:.2f}%" if count else "—"
 
 
+def _line_counts(lines: dict[str, int], source: str) -> tuple[int, int]:
+    covered = lines["covered"]
+    count = lines["count"]
+    if type(covered) is not int or type(count) is not int or covered < 0 or count < 0 or covered > count:
+        raise ValueError(f"invalid line coverage for {source}: {covered}/{count}")
+    return covered, count
+
+
+def load_coverage(path: str, root: str) -> tuple[dict[str, list[int]], dict[str, int]]:
+    with open(path, encoding="utf-8") as fh:
+        export = json.load(fh)
+
+    data = export["data"][0]
+    files = data["files"]
+    total_covered, total_count = _line_counts(data["totals"]["lines"], "totals")
+
+    crates: dict[str, list[int]] = {}
+    for f in files:
+        covered, count = _line_counts(f["summary"]["lines"], f["filename"])
+        acc = crates.setdefault(crate_label(f["filename"], root), [0, 0])
+        acc[0] += covered
+        acc[1] += count
+    return crates, {"covered": total_covered, "count": total_count}
+
+
 def main() -> int:
     if len(sys.argv) < 2 or len(sys.argv) > 3:
         print(__doc__.strip(), file=sys.stderr)
@@ -54,24 +79,12 @@ def main() -> int:
     path = sys.argv[1]
     root = os.path.abspath(sys.argv[2] if len(sys.argv) == 3 else os.getcwd())
 
-    with open(path, encoding="utf-8") as fh:
-        export = json.load(fh)
-
     try:
-        data = export["data"][0]
-        files = data["files"]
-        totals = data["totals"]["lines"]
-    except (KeyError, IndexError) as exc:
+        crates, totals = load_coverage(path, root)
+    except (KeyError, IndexError, ValueError) as exc:
         print(f"error: unexpected llvm-cov JSON shape ({exc})", file=sys.stderr)
         return 1
 
-    crates: dict[str, list[int]] = {}
-    for f in files:
-        lines = f["summary"]["lines"]
-        acc = crates.setdefault(crate_label(f["filename"], root), [0, 0])
-        acc[0] += lines["covered"]
-        acc[1] += lines["count"]
-
     rows = sorted(
         crates.items(),
         key=lambda kv: (100.0 * kv[1][0] / kv[1][1]) if kv[1][1] else 101.0,

scripts/test/vault_ha_kms_live.sh

@@ -241,7 +241,7 @@ env \
   RUSTFS_TEST_VAULT_FAILOVER_MARKER="$MARKER" \
   RUSTFS_TEST_VAULT_OLD_LEADER="$OLD_LEADER" \
   cargo test -p rustfs-kms --test vault_ha_failover_live \
-    vault_raft_leader_failure_preserves_kv2_and_transit_decrypts -- \
+    vault_raft_leader_failure_recovers_kv2_and_transit_decrypts -- \
     --ignored --nocapture --test-threads=1 &
 TEST_PID=$!