test(e2e): activate conditional write regressions

.config/coverage-baselines.toml

@@ -1,20 +0,0 @@
-# Report-only calibration baseline from https://github.com/rustfs/rustfs/actions/runs/29394996173.
-# Update counts only with a linked coverage run and a reviewed explanation.
-phase = "report-only"
-allowed_drop_percentage_points = 1.0
-
-[crates."crates/iam"]
-covered = 5149
-count = 8131
-
-[crates."crates/kms"]
-covered = 2950
-count = 4200
-
-[crates."crates/policy"]
-covered = 4636
-count = 5464
-
-[crates."crates/crypto"]
-covered = 469
-count = 494

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=9f767b37ed8b1c82da62ea441462d75487785c8086e56f08fb6f6cd89c6e2e52
+sha256-darwin=03bdfb6a9d6e25d744c385f1461e651f05e4da78e5b0ead2adb3e8b2463e3834
-sha256-linux=fbdaf42b220958d4b1e8880e0f8b5a7992d38e21051bb60596dd4538424757d6
+sha256-linux=78c46adad135231017fb8679fb91877b5ae3ec6ce1f75d3d045d2c1076c12a49

.config/make/tests.mak

@@ -36,7 +36,6 @@ script-tests: ## Run shell script tests
 	./scripts/test_manual_transition_runbooks.sh
 	./scripts/check_embedded_secrets.sh --self-test
 	python3 ./scripts/check_test_wiring.py --self-test
-	python3 ./scripts/check_security_coverage.py --self-test
 	python3 ./scripts/check_scheduled_validation_freshness.py --self-test
 	python3 ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh

.github/workflows/coverage.yml

@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Workspace line-coverage baseline and security-crate calibration
+# Weekly workspace line-coverage baseline (backlog#1153 infra-5).
-# (backlog#1153 infra-5/infra-6).
 #
-# NON-BLOCKING by design: the weekly job gives coverage a visible baseline and
+# NON-BLOCKING by design: this workflow only runs on schedule and manual
-# trend, while relevant pull requests run a report-only security-crate
+# dispatch, so it never attaches a status to a PR and must never be made a
-# comparison. Neither job is a required check during calibration.
+# required check. It exists to give coverage a visible baseline and trend
+# (per-crate table in the job summary, lcov artifact kept 90 days) — the
+# per-crate ratchet for the security-critical crates builds on it later
+# (backlog#1153 infra-6, report-only first per the ci-11 ladder).
 #
 # Measurement scope matches the PR test gate (ci.yml "Run tests"):
 # `--workspace --exclude e2e_test` with the `ci` nextest profile. Doctests are
@@ -29,17 +31,6 @@
 name: coverage
 
 on:
-  pull_request:
-    branches: [main]
-    paths:
-      - "crates/iam/**"
-      - "crates/kms/**"
-      - "crates/policy/**"
-      - "crates/crypto/**"
-      - ".config/coverage-baselines.toml"
-      - "scripts/coverage_per_crate.py"
-      - "scripts/check_security_coverage.py"
-      - ".github/workflows/coverage.yml"
   workflow_dispatch:
   schedule:
     # 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00),
@@ -48,10 +39,6 @@ on:
     # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
     - cron: "43 7 * * 0"
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.event_name != 'schedule' }}
-
 # Only alert-on-failure needs more than read access; it declares its own
 # job-level `issues: write`.
 permissions:
@@ -59,13 +46,12 @@ permissions:
 
 jobs:
   coverage:
-    name: Workspace line coverage
+    name: Workspace coverage (weekly)
     runs-on: sm-standard-4
     # The instrumented build cannot reuse the regular CI cache (different
-    # RUSTFLAGS), so a cold run rebuilds the workspace before running the
+    # RUSTFLAGS), so a cold week rebuilds the workspace before running the
-    # full suite. Exact-head run 32573798257 needed 119m42s including reports
+    # full suite; give it double the test job's 60-minute budget.
-    # and artifact upload, so keep a bounded 30-minute publication margin.
+    timeout-minutes: 120
-    timeout-minutes: 150
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       # Match the PR gate's nextest semantics (ci.yml runs `--profile ci`):
@@ -105,9 +91,7 @@ jobs:
           cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
 
       - name: Write per-crate summary
-        run: |
+        run: python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
-          python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
-          python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
 
       - name: Upload coverage artifact
         if: always()

.github/workflows/nightly-gnu.yml

@@ -39,10 +39,11 @@ jobs:
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
-      - name: Checkout repository
+      - name: Checkout main branch
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
+          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -88,10 +89,11 @@ jobs:
       # either casing.
       NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout repository
+      - name: Checkout main branch
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
+          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -176,10 +178,11 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout repository
+      - name: Checkout main branch
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
+          ref: main
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup

CLAUDE.md

@@ -30,8 +30,7 @@ make build-docker BUILD_OS=ubuntu22.04
 - Crate membership: `Cargo.toml` `[workspace].members`
 - Architecture, layering, crate map: [ARCHITECTURE.md](ARCHITECTURE.md)
 - Migration guardrails & readiness contracts: [docs/architecture/](docs/architecture/README.md)
-- CI workflow steps: `.github/workflows/`; event, timeout, and required-status
+- CI gates: `.github/workflows/ci.yml` (source of truth; never copy its steps into docs)
-  matrix: [docs/testing/ci-gates.md](docs/testing/ci-gates.md)
 - Test-layer taxonomy, per-layer entry commands, serial/nextest rules, flake
   policy: [docs/testing/README.md](docs/testing/README.md)
 - Tier/ILM transition debugging (xl.meta inspection, versionId tracing):

CONTRIBUTING.md

@@ -70,8 +70,6 @@ make pre-pr
 
 > For the full test-layer taxonomy (unit / ecstore black-box / e2e / s3s-e2e / S3 compatibility / chaos / fuzz / bench), each layer's entry command, the naming conventions the migration gate depends on, and the serial/nextest rules, see [docs/testing/README.md](docs/testing/README.md).
 
-> For the event, timeout, required-status, and local reproduction matrix, see [docs/testing/ci-gates.md](docs/testing/ci-gates.md).
-
 ### 🔒 Automated Pre-commit Hooks
 #### What `make pre-commit` and `make pre-pr` actually run
 

Cargo.lock

@@ -1858,9 +1858,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.4.4"
+version = "1.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273"
+checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -2522,6 +2522,12 @@ dependencies = [
  "subtle",
 ]
 
+[[package]]
+name = "cty"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"
+
 [[package]]
 name = "curve25519-dalek"
 version = "4.1.3"
@@ -5982,6 +5988,15 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
 
+[[package]]
+name = "libmimalloc-sys"
+version = "0.1.49"
+source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
+dependencies = [
+ "cc",
+ "cty",
+]
+
 [[package]]
 name = "libredox"
 version = "0.1.20"
@@ -6382,6 +6397,14 @@ dependencies = [
  "synstructure 0.13.2",
 ]
 
+[[package]]
+name = "mimalloc"
+version = "0.1.52"
+source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
+dependencies = [
+ "libmimalloc-sys",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -9139,11 +9162,13 @@ dependencies = [
  "insta",
  "jiff",
  "libc",
+ "libmimalloc-sys",
  "libsystemd",
  "matchit 0.9.2",
  "md-5 0.11.0",
  "metrics",
  "metrics-util",
+ "mimalloc",
  "mime_guess",
  "opentelemetry",
  "opentelemetry_sdk",
@@ -9179,8 +9204,6 @@ dependencies = [
  "rustfs-lock",
  "rustfs-log-analyzer",
  "rustfs-madmin",
- "rustfs-mimalloc",
- "rustfs-mimalloc-sys",
  "rustfs-notify",
  "rustfs-object-capacity",
  "rustfs-object-data-cache",
@@ -9852,24 +9875,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "rustfs-mimalloc"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a406f4aa07084301d485beec873af6dccc8e3f8762da244743df92038b1db1a6"
-dependencies = [
- "rustfs-mimalloc-sys",
-]
-
-[[package]]
-name = "rustfs-mimalloc-sys"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3051b819175f58445d4c369a72f0ab88149f3885ba8bea2aff3be01f53fe7cd"
-dependencies = [
- "cc",
-]
-
 [[package]]
 name = "rustfs-notify"
 version = "1.0.0-rc.3"

Cargo.toml

@@ -350,8 +350,8 @@ russh-sftp = "2.4.0"
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-rustfs-mimalloc = { version = "0.5.0" }
+mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
-rustfs-mimalloc-sys = { version = "0.5.0" }
+libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
 hotpath = { version = "0.23.3", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.48" }

crates/e2e_test/src/reliant/conditional_writes.rs

@@ -1,52 +1,26 @@
 #![cfg(test)]
 
-use aws_config::meta::region::RegionProviderChain;
+use crate::common::{RustFSTestEnvironment, TEST_BUCKET, init_logging};
 use aws_sdk_s3::Client;
-use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
-use aws_sdk_s3::error::SdkError;
 use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
 use bytes::Bytes;
 use std::error::Error;
+use std::fmt::Debug;
 
-const ENDPOINT: &str = "http://localhost:9000";
+type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
-const ACCESS_KEY: &str = "rustfsadmin";
-const SECRET_KEY: &str = "rustfsadmin";
-const BUCKET: &str = "api-test";
 
-async fn create_aws_s3_client() -> Result<Client, Box<dyn Error>> {
+fn assert_s3_error_code<T, E>(result: Result<T, SdkError<E>>, expected: &str)
-    let region_provider = RegionProviderChain::default_provider().or_else(Region::new("us-east-1"));
+where
-    let shared_config = aws_config::defaults(aws_config::BehaviorVersion::latest())
+    T: Debug,
-        .region(region_provider)
+    E: ProvideErrorMetadata + Debug,
-        .credentials_provider(Credentials::new(ACCESS_KEY, SECRET_KEY, None, None, "static"))
+{
-        .endpoint_url(ENDPOINT)
+    let error = result.expect_err("conditional request must fail");
-        .load()
+    assert_eq!(
-        .await;
+        error.as_service_error().and_then(ProvideErrorMetadata::code),
-
+        Some(expected),
-    let client = Client::from_conf(
+        "unexpected conditional request error: {error:?}"
-        aws_sdk_s3::Config::from(&shared_config)
-            .to_builder()
-            .force_path_style(true)
-            .build(),
     );
-    Ok(client)
-}
-
-/// Setup test bucket, creating it if it doesn't exist
-async fn setup_test_bucket(client: &Client) -> Result<(), Box<dyn Error>> {
-    match client.create_bucket().bucket(BUCKET).send().await {
-        Ok(_) => {}
-        Err(SdkError::ServiceError(e)) => {
-            let e = e.into_err();
-            let error_code = e.meta().code().unwrap_or("");
-            if !error_code.eq("BucketAlreadyExists") {
-                return Err(e.into());
-            }
-        }
-        Err(e) => {
-            return Err(e.into());
-        }
-    }
-    Ok(())
 }
 
 /// Generate test data of specified size
@@ -60,7 +34,12 @@ fn generate_test_data(size: usize) -> Vec<u8> {
 }
 
 /// Upload an object and return its ETag
-async fn upload_object_with_metadata(client: &Client, bucket: &str, key: &str, data: &[u8]) -> Result<String, Box<dyn Error>> {
+async fn upload_object_with_metadata(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    data: &[u8],
+) -> Result<String, Box<dyn Error + Send + Sync>> {
     let response = client
         .put_object()
         .bucket(bucket)
@@ -69,188 +48,164 @@ async fn upload_object_with_metadata(client: &Client, bucket: &str, key: &str, d
         .send()
         .await?;
 
-    let etag = response.e_tag().unwrap_or("").to_string();
+    response
-    Ok(etag)
+        .e_tag()
+        .map(str::to_owned)
+        .ok_or_else(|| std::io::Error::other("put object response did not include an ETag").into())
 }
 
-/// Cleanup test objects from bucket
+async fn object_body(client: &Client, key: &str) -> Result<Bytes, Box<dyn Error + Send + Sync>> {
-async fn cleanup_objects(client: &Client, bucket: &str, keys: &[&str]) {
+    let response = client.get_object().bucket(TEST_BUCKET).key(key).send().await?;
-    for key in keys {
+    Ok(response.body.collect().await?.into_bytes())
-        let _ = client.delete_object().bucket(bucket).key(*key).send().await;
-    }
-}
-
-/// Generate unique test object key
-fn generate_test_key(prefix: &str) -> String {
-    use std::time::{SystemTime, UNIX_EPOCH};
-    let timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos();
-    format!("{prefix}-{timestamp}")
 }
 
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_conditional_put_okay() -> TestResult {
-async fn test_conditional_put_okay() -> Result<(), Box<dyn std::error::Error>> {
+    init_logging();
-    let client = create_aws_s3_client().await?;
+    let mut env = RustFSTestEnvironment::new().await?;
-    setup_test_bucket(&client).await?;
+    env.start_rustfs_server(vec![]).await?;
+    env.create_test_bucket(TEST_BUCKET).await?;
+    let client = env.create_s3_client();
 
-    let test_key = generate_test_key("conditional-put-ok");
+    let test_key = "conditional-put-ok";
     let initial_data = generate_test_data(1024); // 1KB test data
-    let updated_data = generate_test_data(2048); // 2KB updated data
+    let matching_data = generate_test_data(2048); // 2KB updated data
+    let non_matching_data = generate_test_data(3072); // 3KB updated data
 
     // Upload initial object and get its ETag
-    let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &initial_data).await?;
+    let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &initial_data).await?;
 
     // Test 1: PUT with matching If-Match condition (should succeed)
-    let response1 = client
+    client
         .put_object()
-        .bucket(BUCKET)
+        .bucket(TEST_BUCKET)
-        .key(&test_key)
+        .key(test_key)
-        .body(Bytes::from(updated_data.clone()).into())
+        .body(Bytes::from(matching_data.clone()).into())
         .if_match(&initial_etag)
         .send()
-        .await;
+        .await?;
-    assert!(response1.is_ok(), "PUT with matching If-Match should succeed");
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), matching_data);
 
     // Test 2: PUT with non-matching If-None-Match condition (should succeed)
     let fake_etag = "\"fake-etag-12345\"";
-    let response2 = client
+    client
         .put_object()
-        .bucket(BUCKET)
+        .bucket(TEST_BUCKET)
-        .key(&test_key)
+        .key(test_key)
-        .body(Bytes::from(updated_data.clone()).into())
+        .body(Bytes::from(non_matching_data.clone()).into())
         .if_none_match(fake_etag)
         .send()
-        .await;
+        .await?;
-    assert!(response2.is_ok(), "PUT with non-matching If-None-Match should succeed");
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), non_matching_data);
-
-    // Cleanup
-    cleanup_objects(&client, BUCKET, &[&test_key]).await;
 
     Ok(())
 }
 
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_conditional_put_failed() -> TestResult {
-async fn test_conditional_put_failed() -> Result<(), Box<dyn std::error::Error>> {
+    init_logging();
-    let client = create_aws_s3_client().await?;
+    let mut env = RustFSTestEnvironment::new().await?;
-    setup_test_bucket(&client).await?;
+    env.start_rustfs_server(vec![]).await?;
+    env.create_test_bucket(TEST_BUCKET).await?;
+    let client = env.create_s3_client();
 
-    let test_key = generate_test_key("conditional-put-failed");
+    let test_key = "conditional-put-failed";
     let initial_data = generate_test_data(1024);
     let updated_data = generate_test_data(2048);
 
     // Upload initial object and get its ETag
-    let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &initial_data).await?;
+    let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &initial_data).await?;
 
     // Test 1: PUT with non-matching If-Match condition (should fail with 412)
     let fake_etag = "\"fake-etag-should-not-match\"";
     let response1 = client
         .put_object()
-        .bucket(BUCKET)
+        .bucket(TEST_BUCKET)
-        .key(&test_key)
+        .key(test_key)
         .body(Bytes::from(updated_data.clone()).into())
         .if_match(fake_etag)
         .send()
         .await;
 
-    assert!(response1.is_err(), "PUT with non-matching If-Match should fail");
+    assert_s3_error_code(response1, "PreconditionFailed");
-    if let Err(e) = response1 {
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), initial_data);
-        if let SdkError::ServiceError(e) = e {
-            let e = e.into_err();
-            let error_code = e.meta().code().unwrap_or("");
-            assert_eq!("PreconditionFailed", error_code);
-        } else {
-            panic!("Unexpected error: {e:?}");
-        }
-    }
 
     // Test 2: PUT with matching If-None-Match condition (should fail with 412)
     let response2 = client
         .put_object()
-        .bucket(BUCKET)
+        .bucket(TEST_BUCKET)
-        .key(&test_key)
+        .key(test_key)
         .body(Bytes::from(updated_data.clone()).into())
         .if_none_match(&initial_etag)
         .send()
         .await;
 
-    assert!(response2.is_err(), "PUT with matching If-None-Match should fail");
+    assert_s3_error_code(response2, "PreconditionFailed");
-    if let Err(e) = response2 {
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), initial_data);
-        if let SdkError::ServiceError(e) = e {
-            let e = e.into_err();
-            let error_code = e.meta().code().unwrap_or("");
-            assert_eq!("PreconditionFailed", error_code);
-        } else {
-            panic!("Unexpected error: {e:?}");
-        }
-    }
-
-    // Cleanup - only need to clean up the initial object since failed PUTs shouldn't create objects
-    cleanup_objects(&client, BUCKET, &[&test_key]).await;
 
     Ok(())
 }
 
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_conditional_put_when_object_does_not_exist() -> TestResult {
-async fn test_conditional_put_when_object_does_not_exist() -> Result<(), Box<dyn std::error::Error>> {
+    init_logging();
-    let client = create_aws_s3_client().await?;
+    let mut env = RustFSTestEnvironment::new().await?;
-    setup_test_bucket(&client).await?;
+    env.start_rustfs_server(vec![]).await?;
+    env.create_test_bucket(TEST_BUCKET).await?;
+    let client = env.create_s3_client();
 
-    let key = "some_key";
+    let key = "conditional-put-missing";
-    cleanup_objects(&client, BUCKET, &[key]).await;
 
     // When the object does not exist, the If-Match condition should always fail
     let response1 = client
         .put_object()
-        .bucket(BUCKET)
+        .bucket(TEST_BUCKET)
         .key(key)
         .body(Bytes::from(generate_test_data(1024)).into())
         .if_match("*")
         .send()
         .await;
-    assert!(response1.is_err());
+    assert_s3_error_code(response1, "NoSuchKey");
-    if let Err(e) = response1 {
-        if let SdkError::ServiceError(e) = e {
-            let e = e.into_err();
-            let error_code = e.meta().code().unwrap_or("");
-            assert_eq!("NoSuchKey", error_code);
-        } else {
-            panic!("Unexpected error: {e:?}");
-        }
-    }
 
     // When the object does not exist, the If-None-Match condition should be able to succeed
-    let response2 = client
+    let created_data = generate_test_data(1024);
+    client
         .put_object()
-        .bucket(BUCKET)
+        .bucket(TEST_BUCKET)
         .key(key)
-        .body(Bytes::from(generate_test_data(1024)).into())
+        .body(Bytes::from(created_data.clone()).into())
         .if_none_match("*")
         .send()
-        .await;
+        .await?;
-    assert!(response2.is_ok());
+    assert_eq!(object_body(&client, key).await?.as_ref(), created_data);
 
-    cleanup_objects(&client, BUCKET, &[key]).await;
     Ok(())
 }
 
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_conditional_multi_part_upload() -> TestResult {
-async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::Error>> {
+    init_logging();
-    let client = create_aws_s3_client().await?;
+    let mut env = RustFSTestEnvironment::new().await?;
-    setup_test_bucket(&client).await?;
+    env.start_rustfs_server(vec![]).await?;
+    env.create_test_bucket(TEST_BUCKET).await?;
+    let client = env.create_s3_client();
 
-    let test_key = generate_test_key("multipart-upload-ok");
+    let test_key = "conditional-multipart-upload";
     let test_data = generate_test_data(1024);
-    let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &test_data).await?;
+    let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &test_data).await?;
 
     let part_size = 5 * 1024 * 1024; // 5MB per part (minimum for multipart)
     let num_parts = 3;
     let mut parts = Vec::new();
+    let mut expected_data = Vec::with_capacity(part_size * usize::try_from(num_parts)?);
 
     // Initiate multipart upload
-    let initiate_response = client.create_multipart_upload().bucket(BUCKET).key(&test_key).send().await?;
+    let initiate_response = client
+        .create_multipart_upload()
+        .bucket(TEST_BUCKET)
+        .key(test_key)
+        .send()
+        .await?;
 
     let upload_id = initiate_response
         .upload_id()
@@ -258,12 +213,13 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
 
     // Upload parts
     for part_number in 1..=num_parts {
-        let part_data = generate_test_data(part_size);
+        let part_data = vec![u8::try_from(part_number)?; part_size];
+        expected_data.extend_from_slice(&part_data);
 
         let upload_part_response = client
             .upload_part()
-            .bucket(BUCKET)
+            .bucket(TEST_BUCKET)
-            .key(&test_key)
+            .key(test_key)
             .upload_id(upload_id)
             .part_number(part_number)
             .body(Bytes::from(part_data).into())
@@ -286,57 +242,62 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
     // Test 1: Multipart upload with wildcard If-None-Match, should fail
     let complete_response = client
         .complete_multipart_upload()
-        .bucket(BUCKET)
+        .bucket(TEST_BUCKET)
-        .key(&test_key)
+        .key(test_key)
         .upload_id(upload_id)
         .multipart_upload(completed_upload.clone())
         .if_none_match("*")
         .send()
         .await;
 
-    assert!(complete_response.is_err());
+    assert_s3_error_code(complete_response, "PreconditionFailed");
 
     // Test 2: Multipart upload with matching If-None-Match, should fail
     let complete_response = client
         .complete_multipart_upload()
-        .bucket(BUCKET)
+        .bucket(TEST_BUCKET)
-        .key(&test_key)
+        .key(test_key)
         .upload_id(upload_id)
         .multipart_upload(completed_upload.clone())
         .if_none_match(initial_etag.clone())
         .send()
         .await;
 
-    assert!(complete_response.is_err());
+    assert_s3_error_code(complete_response, "PreconditionFailed");
 
     // Test 3: Multipart upload with unmatching If-Match, should fail
     let complete_response = client
         .complete_multipart_upload()
-        .bucket(BUCKET)
+        .bucket(TEST_BUCKET)
-        .key(&test_key)
+        .key(test_key)
         .upload_id(upload_id)
         .multipart_upload(completed_upload.clone())
         .if_match("\"abcdef\"")
         .send()
         .await;
 
-    assert!(complete_response.is_err());
+    assert_s3_error_code(complete_response, "PreconditionFailed");
+
+    let staged_parts = client
+        .list_parts()
+        .bucket(TEST_BUCKET)
+        .key(test_key)
+        .upload_id(upload_id)
+        .send()
+        .await?;
+    assert_eq!(staged_parts.parts().len(), usize::try_from(num_parts)?);
 
     // Test 4: Multipart upload with matching If-Match, should succeed
-    let complete_response = client
+    client
         .complete_multipart_upload()
-        .bucket(BUCKET)
+        .bucket(TEST_BUCKET)
-        .key(&test_key)
+        .key(test_key)
         .upload_id(upload_id)
-        .multipart_upload(completed_upload.clone())
+        .multipart_upload(completed_upload)
         .if_match(initial_etag)
         .send()
-        .await;
+        .await?;
-
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), expected_data);
-    assert!(complete_response.is_ok());
-
-    // Cleanup
-    cleanup_objects(&client, BUCKET, &[&test_key]).await;
 
     Ok(())
 }

crates/ecstore/src/io_support/bitrot.rs

@@ -784,24 +784,6 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle(
 ///
 /// # Returns
 /// A Result containing the BitrotWriterWrapper or an error
-/// Size hint handed to `DiskAPI::create_file` for a bitrot-wrapped shard.
-///
-/// A known length is grown by one checksum per shard so the on-disk file size
-/// matches what the bitrot writer emits. A negative length is the
-/// unknown-size sentinel (`HashReader::SIZE_PRESERVE_LAYER`, used by SSE and
-/// compression) and must be preserved: `RemoteDisk::create_file` forwards it
-/// in the `put_file_stream` query, and the receiver only treats `size > 0` as
-/// a fixed body length when locating the authenticated trailer. Clamping it
-/// to `0` would claim an empty body and misframe the stream. `0` stays `0`
-/// because a genuinely empty object still means an empty body.
-fn bitrot_create_file_size(length: i64, shard_size: usize, checksum_algo: &HashAlgorithm) -> i64 {
-    if length <= 0 {
-        return length;
-    }
-    let length = length as usize;
-    (length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
-}
-
 pub async fn create_bitrot_writer(
     is_inline_buffer: bool,
     disk: Option<&DiskStore>,
@@ -814,7 +796,12 @@ pub async fn create_bitrot_writer(
     let writer = if is_inline_buffer {
         CustomWriter::new_inline_buffer()
     } else if let Some(disk) = disk {
-        let length = bitrot_create_file_size(length, shard_size, &checksum_algo);
+        let length = if length > 0 {
+            let length = length as usize;
+            (length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
+        } else {
+            0
+        };
 
         let file = disk.create_file("", volume, path, length).await?;
         #[cfg(feature = "hotpath")]
@@ -833,25 +820,6 @@ mod tests {
     use rustfs_rio::ChunkReader;
     use std::collections::VecDeque;
 
-    #[test]
-    fn bitrot_create_file_size_grows_known_length_by_checksums() {
-        // 10 bytes over 4-byte shards = 3 shards, each followed by a 32-byte hash.
-        assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::HighwayHash256), 10 + 3 * 32);
-        assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::None), 10);
-    }
-
-    #[test]
-    fn bitrot_create_file_size_keeps_empty_and_unknown_distinct() {
-        assert_eq!(bitrot_create_file_size(0, 4, &HashAlgorithm::HighwayHash256), 0);
-        // SSE/compression streams advertise SIZE_PRESERVE_LAYER (-1); the remote
-        // put_file_stream receiver relies on a non-positive size to parse the auth
-        // trailer from the stream tail, so the sentinel must survive untouched.
-        assert_eq!(
-            bitrot_create_file_size(rustfs_rio::HashReader::SIZE_PRESERVE_LAYER, 4, &HashAlgorithm::HighwayHash256),
-            rustfs_rio::HashReader::SIZE_PRESERVE_LAYER
-        );
-    }
-
     struct TestChunkReader {
         chunks: VecDeque<Bytes>,
     }

crates/ecstore/src/set_disk/ops/object.rs

@@ -2124,13 +2124,26 @@ impl SetDisks {
 
             let put_object_size = known_put_object_storage_size(data.size());
             let shard_file_size_raw = erasure.shard_file_size(put_object_size);
-            let is_inline_buffer = storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned);
+            let is_inline_buffer =
+                storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned);
 
             let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled();
             let shard_file_size = shard_file_size_raw;
             let shard_size = erasure.shard_size();
             let write_path = classify_put_write_path(is_inline_buffer, put_object_size, fi.erasure.block_size);
             let direct_inline_commit = matches!(write_path, SmallWritePath::Inline);
+            {
+                use std::io::Write;
+                let msg = format!(
+                    "INLINE_DEBUG: bucket={} obj={} size={} shard_fs={} ds={} bs={} inline={} direct={} path={} iblock={} ver={}\n",
+                    bucket, object, put_object_size, shard_file_size_raw, erasure.data_shards, fi.erasure.block_size,
+                    is_inline_buffer, direct_inline_commit, write_path.metric_label(), storage_class_config.inline_block(), opts.versioned
+                );
+                if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open("/tmp/rustfs_inline_debug.log") {
+                    let _ = f.write_all(msg.as_bytes());
+                }
+                let _ = std::io::stderr().write_all(msg.as_bytes());
+            }
             rustfs_io_metrics::record_put_object_path(write_path.metric_label());
             let writer_setup_stage_start = collect_stage_timing.then(Instant::now);
             let (mut writers, errors) = if direct_inline_commit {

crates/ecstore/src/store/object.rs

@@ -3194,7 +3194,7 @@ impl ECStore {
 
         // Default return value
         let mut del_objects = vec![DeletedObject::default(); objects.len()];
-        let accounting = vec![None; objects.len()];
+        let mut accounting = vec![None; objects.len()];
 
         let mut del_errs = Vec::with_capacity(objects.len());
         for _ in 0..objects.len() {

crates/ecstore/src/store/rebalance/support.rs

@@ -271,7 +271,7 @@ pub(super) fn resolve_latest_object_info_candidates(
             .filter(|candidate| latest_candidate_mod_time(candidate) == Some(latest_mod_time))
             .collect::<Vec<_>>();
 
-        latest_candidates.sort_by_key(|candidate| std::cmp::Reverse(candidate.idx));
+        latest_candidates.sort_by(|left, right| right.idx.cmp(&left.idx));
 
         let Some(winner) = latest_candidates.first() else {
             return Err(Error::ErasureReadQuorum);

crates/kms/tests/vault_ha_failover_live.rs

@@ -16,14 +16,14 @@
 //!
 //! `scripts/test/vault_ha_kms_live.sh` owns the official Vault containers and
 //! kills the active node while this test continuously decrypts through a
-//! surviving standby. KV2 and Transit must recover after the bounded circuit
+//! surviving standby. KV2 and Transit requests must remain successful, use a
-//! interval, use a bounded number of attempts, and leave the circuit and
+//! bounded number of attempts, and leave the circuit and in-flight gauges at
-//! in-flight gauges at zero after a new leader is elected.
+//! zero after a new leader is elected.
 
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
+use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
-use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
 use metrics_util::MetricKind;
@@ -43,11 +43,6 @@ const OPERATION_ATTEMPTS: &str = "rustfs_kms_backend_operation_attempts";
 const IN_FLIGHT: &str = "rustfs_kms_backend_in_flight";
 const CIRCUIT_OPEN: &str = "rustfs_kms_backend_circuit_open";
 const MAX_ATTEMPTS: u32 = 10;
-const ATTEMPT_TIMEOUT: Duration = Duration::from_secs(2);
-const HEALTHY_PROGRESS_TIMEOUT: Duration = Duration::from_secs(20);
-// The circuit remains open for 30s after five failed attempts.
-const POST_FAILOVER_PROGRESS_TIMEOUT: Duration = Duration::from_secs(35);
-const FAILOVER_ERROR_POLL_INTERVAL: Duration = Duration::from_millis(100);
 
 type MetricEntry = (
     metrics_util::CompositeKey,
@@ -69,7 +64,7 @@ fn config(backend: KmsBackend, backend_config: BackendConfig) -> KmsConfig {
         backend,
         backend_config,
         allow_insecure_dev_defaults: true,
-        timeout: ATTEMPT_TIMEOUT,
+        timeout: Duration::from_secs(2),
         retry_attempts: MAX_ATTEMPTS,
         enable_cache: false,
         ..KmsConfig::default()
@@ -169,31 +164,14 @@ fn retryable_failures(snapshot: &[MetricEntry], operation: &str) -> u64 {
         .sum()
 }
 
-async fn wait_for_count(
+async fn wait_for_count(counter: &AtomicU64, minimum: u64, description: &str) {
-    counter: &AtomicU64,
+    tokio::time::timeout(Duration::from_secs(20), async {
-    failure: &Mutex<Option<String>>,
-    minimum: u64,
-    description: &str,
-    timeout: Duration,
-) {
-    tokio::time::timeout(timeout, async {
         while counter.load(Ordering::SeqCst) < minimum {
-            if let Some(error) = failure.lock().expect("decrypt failure lock poisoned").as_ref() {
-                panic!(
-                    "{description} worker failed after {} successful decrypts: {error}",
-                    counter.load(Ordering::SeqCst)
-                );
-            }
             tokio::time::sleep(Duration::from_millis(25)).await;
         }
     })
     .await
-    .unwrap_or_else(|_| {
+    .unwrap_or_else(|_| panic!("timed out waiting for {description}"));
-        panic!(
-            "timed out after {timeout:?} waiting for {description}: completed {}, expected {minimum}",
-            counter.load(Ordering::SeqCst)
-        )
-    });
 }
 
 async fn wait_for_file(path: &Path, description: &str) {
@@ -211,8 +189,7 @@ async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
     request: DecryptRequest,
     expected: Vec<u8>,
     completed: Arc<AtomicU64>,
-    allow_failover_errors: Arc<AtomicBool>,
+    failed: Arc<AtomicBool>,
-    failure: Arc<Mutex<Option<String>>>,
     stop: CancellationToken,
 ) {
     while !stop.is_cancelled() {
@@ -220,18 +197,8 @@ async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
             Ok(response) if response.plaintext == expected => {
                 completed.fetch_add(1, Ordering::SeqCst);
             }
-            Ok(_) => {
+            Ok(_) | Err(_) => {
-                *failure.lock().expect("decrypt failure lock poisoned") =
+                failed.store(true, Ordering::SeqCst);
-                    Some("decrypt returned unexpected plaintext".to_string());
-                return;
-            }
-            Err(rustfs_kms::KmsError::BackendError { .. } | rustfs_kms::KmsError::OperationTimedOut { .. })
-                if allow_failover_errors.load(Ordering::SeqCst) =>
-            {
-                tokio::time::sleep(FAILOVER_ERROR_POLL_INTERVAL).await;
-            }
-            Err(error) => {
-                *failure.lock().expect("decrypt failure lock poisoned") = Some(error.to_string());
                 return;
             }
         }
@@ -329,9 +296,7 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
     );
 
     let stop = CancellationToken::new();
-    let allow_failover_errors = Arc::new(AtomicBool::new(false));
+    let failed = Arc::new(AtomicBool::new(false));
-    let kv2_failure = Arc::new(Mutex::new(None));
-    let transit_failure = Arc::new(Mutex::new(None));
     let kv2_completed = Arc::new(AtomicU64::new(0));
     let transit_completed = Arc::new(AtomicU64::new(0));
     let kv2_worker = tokio::spawn(decrypt_loop(
@@ -339,8 +304,7 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
         kv2_request,
         kv2_data_key.plaintext_key,
         Arc::clone(&kv2_completed),
-        Arc::clone(&allow_failover_errors),
+        Arc::clone(&failed),
-        Arc::clone(&kv2_failure),
         stop.clone(),
     ));
     let transit_worker = tokio::spawn(decrypt_loop(
@@ -348,21 +312,12 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
         transit_request,
         transit_data_key.plaintext_key,
         Arc::clone(&transit_completed),
-        Arc::clone(&allow_failover_errors),
+        Arc::clone(&failed),
-        Arc::clone(&transit_failure),
         stop.clone(),
     ));
 
-    wait_for_count(&kv2_completed, &kv2_failure, 2, "two healthy KV2 decrypts", HEALTHY_PROGRESS_TIMEOUT).await;
+    wait_for_count(&kv2_completed, 2, "two healthy KV2 decrypts").await;
-    wait_for_count(
+    wait_for_count(&transit_completed, 2, "two healthy Transit decrypts").await;
-        &transit_completed,
-        &transit_failure,
-        2,
-        "two healthy Transit decrypts",
-        HEALTHY_PROGRESS_TIMEOUT,
-    )
-    .await;
-    allow_failover_errors.store(true, Ordering::SeqCst);
     std::fs::write(&marker, b"ready").expect("publish failover readiness marker");
 
     wait_for_file(&elected, "the replacement Vault leader").await;
@@ -371,39 +326,18 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
 
     let kv2_after_election = kv2_completed.load(Ordering::SeqCst) + 2;
     let transit_after_election = transit_completed.load(Ordering::SeqCst) + 2;
-    wait_for_count(
+    wait_for_count(&kv2_completed, kv2_after_election, "post-failover KV2 decrypts").await;
-        &kv2_completed,
+    wait_for_count(&transit_completed, transit_after_election, "post-failover Transit decrypts").await;
-        &kv2_failure,
-        kv2_after_election,
-        "post-failover KV2 decrypts",
-        POST_FAILOVER_PROGRESS_TIMEOUT,
-    )
-    .await;
-    wait_for_count(
-        &transit_completed,
-        &transit_failure,
-        transit_after_election,
-        "post-failover Transit decrypts",
-        POST_FAILOVER_PROGRESS_TIMEOUT,
-    )
-    .await;
 
     stop.cancel();
     kv2_worker.await.expect("KV2 decrypt worker must join");
     transit_worker.await.expect("Transit decrypt worker must join");
-    assert!(
+    assert!(!failed.load(Ordering::SeqCst), "no decrypt may fail or return different plaintext");
-        kv2_failure.lock().expect("KV2 failure lock poisoned").is_none(),
-        "no KV2 decrypt may fail or return different plaintext"
-    );
-    assert!(
-        transit_failure.lock().expect("Transit failure lock poisoned").is_none(),
-        "no Transit decrypt may fail or return different plaintext"
-    );
 }
 
 #[test]
 #[ignore = "requires a real three-node Vault Raft cluster; run scripts/test/vault_ha_kms_live.sh"]
-fn vault_raft_leader_failure_recovers_kv2_and_transit_decrypts() {
+fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() {
     let recorder = DebuggingRecorder::new();
     let snapshotter = recorder.snapshotter();
     metrics::with_local_recorder(&recorder, || {
@@ -415,6 +349,11 @@ fn vault_raft_leader_failure_recovers_kv2_and_transit_decrypts() {
     });
     let snapshot = snapshotter.snapshot().into_vec();
 
+    assert_eq!(
+        counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "circuit_open")]),
+        0,
+        "a bounded leader election must not open the circuit"
+    );
     assert_eq!(
         counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]),
         0,

deny.toml

@@ -43,6 +43,9 @@ allow-git = [
     # RustFS fork carrying presigned expiry and constant-time authentication fixes.
     # owner: rustfs-maintainers  review: 2026-10
     "https://github.com/rustfs/s3s.git",
+    # MiMalloc fork pinned for hotpath allocation counting support.
+    # owner: houseme  review: 2026-10
+    "https://github.com/xonatius/mimalloc_rust.git",
 ]
 
 [bans]

docs/testing/README.md

@@ -158,11 +158,10 @@ added by backlog#1153 infra-4.
 
 ## Coverage
 
-Workspace line coverage is measured weekly. Pull requests that touch iam, kms,
+Line coverage is measured **weekly, not per-PR**, and is non-blocking: it
-policy, or crypto also run a non-required, report-only comparison against
+exists for visibility and trend, never as a required check. Per-crate ratchets
-`.config/coverage-baselines.toml`. During calibration, a regression is recorded
+for the security-critical crates (iam / kms / policy / crypto) build on this
-in the job summary without failing the job; missing or malformed coverage
+baseline later (backlog#1153 infra-6, report-only first).
-evidence still fails closed (backlog#1153 infra-6).
 
 - **CI**: `.github/workflows/coverage.yml` runs every Sunday and on manual
   dispatch: `cargo llvm-cov nextest --workspace --exclude e2e_test` under the
@@ -175,12 +174,6 @@ evidence still fails closed (backlog#1153 infra-6).
   plus the full suite). It prints the same per-crate table via
   `scripts/coverage_per_crate.py` and writes `target/llvm-cov/lcov.info` and
   `coverage.json`.
-- **Security-critical ratchet**: relevant pull requests compare iam / kms /
-  policy / crypto line coverage with the versioned baseline. Drops greater than
-  the configured one-percentage-point calibration threshold are marked
-  `REGRESSION (report-only)`. The weekly summary runs the same comparison so
-  calibration continues even when no relevant pull request is open. Baseline
-  changes require a linked coverage run and a reviewed explanation.
 - **Trend comparison**: each run's job summary is the weekly per-crate
   snapshot — open two runs from the Actions history (workflow "coverage") and
   compare their tables. For line-level diffs, download the two runs'

docs/testing/ci-gates.md

@@ -1,149 +0,0 @@
-# CI gate matrix
-
-This file is the source of truth for which validation runs on each event, its
-configured wall-clock budget, and whether it can block a merge. Test taxonomy,
-naming, and nextest serialization rules remain in [README.md](README.md); e2e
-membership and counts remain in
-[e2e-suite-inventory.md](e2e-suite-inventory.md).
-
-The distinction between **required** and **report-only** is load-bearing:
-a failing job blocks a merge only when its exact check name is present in the
-live `main` ruleset. A workflow name, a `merge_group` trigger, or a red PR check
-does not make a job required by itself.
-
-## Required merge checks
-
-The live `main` ruleset (`6436880`) currently requires exactly these contexts:
-
-| Required context | Producer | Validation |
-|---|---|---|
-| `CLA Check` | `.github/workflows/cla.yml` | Contributor agreement |
-| `Quick Checks` | `.github/workflows/ci.yml` | Formatting and repository guard scripts |
-| `Test and Lint` | `.github/workflows/ci.yml` | Clippy, workspace nextest excluding `e2e_test`, doctests, and migration proofs |
-
-For pull requests limited to the paths excluded by the main CI workflow,
-`.github/workflows/ci-docs-only.yml` reports `Quick Checks` and
-`Test and Lint` under the same names. It runs the real quick checks and the
-planning-document guard; it does not claim that Rust compilation or runtime
-tests ran. Despite the workflow name, these paths also include selected deploy,
-workflow, and lock files.
-
-Verify the live rule rather than trusting this snapshot before changing merge
-policy:
-
-```bash
-gh api repos/rustfs/rustfs/rulesets/6436880 \
-  --jq '.rules[] | select(.type == "required_status_checks") | .parameters'
-```
-
-The ruleset currently has `strict_required_status_checks_policy=false`.
-`Continuous Integration` accepts `merge_group` events and runs `e2e-full` for
-them, but `End-to-End Tests (full merge gate)` is not currently a required
-context. Therefore the repository is prepared to test a merge-queue SHA, but
-the workflow alone does not prove that every merge passed that lane.
-
-## Pull request and merge matrix
-
-Budgets below are job `timeout-minutes`, not typical runtimes. “Report-only”
-means the result is visible and actionable but is not in the live required
-context list.
-
-| Event | Validation | Budget | Merge status | Reproduction |
-|---|---|---:|---|---|
-| PR, non-doc change | `Quick Checks` | 10 min | Required | `make pre-commit` (broader local umbrella) |
-| PR, non-doc change | `Test and Lint` | 90 min | Required | `cargo nextest run --profile ci --all --exclude e2e_test` |
-| PR, non-doc change | `Typos` | 10 min | Report-only | `typos` |
-| PR, non-doc change | `ILM Integration (serial)` | 90 min | Report-only | Use the exact command in `.github/workflows/ci.yml` |
-| PR, non-doc change | rio-v2 / swift / sftp test-and-lint variants | 90 min each | Report-only | `cargo nextest run` with the workflow's feature set |
-| PR, non-doc change | `Build RustFS Debug Binary` | 30 min | Report-only; prerequisite for black-box lanes | `cargo build -p rustfs --bins` |
-| PR, non-doc change | `io_uring Integration (real)` | 30 min | Report-only | `cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture` |
-| PR, non-doc change | `End-to-End Tests` (`e2e-smoke` plus `s3s-e2e`) | 30 min | Report-only | `cargo nextest run --profile e2e-smoke -p e2e_test`; then `./scripts/e2e-run.sh ./target/debug/rustfs <data-dir>` |
-| PR, non-doc change | `S3 Implemented Tests` | 60 min | Report-only | Build `rustfs`, then run `scripts/s3-tests/run.sh` with `DEPLOY_MODE=binary`, `TEST_MODE=single`, and `MAXFAIL=0` |
-| PR, non-doc change | `S3 Lifecycle Behavior Tests` | 30 min | Report-only | Use the accelerated scanner environment in `.github/workflows/ci.yml` with `scripts/s3-tests/run.sh` |
-| PR touching dependency or workflow inputs | Cargo Deny / Workflow Pin Report / Dependency Review | 20 / 5 / 30 min | Report-only | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
-| PR touching architecture rules or architecture docs | `Architecture Migration Rules` | 10 min | Report-only | `scripts/check_architecture_migration_rules.sh` |
-| PR touching Nix or workspace manifests | `Nix Build & Check` | 60 min | Report-only | `nix flake check` |
-| PR limited to main-CI-excluded paths | companion `Quick Checks` and `Test and Lint` | 10 min each | Required | `git diff --check`; `make doc-paths-check` when documentation paths changed |
-| `merge_group` | Standard CI plus `e2e-full` | 55 min for `e2e-full` | Standard required contexts only; `e2e-full` report-only | `cargo nextest run --profile e2e-full -p e2e_test` |
-| Push to `main` | Standard CI plus `e2e-full` | 55 min for `e2e-full` | Post-merge detection | Same as `merge_group` |
-| PR touching fuzz inputs or harness paths | Build plus five 60-second fuzz smoke targets | 60 min build; 30 min per target | Report-only | `MAX_TOTAL_TIME=60 ./scripts/fuzz/run.sh` |
-| PR touching selected ecstore disk/format paths | `Rename Safety` on Windows | 60 min | Report-only | Run the four `cargo test -p rustfs-ecstore --lib <filter>` commands in `windows-filesystem.yml` on Windows |
-
-The authoritative e2e filters live in `.config/nextest.toml`; extend a profile
-instead of adding a second ad-hoc selector. Before a profile runs,
-`scripts/check_test_wiring.py` compares its exact membership to the committed
-digest so a silent test drop fails closed.
-
-## Scheduled and manual validation
-
-Scheduled lanes are independent fault domains. They do not block a pull
-request, but their workflow-local gate can fail the run and scheduled failures
-are routed to the shared failure-issue action. The scheduled-validation
-watchdog and freshness workflow separately detect incomplete runs and missing
-schedules.
-
-| Cadence (UTC unless noted) | Workflow / validation | Budget | Verdict and artifacts | Reproduction |
-|---|---|---:|---|---|
-| Daily 02:17 | Fuzz: five nightly corpus targets | 60 min build; 60 min per target | Gate; corpus/crash artifacts, scheduled failure alert | `MAX_TOTAL_TIME=<seconds> ./scripts/fuzz/run.sh` |
-| Daily 03:17 | MinIO interop (EC + SSE read parity) | 40 min | Gate; scheduled failure alert | Dispatch `minio-interop.yml` or follow its pinned Docker fixture steps |
-| Daily 04:29 | Replication / cluster-fault / protocol e2e | 45 / 90 / 90 min | Three independent gates; JUnit, membership, and server logs | `cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--profile e2e-nightly`; `-j 1 --profile e2e-protocols` |
-| Daily 06:31 | Warp performance A/B | 180 min | Regression budget gate; A/B summaries and server logs | `bash scripts/run_hotpath_warp_abba.sh --help` |
-| Daily 00:07 Asia/Shanghai (16:07 UTC previous day) | Nightly GNU build and Vault lanes | 150 / 90 / 60 min | Build, live Vault, and HA failover gates | Use the commands and pinned Vault images in `nightly-gnu.yml` |
-| Daily 03:23 | Security Audit | 20 / 5 min, plus 30 min on PR dependency review | Cargo Deny and workflow-pin gates; scheduled failure alert | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
-| Daily 23:47 | Scheduled Validation Freshness | 10 min | Fails when a critical schedule was never created or is stale | Dispatch `scheduled-validation-freshness.yml` |
-| Sunday 00:11 | Full `Continuous Integration` matrix | Per-job budgets above | Weekly variant coverage, including dormant rio-v2 binary/e2e lanes | Dispatch `ci.yml` |
-| Sunday 01:13 | Seven-platform build matrix | 150 min per platform | Build/package integrity; scheduled failure alert | Dispatch `build.yml` with an exact platform set |
-| Sunday 02:19 | Ceph s3-tests full sweep: single and real four-node, four shards each | 180 min per shard | Compatibility gate; report, JUnit, exact node IDs, and server logs | `scripts/s3-tests/run.sh` against an existing single or distributed target |
-| Sunday 06:41 | Mint | 120 min | **Report-only by design**; per-suite PASS/FAIL/NA and raw `log.json` | Reproduce the pinned Docker sequence in `mint.yml` or dispatch it |
-| Sunday 07:43 | Workspace line coverage | 120 min | Report-only trend; lcov and JSON retained 90 days | `make coverage` |
-| Monthly, day 1 06:37 | Runner Hygiene | 15 min | Validates runner ephemerality; scheduled failure alert | Dispatch `runner-hygiene.yml` |
-
-Manual `workflow_dispatch` exists for the scheduled workflows above. Manual
-runs are debugging evidence and intentionally do not open scheduled-failure
-issues. A manual performance run may explicitly allow a known regression; that
-override must not be treated as an ordinary passing baseline.
-
-## Release validation
-
-Release validation is post-merge and tag-driven; it does not substitute for a
-pull-request gate.
-
-| Event | Validation | Budget | Result |
-|---|---|---:|---|
-| Push to `main` or weekly schedule | `Build and Release` platform matrix | 150 min per platform | Build artifacts for all selected targets; no release publication on a main push |
-| Valid release or preview tag | `Build and Release` plus asset checks | 150 min per platform | Draft release, checksummed assets, and publish step |
-| Successful non-preview release-tag build | Docker image build and image scan | 60 min build; 30 min scan | Multi-architecture images plus vulnerability report |
-| Successful release-tag build | DEB/RPM packaging | 30 min per architecture | Packages and checksum files uploaded to the release |
-| Successful non-preview release-tag build | Helm template test and package | 30 min build; 30 min publish | Versioned chart and repository index |
-
-Use an exact preview tag for end-to-end release rehearsal. Manual dispatches
-are backfill/debug paths and do not prove the automatic `workflow_run` chain.
-
-## Evidence requirements
-
-A green check is useful only when it proves the intended behavior ran:
-
-- Record the exact commit SHA and run URL.
-- Separate product failure from runner prerequisites, service readiness, and
-  cancellation. Repair the precondition, then rerun the exact workload.
-- Preserve membership manifests, JUnit, raw compatibility logs, seeds, and
-  server logs where the workflow provides them.
-- For a bug fix or a new fault checker, provide sensitivity evidence: the old
-  behavior or an intentional mutation must fail the new oracle, and the fixed
-  behavior must pass it.
-- Never promote a report-only lane to required from one green run. Require at
-  least 14 days and 30 representative pull requests with at least 99% complete
-  execution, then update the ruleset and this table together.
-
-## Change checklist
-
-Update this file in the same pull request when any of these change:
-
-- workflow triggers, job names, timeouts, or nextest profile ownership;
-- required status contexts or strict/merge-queue policy;
-- scheduled cadence, alert routing, artifact contract, or local reproduction;
-- report-only versus gating semantics.
-
-Do not copy per-module test counts here. Update
-[e2e-suite-inventory.md](e2e-suite-inventory.md) and its enforced membership
-digest instead.

docs/testing/e2e-suite-inventory.md

@@ -84,7 +84,7 @@
 | protocols | 16 | 🌙 |
 | quota_test | 14 |  |
 | reliability_disk_fault_test | 4 |  |
-| reliant | 25 | 19 ✅ |
+| reliant | 29 | 19 ✅ |
 | replication_extension_test | 75 | 20 ✅ +55 🌙 |
 | security_boundary_test | 4 |  |
 | server_startup_failfast_test | 1 |  |

rustfs/Cargo.toml

@@ -336,13 +336,13 @@ opentelemetry = { workspace = true }
 tracing-opentelemetry = { workspace = true }
 # Data structures
 hashbrown = { workspace = true, features = ["serde", "rayon"] }
-rustfs-mimalloc = { workspace = true }
+mimalloc = { workspace = true }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 libsystemd.workspace = true
 
 [target.'cfg(not(target_os = "windows"))'.dependencies]
-rustfs-mimalloc-sys.workspace = true
+libmimalloc-sys.workspace = true
 
 [dev-dependencies]
 uuid = { workspace = true, features = ["v4", "v5", "fast-rng", "macro-diagnostics"] }

rustfs/src/allocator_reclaim.rs

@@ -369,8 +369,14 @@ pub fn allocator_reclaim_controller_snapshot(ctx: &CancellationToken) -> Allocat
 }
 
 #[cfg(not(target_os = "windows"))]
+#[allow(unsafe_code)]
 fn collect_allocator_memory(force: bool) -> Result<(), String> {
-    rustfs_mimalloc::MiMalloc::collect(force);
+    // SAFETY: `mi_collect` is provided by the active global allocator backend
+    // on this target family. It is explicitly intended to reclaim retained
+    // pages/segments and does not require additional invariants from the caller.
+    unsafe {
+        libmimalloc_sys::mi_collect(force);
+    }
     Ok(())
 }
 

rustfs/src/main.rs

@@ -26,22 +26,22 @@ struct MiMallocAllocator;
 unsafe impl GlobalAlloc for MiMallocAllocator {
     unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
         // SAFETY: the caller upholds GlobalAlloc's contract for layout.
-        unsafe { rustfs_mimalloc::MiMalloc.alloc(layout) }
+        unsafe { mimalloc::MiMalloc.alloc(layout) }
     }
 
     unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
         // SAFETY: the caller upholds GlobalAlloc's contract for layout.
-        unsafe { rustfs_mimalloc::MiMalloc.alloc_zeroed(layout) }
+        unsafe { mimalloc::MiMalloc.alloc_zeroed(layout) }
     }
 
     unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
         // SAFETY: ptr and layout came from this allocator and are forwarded unchanged.
-        unsafe { rustfs_mimalloc::MiMalloc.dealloc(ptr, layout) }
+        unsafe { mimalloc::MiMalloc.dealloc(ptr, layout) }
     }
 
     unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
         // SAFETY: ptr and layout came from this allocator and are forwarded unchanged.
-        unsafe { rustfs_mimalloc::MiMalloc.realloc(ptr, layout, new_size) }
+        unsafe { mimalloc::MiMalloc.realloc(ptr, layout, new_size) }
     }
 }
 
@@ -51,7 +51,7 @@ static GLOBAL: hotpath::CountingAllocator<MiMallocAllocator> = hotpath::Counting
 
 #[cfg(not(all(feature = "hotpath", feature = "hotpath-alloc")))]
 #[global_allocator]
-static GLOBAL: rustfs_mimalloc::MiMalloc = rustfs_mimalloc::MiMalloc;
+static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
 fn main() {
     let _hotpath_guard = hotpath::HotpathGuardBuilder::new("main").build();
@@ -71,9 +71,8 @@ mod tests {
         allocation.extend_from_slice(&[7_u8; 64]);
 
         assert_eq!(allocation.len(), 64);
-        let heap = rustfs_mimalloc::heap::Heap::main();
         // SAFETY: the live Vec pointer is valid to inspect for heap ownership.
-        assert!(unsafe { heap.contains(allocation.as_ptr()) });
+        assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(allocation.as_ptr().cast()) });
     }
 
     #[test]
@@ -86,13 +85,12 @@ mod tests {
         let layout = Layout::from_size_align(32, 8).expect("valid test allocation layout");
         let grown_layout = Layout::from_size_align(64, 8).expect("valid grown test allocation layout");
         let allocator = super::MiMallocAllocator;
-        let heap = rustfs_mimalloc::heap::Heap::main();
 
         // SAFETY: The pointer is checked for null before use and later released
         // through the same allocator with the corresponding layout.
         let ptr = unsafe { allocator.alloc_zeroed(layout) };
         assert!(!ptr.is_null());
-        assert!(unsafe { heap.contains(ptr) });
+        assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(ptr.cast()) });
         assert!(unsafe { std::slice::from_raw_parts(ptr, 32).iter().all(|byte| *byte == 0) });
 
         // SAFETY: `ptr` was allocated by `allocator` with `layout`; on failure
@@ -104,7 +102,7 @@ mod tests {
             panic!("mimalloc realloc failed in allocator smoke test");
         }
 
-        assert!(unsafe { heap.contains(grown_ptr) });
+        assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(grown_ptr.cast()) });
         // SAFETY: `grown_ptr` was reallocated by `allocator` and is released
         // with the matching grown layout.
         unsafe { allocator.dealloc(grown_ptr, grown_layout) };

rustfs/src/memory_observability.rs

@@ -17,7 +17,10 @@ use rustfs_io_metrics::{
     record_cpu_usage, record_memory_usage, record_process_memory_split,
 };
 use serde::Serialize;
+#[cfg(any(test, not(target_os = "windows")))]
 use serde_json::Value;
+#[cfg(not(target_os = "windows"))]
+use std::ffi::CStr;
 use std::path::Path;
 use std::sync::{Arc, Mutex, OnceLock};
 use std::time::Duration;
@@ -228,18 +231,7 @@ fn read_cgroup_memory_snapshot() -> Option<CgroupMemorySnapshot> {
     read_cgroup_v2().or_else(read_cgroup_v1)
 }
 
-fn read_allocator_memory_snapshot() -> Option<AllocatorMemorySnapshot> {
+#[cfg(any(test, not(target_os = "windows")))]
-    let json = rustfs_mimalloc::MiMalloc::stats_json();
-    if json.is_empty() {
-        return None;
-    }
-    let observation = parse_mimalloc_stats_json(&json)?;
-    Some(AllocatorMemorySnapshot {
-        backend: crate::allocator_reclaim::allocator_backend(),
-        observation,
-    })
-}
-
 fn numeric_json_value(value: &Value) -> Option<u64> {
     match value {
         Value::Number(number) => number
@@ -250,6 +242,7 @@ fn numeric_json_value(value: &Value) -> Option<u64> {
     }
 }
 
+#[cfg(any(test, not(target_os = "windows")))]
 fn numeric_json_field(value: &Value, field: &str) -> Option<u64> {
     match value {
         Value::Object(fields) => fields
@@ -261,6 +254,7 @@ fn numeric_json_field(value: &Value, field: &str) -> Option<u64> {
     }
 }
 
+#[cfg(any(test, not(target_os = "windows")))]
 fn mimalloc_stat_field(value: &Value, metric: &str, field: &str) -> Option<u64> {
     match value {
         Value::Object(fields) => {
@@ -277,10 +271,12 @@ fn mimalloc_stat_field(value: &Value, metric: &str, field: &str) -> Option<u64>
     }
 }
 
+#[cfg(any(test, not(target_os = "windows")))]
 fn mimalloc_stat_current(value: &Value, metric: &str) -> Option<u64> {
     mimalloc_stat_field(value, metric, "current")
 }
 
+#[cfg(any(test, not(target_os = "windows")))]
 fn mimalloc_stat_sum(value: &Value, metrics: &[&str], field: &str) -> Option<u64> {
     metrics
         .iter()
@@ -289,6 +285,7 @@ fn mimalloc_stat_sum(value: &Value, metrics: &[&str], field: &str) -> Option<u64
         .filter(|value| *value > 0)
 }
 
+#[cfg(any(test, not(target_os = "windows")))]
 fn parse_mimalloc_stats_json(stats_json: &str) -> Option<AllocatorMemoryObservation> {
     let value = serde_json::from_str::<Value>(stats_json).ok()?;
     let malloc_metrics = ["malloc_normal", "malloc_huge"];
@@ -315,6 +312,33 @@ fn parse_mimalloc_stats_json(stats_json: &str) -> Option<AllocatorMemoryObservat
     }
 }
 
+#[cfg(not(target_os = "windows"))]
+#[allow(unsafe_code)]
+fn read_allocator_memory_snapshot() -> Option<AllocatorMemorySnapshot> {
+    // SAFETY: `mi_stats_get_json` returns a null-terminated JSON buffer owned by
+    // mimalloc when called with a null input buffer. The mimalloc API requires
+    // freeing that buffer with `mi_free`; parsing finishes before the buffer is freed.
+    let observation = unsafe {
+        let stats_ptr = libmimalloc_sys::mi_stats_get_json(0, std::ptr::null_mut());
+        if stats_ptr.is_null() {
+            return None;
+        }
+
+        let observation = CStr::from_ptr(stats_ptr).to_str().ok().and_then(parse_mimalloc_stats_json);
+        libmimalloc_sys::mi_free(stats_ptr.cast());
+        observation?
+    };
+    Some(AllocatorMemorySnapshot {
+        backend: crate::allocator_reclaim::allocator_backend(),
+        observation,
+    })
+}
+
+#[cfg(target_os = "windows")]
+fn read_allocator_memory_snapshot() -> Option<AllocatorMemorySnapshot> {
+    None
+}
+
 fn configured_memory_observability_interval_secs() -> u64 {
     rustfs_utils::get_env_u64(ENV_MEMORY_OBSERVABILITY_INTERVAL_SECS, DEFAULT_MEMORY_OBSERVABILITY_INTERVAL_SECS).max(1)
 }
@@ -542,13 +566,6 @@ mod tests {
         assert_eq!(parse_mimalloc_stats_json(r#"{ "allocator": "unknown" }"#), None);
     }
 
-    #[test]
-    fn read_allocator_memory_snapshot_uses_mimalloc_stats_json() {
-        let snapshot = super::read_allocator_memory_snapshot();
-        #[cfg(not(target_os = "windows"))]
-        assert!(snapshot.is_some(), "allocator snapshot should be available on non-Windows");
-    }
-
     #[test]
     fn memory_observability_snapshot_reports_disabled_when_metrics_are_disabled() {
         let snapshot = build_memory_observability_status_snapshot(false, 15, false);

scripts/check_security_coverage.py

@@ -1,264 +0,0 @@
-#!/usr/bin/env python3
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Compare security-critical crate line coverage with the report-only baseline."""
-
-import argparse
-import json
-import math
-import os
-import sys
-import tempfile
-import tomllib
-from pathlib import Path
-
-from coverage_per_crate import fmt_pct, load_coverage
-
-
-SECURITY_CRATES = ("crates/iam", "crates/kms", "crates/policy", "crates/crypto")
-
-
-def load_baselines(path: str) -> tuple[float, dict[str, tuple[int, int]]]:
-    with open(path, "rb") as fh:
-        config = tomllib.load(fh)
-
-    if config.get("phase") != "report-only":
-        raise ValueError("coverage baseline phase must be report-only")
-
-    raw_allowed_drop = config["allowed_drop_percentage_points"]
-    if isinstance(raw_allowed_drop, bool) or not isinstance(raw_allowed_drop, (int, float)):
-        raise ValueError("allowed_drop_percentage_points must be a number")
-    allowed_drop = float(raw_allowed_drop)
-    if not math.isfinite(allowed_drop) or allowed_drop < 0:
-        raise ValueError("allowed_drop_percentage_points must be finite and non-negative")
-
-    baselines: dict[str, tuple[int, int]] = {}
-    for crate, values in config["crates"].items():
-        covered = values["covered"]
-        count = values["count"]
-        if type(covered) is not int or type(count) is not int:
-            raise ValueError(f"invalid baseline for {crate}: covered and count must be integers")
-        if covered < 0 or count <= 0 or covered > count:
-            raise ValueError(f"invalid baseline for {crate}: {covered}/{count}")
-        baselines[crate] = (covered, count)
-    missing = [crate for crate in SECURITY_CRATES if crate not in baselines]
-    unexpected = sorted(set(baselines).difference(SECURITY_CRATES))
-    if missing or unexpected:
-        raise ValueError(f"coverage baseline crate set mismatch: missing={missing}, unexpected={unexpected}")
-    return allowed_drop, baselines
-
-
-def compare(
-    current: dict[str, list[int]],
-    baselines: dict[str, tuple[int, int]],
-    allowed_drop: float,
-) -> list[tuple[str, int, int, int, int, float, bool]]:
-    rows = []
-    for crate, (baseline_covered, baseline_count) in baselines.items():
-        if crate not in current:
-            raise ValueError(f"coverage report is missing {crate}")
-        covered, count = current[crate]
-        if type(covered) is not int or type(count) is not int:
-            raise ValueError(f"invalid coverage for {crate}: covered and count must be integers")
-        if covered < 0 or count <= 0 or covered > count:
-            raise ValueError(f"invalid coverage for {crate}: {covered}/{count}")
-        current_pct = 100.0 * covered / count
-        baseline_pct = 100.0 * baseline_covered / baseline_count
-        delta = current_pct - baseline_pct
-        rows.append((crate, covered, count, baseline_covered, baseline_count, delta, delta < -allowed_drop))
-    return rows
-
-
-def print_report(rows: list[tuple[str, int, int, int, int, float, bool]], allowed_drop: float) -> None:
-    print("## Security-critical coverage ratchet (report-only)")
-    print()
-    print(f"Calibration threshold: a drop greater than {allowed_drop:.2f} percentage points is reported as a regression.")
-    print()
-    print("| Crate | Current | Baseline | Delta | Status |")
-    print("|---|---:|---:|---:|---|")
-    for crate, covered, count, baseline_covered, baseline_count, delta, regressed in rows:
-        status = "REGRESSION (report-only)" if regressed else "OK"
-        print(
-            f"| `{crate}` | {fmt_pct(covered, count)} ({covered}/{count}) "
-            f"| {fmt_pct(baseline_covered, baseline_count)} ({baseline_covered}/{baseline_count}) "
-            f"| {delta:+.2f} pp | {status} |"
-        )
-    print()
-    print("This calibration phase records regressions without failing the job; malformed or incomplete evidence still fails closed.")
-
-
-def self_test() -> None:
-    with tempfile.TemporaryDirectory() as tmp:
-        root = Path(tmp)
-        coverage = root / "coverage.json"
-        baseline = root / "baseline.toml"
-        coverage_data = {
-            "data": [
-                {
-                    "files": [
-                        {
-                            "filename": str(root / "crates/iam/src/lib.rs"),
-                            "summary": {"lines": {"covered": 80, "count": 100}},
-                        },
-                        {
-                            "filename": str(root / "crates/kms/src/lib.rs"),
-                            "summary": {"lines": {"covered": 90, "count": 100}},
-                        },
-                        {
-                            "filename": str(root / "crates/policy/src/lib.rs"),
-                            "summary": {"lines": {"covered": 90, "count": 100}},
-                        },
-                        {
-                            "filename": str(root / "crates/crypto/src/lib.rs"),
-                            "summary": {"lines": {"covered": 90, "count": 100}},
-                        },
-                    ],
-                    "totals": {"lines": {"covered": 350, "count": 400}},
-                }
-            ]
-        }
-        coverage.write_text(json.dumps(coverage_data), encoding="utf-8")
-        baseline_text = """phase = "report-only"
-allowed_drop_percentage_points = 1.0
-[crates."crates/iam"]
-covered = 90
-count = 100
-[crates."crates/kms"]
-covered = 85
-count = 100
-[crates."crates/policy"]
-covered = 90
-count = 100
-[crates."crates/crypto"]
-covered = 90
-count = 100
-"""
-        baseline.write_text(baseline_text, encoding="utf-8")
-        current, _ = load_coverage(str(coverage), str(root))
-        allowed_drop, baselines = load_baselines(str(baseline))
-        rows = compare(current, baselines, allowed_drop)
-        assert [row[-1] for row in rows] == [True, False, False, False]
-        try:
-            compare({"crates/iam": current["crates/iam"]}, baselines, allowed_drop)
-        except ValueError as error:
-            assert str(error) == "coverage report is missing crates/kms"
-        else:
-            raise AssertionError("missing crate must fail closed")
-        try:
-            compare({**current, "crates/iam": [101, 100]}, baselines, allowed_drop)
-        except ValueError as error:
-            assert str(error) == "invalid coverage for crates/iam: 101/100"
-        else:
-            raise AssertionError("invalid coverage must fail closed")
-        for invalid_threshold in ("true", '"1.0"', "nan", "inf", "-inf"):
-            baseline.write_text(
-                baseline_text.replace("allowed_drop_percentage_points = 1.0", f"allowed_drop_percentage_points = {invalid_threshold}"),
-                encoding="utf-8",
-            )
-            try:
-                load_baselines(str(baseline))
-            except ValueError:
-                pass
-            else:
-                raise AssertionError(f"non-finite threshold {invalid_threshold} must fail closed")
-        for field, invalid_values in (
-            ("covered", ("true", '"90"', "90.0", "90.5")),
-            ("count", ("true", '"100"', "100.0", "100.5")),
-        ):
-            for invalid_value in invalid_values:
-                baseline.write_text(
-                    baseline_text.replace(f"{field} = {90 if field == 'covered' else 100}", f"{field} = {invalid_value}", 1),
-                    encoding="utf-8",
-                )
-                try:
-                    load_baselines(str(baseline))
-                except ValueError:
-                    pass
-                else:
-                    raise AssertionError(f"non-integer baseline {field} {invalid_value} must fail closed")
-        for covered, count in (
-            (True, 100),
-            (80, True),
-            (80.0, 100),
-            (80, 100.0),
-            (float("nan"), 100),
-            (80, float("inf")),
-        ):
-            try:
-                compare({**current, "crates/iam": [covered, count]}, baselines, allowed_drop)
-            except ValueError:
-                pass
-            else:
-                raise AssertionError(f"invalid aggregate coverage {covered}/{count} must fail closed")
-        lines = coverage_data["data"][0]["files"][0]["summary"]["lines"]
-        for field, invalid_values in (
-            ("covered", (True, "80", 80.0, 80.5, float("nan"), float("inf"), float("-inf"))),
-            ("count", (True, "100", 100.0, 100.5, float("nan"), float("inf"), float("-inf"))),
-        ):
-            original = lines[field]
-            for invalid_value in invalid_values:
-                lines[field] = invalid_value
-                coverage.write_text(json.dumps(coverage_data), encoding="utf-8")
-                try:
-                    load_coverage(str(coverage), str(root))
-                except ValueError:
-                    pass
-                else:
-                    raise AssertionError(f"invalid raw coverage {field} {invalid_value} must fail closed")
-            lines[field] = original
-        baseline.write_text(
-            baseline_text.replace(
-                '[crates."crates/crypto"]\ncovered = 90\ncount = 100\n',
-                "",
-            ),
-            encoding="utf-8",
-        )
-        try:
-            load_baselines(str(baseline))
-        except ValueError:
-            pass
-        else:
-            raise AssertionError("missing security-crate baseline must fail closed")
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("coverage_json", nargs="?")
-    parser.add_argument("--baseline", default=".config/coverage-baselines.toml")
-    parser.add_argument("--repo-root", default=os.getcwd())
-    parser.add_argument("--self-test", action="store_true")
-    args = parser.parse_args()
-
-    if args.self_test:
-        self_test()
-        print("security coverage self-test passed")
-        return 0
-    if not args.coverage_json:
-        parser.error("coverage_json is required unless --self-test is used")
-
-    try:
-        current, _ = load_coverage(args.coverage_json, os.path.abspath(args.repo_root))
-        allowed_drop, baselines = load_baselines(args.baseline)
-        rows = compare(current, baselines, allowed_drop)
-    except (OSError, ValueError, KeyError, IndexError, json.JSONDecodeError, tomllib.TOMLDecodeError) as error:
-        print(f"error: {error}", file=sys.stderr)
-        return 1
-
-    print_report(rows, allowed_drop)
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

scripts/coverage_per_crate.py

@@ -47,31 +47,6 @@ def fmt_pct(covered: int, count: int) -> str:
     return f"{100.0 * covered / count:.2f}%" if count else "—"
 
 
-def _line_counts(lines: dict[str, int], source: str) -> tuple[int, int]:
-    covered = lines["covered"]
-    count = lines["count"]
-    if type(covered) is not int or type(count) is not int or covered < 0 or count < 0 or covered > count:
-        raise ValueError(f"invalid line coverage for {source}: {covered}/{count}")
-    return covered, count
-
-
-def load_coverage(path: str, root: str) -> tuple[dict[str, list[int]], dict[str, int]]:
-    with open(path, encoding="utf-8") as fh:
-        export = json.load(fh)
-
-    data = export["data"][0]
-    files = data["files"]
-    total_covered, total_count = _line_counts(data["totals"]["lines"], "totals")
-
-    crates: dict[str, list[int]] = {}
-    for f in files:
-        covered, count = _line_counts(f["summary"]["lines"], f["filename"])
-        acc = crates.setdefault(crate_label(f["filename"], root), [0, 0])
-        acc[0] += covered
-        acc[1] += count
-    return crates, {"covered": total_covered, "count": total_count}
-
-
 def main() -> int:
     if len(sys.argv) < 2 or len(sys.argv) > 3:
         print(__doc__.strip(), file=sys.stderr)
@@ -79,12 +54,24 @@ def main() -> int:
     path = sys.argv[1]
     root = os.path.abspath(sys.argv[2] if len(sys.argv) == 3 else os.getcwd())
 
+    with open(path, encoding="utf-8") as fh:
+        export = json.load(fh)
+
     try:
-        crates, totals = load_coverage(path, root)
+        data = export["data"][0]
-    except (KeyError, IndexError, ValueError) as exc:
+        files = data["files"]
+        totals = data["totals"]["lines"]
+    except (KeyError, IndexError) as exc:
         print(f"error: unexpected llvm-cov JSON shape ({exc})", file=sys.stderr)
         return 1
 
+    crates: dict[str, list[int]] = {}
+    for f in files:
+        lines = f["summary"]["lines"]
+        acc = crates.setdefault(crate_label(f["filename"], root), [0, 0])
+        acc[0] += lines["covered"]
+        acc[1] += lines["count"]
+
     rows = sorted(
         crates.items(),
         key=lambda kv: (100.0 * kv[1][0] / kv[1][1]) if kv[1][1] else 101.0,

scripts/test/vault_ha_kms_live.sh

@@ -241,7 +241,7 @@ env \
   RUSTFS_TEST_VAULT_FAILOVER_MARKER="$MARKER" \
   RUSTFS_TEST_VAULT_OLD_LEADER="$OLD_LEADER" \
   cargo test -p rustfs-kms --test vault_ha_failover_live \
-    vault_raft_leader_failure_recovers_kv2_and_transit_decrypts -- \
+    vault_raft_leader_failure_preserves_kv2_and_transit_decrypts -- \
     --ignored --nocapture --test-threads=1 &
 TEST_PID=$!