From b3a781bce06d68d8cc0a7eb844d6660c605a56ce Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Thu, 16 Jul 2026 04:34:18 +0800 Subject: [PATCH] test(replication): pin SSE replication contracts (#4883) --- .config/nextest.toml | 10 +- .github/workflows/e2e-replication-nightly.yml | 10 +- .../src/replication_extension_test.rs | 428 +++++++++++++++++- docs/testing/e2e-suite-inventory.md | 20 +- 4 files changed, 451 insertions(+), 17 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 138d8711d..e775eaede 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -152,7 +152,7 @@ test-group = 'ecstore-serial-flaky' # the nightly profile derives its set as "the replication module MINUS this # allowlist", so any new replication test lands in nightly by default (never # silently unrun) until it is explicitly blessed as fast here. Keep the two -# regexes byte-identical. Count invariant: 20 here + 24 nightly = 44 total +# regexes byte-identical. Count invariant: 20 here + 27 nightly = 47 total # (authority: `cargo nextest list`; docs/testing/e2e-suite-inventory.md). # HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane # (#4724) because they set a loopback (127.0.0.1) replication target that the @@ -203,8 +203,10 @@ fail-fast = false # tests that are unfit for the per-PR e2e-smoke gate: # # * 2 remote-target TLS validation tests. -# * 9 bucket-replication data-plane tests — they PUT/delete objects and poll -# until source and target converge; two replicate over HTTPS. +# * 12 bucket-replication data-plane/helper tests — they PUT/delete objects +# and poll until source and target converge; two replicate over HTTPS, two +# pin active SSE failure contracts, and one guards event/history observers. +# The SSE-S3 contract remains ignored under backlog#1291. # * 11 `_real_dual_node` site-replication tests — each spawns TWO full rustfs # servers and drives the cross-process site-replication control plane. # * 1 `_real_three_node` site-replication test. @@ -263,7 +265,7 @@ path = "junit.xml" # object_lambda) — too heavy for the merge budget; they run in ci-7's # nightly 4-node lane. # * replication_extension_test — repl-1 already splits it into the PR -# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (24 slow) lanes and reserves +# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (27 slow) lanes and reserves # it for those, so e2e-full does not double-run it. # * #[ignore]d tests — nextest skips them by default (no --run-ignored); the # manual-localhost:9000 reliant/policy tests are ci-13's migration. diff --git a/.github/workflows/e2e-replication-nightly.yml b/.github/workflows/e2e-replication-nightly.yml index 9936c88a1..1d94753e5 100644 --- a/.github/workflows/e2e-replication-nightly.yml +++ b/.github/workflows/e2e-replication-nightly.yml @@ -15,12 +15,14 @@ # Nightly full replication e2e lane (backlog#1147 repl-1, deps: ci-4). # # The per-PR gate (ci.yml `e2e-tests` job, `--profile e2e-smoke`) runs the 20 -# FAST replication tests. This scheduled lane runs the remaining 24 +# FAST replication tests. This scheduled lane runs the remaining 27 # heavier replication e2e tests that are unfit for a per-PR gate: # # * 2 remote-target TLS validation tests. -# * 9 bucket-replication data-plane tests (PUT/delete + poll for convergence; -# two replicate over HTTPS). +# * 12 bucket-replication data-plane/helper tests (PUT/delete + poll for +# convergence; two replicate over HTTPS, two pin active SSE failure +# contracts, and one guards event/history observers). The SSE-S3 contract +# remains ignored under backlog#1291. # * 11 `_real_dual_node` site-replication tests (each spawns TWO rustfs # servers and drives the cross-process site-replication control plane). # * 1 `_real_three_node` site-replication test. @@ -30,7 +32,7 @@ # .config/nextest.toml — the single wiring mechanism (repl-1 / ci-4). Do NOT # add ad-hoc cargo-test steps here; change the filterset instead. # -# Explicit division of labor: these 24 tests run ONLY here, never double-run +# Explicit division of labor: these 27 tests run ONLY here, never double-run # in ci-5's future e2e-full merge gate. TODO(ci-7): once the ci domain's # consolidated scheduled e2e workflow exists, fold this interim repl-owned lane # into it rather than growing a second scheduled entrypoint. diff --git a/crates/e2e_test/src/replication_extension_test.rs b/crates/e2e_test/src/replication_extension_test.rs index 93fb5afb7..b28f85318 100644 --- a/crates/e2e_test/src/replication_extension_test.rs +++ b/crates/e2e_test/src/replication_extension_test.rs @@ -16,12 +16,20 @@ use crate::common::{ RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client, replication_fast_env, rustfs_binary_path, }; +use crate::kms::common::{create_key_with_specific_id, sse_customer_key_md5_base64}; use crate::storage_api::replication_extension::BucketTargetSys; use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::error::ProvideErrorMetadata; +use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput; use aws_sdk_s3::primitives::ByteStream; -use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration}; +use aws_sdk_s3::types::{ + BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption, + VersioningConfiguration, +}; use aws_sdk_s3::{Client, Config}; +use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD}; +use bytes::Bytes; +use futures::{Stream, StreamExt}; use http::header::{CONTENT_TYPE, HOST}; use http_body_util::{BodyExt, Full}; use hyper::body::Incoming; @@ -58,7 +66,7 @@ use tokio::fs; use tokio::net::TcpListener; use tokio::sync::watch; use tokio::task::JoinSet; -use tokio::time::{Duration, sleep}; +use tokio::time::{Duration, sleep, timeout}; type TestResult = Result<(), Box>; @@ -95,6 +103,11 @@ const FAST_SCANNER_ENV: &[(&str, &str)] = &[ ("RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES", "1"), ]; +const REPL17_KMS_KEY_ID: &str = "repl17-local-key"; +const REPL17_SSEC_KEY: &str = "01234567890123456789012345678901"; +const REPLICATION_FAILED_EVENT: &str = "s3:Replication:OperationFailedReplication"; +const REPLICATION_EVENT_MAX_BUFFER_BYTES: usize = 1024 * 1024; + #[derive(Debug, Clone, serde::Deserialize)] struct ReplicationResetStatusResponse { #[serde(rename = "Targets", default)] @@ -1049,6 +1062,275 @@ async fn wait_for_source_replication_pending_or_failed( } } +async fn wait_for_source_replication_status(client: &Client, bucket: &str, key: &str, expected: &str, ssec: bool) -> TestResult { + let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY); + let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY); + let wait = async { + loop { + let request = client.head_object().bucket(bucket).key(key); + let head = if ssec { + request + .sse_customer_algorithm("AES256") + .sse_customer_key(&customer_key) + .sse_customer_key_md5(&customer_key_md5) + .send() + .await? + } else { + request.send().await? + }; + if head.replication_status().map(|status| status.as_str()) == Some(expected) { + return Ok(()); + } + sleep(Duration::from_millis(200)).await; + } + }; + + match timeout(Duration::from_secs(30), wait).await { + Ok(result) => result, + Err(_) => Err(format!("source object {key} did not reach replication status {expected} within 30 seconds").into()), + } +} + +async fn wait_for_replication_failure_event_stream(stream: S, expected_key: &str, max_wait: Duration) -> TestResult +where + S: Stream>, + E: Error + Send + Sync + 'static, +{ + let wait = async { + futures::pin_mut!(stream); + let mut pending = Vec::new(); + loop { + let Some(chunk) = stream.next().await else { + return Err("replication failure event stream ended before the expected event".into()); + }; + let chunk = chunk.map_err(|err| -> Box { Box::new(err) })?; + if chunk.is_empty() { + continue; + } + if pending.len().saturating_add(chunk.len()) > REPLICATION_EVENT_MAX_BUFFER_BYTES { + return Err("replication failure event buffer exceeded 1 MiB".into()); + } + pending.extend_from_slice(&chunk); + + while let Some(newline) = pending.iter().position(|byte| *byte == b'\n') { + let line = pending.drain(..=newline).collect::>(); + let payload = std::str::from_utf8(&line)?.trim(); + if payload.is_empty() { + continue; + } + let envelope: serde_json::Value = serde_json::from_str(payload)?; + let Some(records) = envelope["Records"].as_array() else { + continue; + }; + for record in records { + let Some(object_key) = record["s3"]["object"]["key"].as_str() else { + continue; + }; + let form_decoded_key = object_key.replace('+', " "); + let object_key = urlencoding::decode(&form_decoded_key) + .map(|decoded| decoded.into_owned()) + .unwrap_or_else(|_| object_key.to_string()); + if object_key == expected_key && record["eventName"].as_str() == Some(REPLICATION_FAILED_EVENT) { + return Ok(()); + } + } + } + } + }; + + match timeout(max_wait, wait).await { + Ok(result) => result, + Err(_) => Err(format!("replication failure event for {expected_key} was not received within {max_wait:?}").into()), + } +} + +async fn wait_for_replication_failure_event(response: reqwest::Response, expected_key: &str) -> TestResult { + wait_for_replication_failure_event_stream(response.bytes_stream(), expected_key, Duration::from_secs(30)).await +} + +fn target_history_contains_key(output: &ListObjectVersionsOutput, key: &str) -> bool { + output.versions().iter().any(|version| version.key() == Some(key)) + || output.delete_markers().iter().any(|marker| marker.key() == Some(key)) +} + +async fn assert_failed_replication_stays_absent_for( + source_client: &Client, + source_bucket: &str, + target_client: &Client, + target_bucket: &str, + key: &str, + ssec: bool, + duration: Duration, +) -> TestResult { + let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY); + let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY); + let wait = async { + let deadline = tokio::time::Instant::now() + duration; + loop { + let source_request = source_client.head_object().bucket(source_bucket).key(key); + let source = if ssec { + source_request + .sse_customer_algorithm("AES256") + .sse_customer_key(&customer_key) + .sse_customer_key_md5(&customer_key_md5) + .send() + .await? + } else { + source_request.send().await? + }; + if source.replication_status().map(|status| status.as_str()) != Some("FAILED") { + return Err(format!("source replication status for {source_bucket}/{key} did not remain FAILED").into()); + } + + let output = target_client + .list_object_versions() + .bucket(target_bucket) + .prefix(key) + .send() + .await?; + if target_history_contains_key(&output, key) { + return Err(format!("failed replication created target history for {target_bucket}/{key}").into()); + } + if tokio::time::Instant::now() >= deadline { + return Ok(()); + } + sleep(Duration::from_millis(250)).await; + } + }; + + match timeout(duration + Duration::from_secs(5), wait).await { + Ok(result) => result, + Err(_) => Err(format!("timed out while checking failed replication stability for {target_bucket}/{key}").into()), + } +} + +async fn subscribe_to_replication_failure( + env: &RustFSTestEnvironment, + bucket: &str, + key: &str, +) -> Result> { + let url = format!( + "{}/{bucket}?events={}&prefix={}&ping=1", + env.url, + urlencoding::encode(REPLICATION_FAILED_EVENT), + urlencoding::encode(key) + ); + let response = timeout( + Duration::from_secs(30), + signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None), + ) + .await + .map_err(|_| "replication failure event subscription did not respond within 30 seconds")??; + if response.status() != StatusCode::OK { + return Err(format!("failed to subscribe to replication failure events: {}", response.status()).into()); + } + Ok(response) +} + +async fn build_sse_replication_pair( + label: &str, + enable_kms: bool, +) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment, String, String), Box> { + let mut source_env = RustFSTestEnvironment::new().await?; + let mut target_env = RustFSTestEnvironment::new().await?; + let source_kms_key_dir = format!("{}/kms-keys", source_env.temp_dir); + let target_kms_key_dir = format!("{}/kms-keys", target_env.temp_dir); + if enable_kms { + fs::create_dir_all(&source_kms_key_dir).await?; + fs::create_dir_all(&target_kms_key_dir).await?; + create_key_with_specific_id(&source_kms_key_dir, REPL17_KMS_KEY_ID).await?; + create_key_with_specific_id(&target_kms_key_dir, REPL17_KMS_KEY_ID).await?; + } + + let mut source_process_env = replication_fast_env(); + source_process_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV); + source_process_env.extend_from_slice(FAST_SCANNER_ENV); + source_process_env.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]); + if enable_kms { + source_process_env.extend_from_slice(&[ + ("RUSTFS_KMS_ENABLE", "true"), + ("RUSTFS_KMS_BACKEND", "local"), + ("RUSTFS_KMS_KEY_DIR", source_kms_key_dir.as_str()), + ("RUSTFS_KMS_DEFAULT_KEY_ID", REPL17_KMS_KEY_ID), + ("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"), + ]); + } + source_env.start_rustfs_server_with_env(vec![], &source_process_env).await?; + + let mut target_process_env = vec![("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]; + if enable_kms { + target_process_env.extend_from_slice(&[ + ("RUSTFS_KMS_ENABLE", "true"), + ("RUSTFS_KMS_BACKEND", "local"), + ("RUSTFS_KMS_KEY_DIR", target_kms_key_dir.as_str()), + ("RUSTFS_KMS_DEFAULT_KEY_ID", REPL17_KMS_KEY_ID), + ("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"), + ]); + } + target_env + .start_rustfs_server_without_cleanup_with_env(&target_process_env) + .await?; + + let source_bucket = format!("repl17-{label}-src"); + let target_bucket = format!("repl17-{label}-dst"); + let source_client = source_env.create_s3_client(); + let target_client = target_env.create_s3_client(); + source_client.create_bucket().bucket(&source_bucket).send().await?; + target_client.create_bucket().bucket(&target_bucket).send().await?; + enable_bucket_versioning(&source_env, &source_bucket).await?; + enable_bucket_versioning(&target_env, &target_bucket).await?; + let target_arn = set_replication_target(&source_env, &source_bucket, &target_env, &target_bucket).await?; + put_bucket_replication(&source_env, &source_bucket, &target_arn).await?; + + Ok((source_env, target_env, source_bucket, target_bucket)) +} + +async fn assert_managed_sse_replication_fails_explicitly(label: &str, kms: bool) -> TestResult { + let (source_env, target_env, source_bucket, target_bucket) = build_sse_replication_pair(label, true).await?; + let source_client = source_env.create_s3_client(); + let target_client = target_env.create_s3_client(); + let key = format!("{label}-contract.txt"); + let body = format!("repl-17 {label} payload").into_bytes(); + let failure_events = subscribe_to_replication_failure(&source_env, &source_bucket, &key).await?; + + let encryption = if kms { + ServerSideEncryption::AwsKms + } else { + ServerSideEncryption::Aes256 + }; + let request = source_client + .put_object() + .bucket(&source_bucket) + .key(&key) + .body(ByteStream::from(body.clone())) + .server_side_encryption(encryption.clone()); + let request = if kms { + request.ssekms_key_id(REPL17_KMS_KEY_ID) + } else { + request + }; + request.send().await?; + + let source = source_client.get_object().bucket(&source_bucket).key(&key).send().await?; + assert_eq!(source.server_side_encryption(), Some(&encryption)); + assert_eq!(source.body.collect().await?.into_bytes().as_ref(), body.as_slice()); + + wait_for_replication_failure_event(failure_events, &key).await?; + wait_for_source_replication_status(&source_client, &source_bucket, &key, "FAILED", false).await?; + assert_failed_replication_stays_absent_for( + &source_client, + &source_bucket, + &target_client, + &target_bucket, + &key, + false, + Duration::from_secs(5), + ) + .await?; + + Ok(()) +} + /// Return the `LastModified` of the (single) delete marker for `key`, if present. async fn delete_marker_last_modified( client: &Client, @@ -3087,6 +3369,148 @@ async fn test_single_bucket_multipart_replication_fans_out_to_multiple_targets() Ok(()) } +#[tokio::test] +async fn test_repl17_failure_observation_helpers() -> TestResult { + let expected_key = "space + percent%.txt"; + let payload = format!( + "{{\"note\":\"é\",\"Records\":[{{\"eventName\":\"{REPLICATION_FAILED_EVENT}\",\"s3\":{{\"object\":{{\"key\":\"space+%2B+percent%25.txt\"}}}}}}]}}\n" + ); + let utf8_split = payload.find('é').expect("multibyte fixture must be present") + 1; + let encoded_plus_split = payload.find("%2B").expect("encoded plus must be present") + 1; + let chunks = vec![ + Ok::<_, Infallible>(Bytes::copy_from_slice(&payload.as_bytes()[..utf8_split])), + Ok::<_, Infallible>(Bytes::copy_from_slice(&payload.as_bytes()[utf8_split..encoded_plus_split])), + Ok::<_, Infallible>(Bytes::copy_from_slice(&payload.as_bytes()[encoded_plus_split..])), + ]; + wait_for_replication_failure_event_stream(futures::stream::iter(chunks), expected_key, Duration::from_millis(100)).await?; + + let ping_only = futures::stream::unfold((), |_| async { + sleep(Duration::from_millis(5)).await; + Some((Ok::<_, Infallible>(Bytes::from_static(b"{}\n")), ())) + }); + let timeout_error = wait_for_replication_failure_event_stream(ping_only, expected_key, Duration::from_millis(25)) + .await + .expect_err("ping-only event stream must hit the global deadline"); + assert!(timeout_error.to_string().contains("was not received within")); + + let mut oversized_line = vec![b'x'; REPLICATION_EVENT_MAX_BUFFER_BYTES]; + oversized_line.push(b'\n'); + let oversized = futures::stream::once(async { Ok::<_, Infallible>(Bytes::from(oversized_line)) }); + let oversized_error = wait_for_replication_failure_event_stream(oversized, expected_key, Duration::from_millis(100)) + .await + .expect_err("oversized event lines must be rejected"); + assert!(oversized_error.to_string().contains("buffer exceeded 1 MiB")); + + let version_output = ListObjectVersionsOutput::builder() + .versions(ObjectVersion::builder().key(expected_key).build()) + .build(); + assert!(target_history_contains_key(&version_output, expected_key)); + let marker_output = ListObjectVersionsOutput::builder() + .delete_markers(DeleteMarkerEntry::builder().key(expected_key).build()) + .build(); + assert!(target_history_contains_key(&marker_output, expected_key)); + assert!(!target_history_contains_key(&marker_output, "other-key")); + + Ok(()) +} + +/// backlog#1147 repl-17: SSE-C currently fails replication explicitly. Pin the +/// observed contract: the source remains decryptable with its customer key, +/// reports FAILED, emits the standard failure event, and leaves no target data. +#[tokio::test] +#[serial] +async fn test_bucket_replication_sse_c_contract() -> TestResult { + init_logging(); + + let (source_env, target_env, source_bucket, target_bucket) = build_sse_replication_pair("ssec", false).await?; + let source_client = source_env.create_s3_client(); + let target_client = target_env.create_s3_client(); + let key = "ssec-contract.txt"; + let body = b"repl-17 SSE-C payload"; + let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY); + let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY); + let failure_events = subscribe_to_replication_failure(&source_env, &source_bucket, key).await?; + + source_client + .put_object() + .bucket(&source_bucket) + .key(key) + .body(ByteStream::from_static(body)) + .sse_customer_algorithm("AES256") + .sse_customer_key(&customer_key) + .sse_customer_key_md5(&customer_key_md5) + .send() + .await?; + + let source = source_client + .get_object() + .bucket(&source_bucket) + .key(key) + .sse_customer_algorithm("AES256") + .sse_customer_key(&customer_key) + .sse_customer_key_md5(&customer_key_md5) + .send() + .await?; + assert_eq!(source.body.collect().await?.into_bytes().as_ref(), body); + + wait_for_replication_failure_event(failure_events, key).await?; + wait_for_source_replication_status(&source_client, &source_bucket, key, "FAILED", true).await?; + assert_failed_replication_stays_absent_for( + &source_client, + &source_bucket, + &target_client, + &target_bucket, + key, + true, + Duration::from_secs(5), + ) + .await?; + + target_client + .put_object() + .bucket(&target_bucket) + .key(key) + .body(ByteStream::from_static(b"observer negative-path fixture")) + .send() + .await?; + target_client.delete_object().bucket(&target_bucket).key(key).send().await?; + let history_error = assert_failed_replication_stays_absent_for( + &source_client, + &source_bucket, + &target_client, + &target_bucket, + key, + true, + Duration::ZERO, + ) + .await + .expect_err("target history must violate the failed replication contract"); + assert!(history_error.to_string().contains("created target history")); + + Ok(()) +} + +/// backlog#1147 repl-17 / backlog#1291: SSE-S3 must fail closed until managed +/// encryption is supported on the target. The current plaintext replication is +/// a known security bug, so this pins the required contract without blessing it. +#[tokio::test] +#[serial] +#[ignore = "backlog#1291: SSE-S3 replication silently drops encryption"] +async fn test_bucket_replication_sse_s3_contract() -> TestResult { + init_logging(); + assert_managed_sse_replication_fails_explicitly("sse-s3", false).await +} + +/// backlog#1147 repl-17: SSE-KMS currently fails closed rather than creating an +/// unreadable replica; the shared helper verifies FAILED, the failure event, +/// source readability, and a stable absence of all target versions. +#[tokio::test] +#[serial] +async fn test_bucket_replication_sse_kms_failure_contract() -> TestResult { + init_logging(); + assert_managed_sse_replication_fails_explicitly("sse-kms", true).await +} + /// backlog#1147 repl-5, scenario (a) — target outage + recovery (rustfs#3421 / #2071). /// /// Kills the replication target mid-workload, asserts the source records the diff --git a/docs/testing/e2e-suite-inventory.md b/docs/testing/e2e-suite-inventory.md index bbd106fdb..f6dd819a5 100644 --- a/docs/testing/e2e-suite-inventory.md +++ b/docs/testing/e2e-suite-inventory.md @@ -9,22 +9,26 @@ > (`.config/nextest.toml`); admission criteria: `crates/e2e_test/README.md`. > 🌙 marks tests in the scheduled `e2e-repl-nightly` profile (backlog#1147 > repl-1): `replication_extension_test` splits 20 fast tests into the PR smoke -> lane and 24 slow / `_real_dual_node` / `_real_three_node` / `_real_single_node` tests into the +> lane and 27 slow / `_real_dual_node` / `_real_three_node` / `_real_single_node` tests into the > nightly lane (`.github/workflows/e2e-replication-nightly.yml`). > Note: counts exclude `#[ignore]`d tests (nextest lists them separately). +> The SSE-S3 replication contract is ignored under backlog#1291 until its +> plaintext downgrade is fixed. | module | tests | PR smoke | |---|---|---| -| admin_auth_test | 3 | | +| admin_auth_test | 3 | ✅ | +| admin_iam_crud_test | 2 | ✅ | | admin_timeout_regression_test | 1 | | | anonymous_access_test | 3 | ✅ | | archive_download_integrity_test | 13 | | | bucket_logging_test | 3 | | | bucket_policy_check_test | 1 | ✅ | -| checksum_upload_test | 6 | | +| checksum_upload_test | 7 | | | cluster_concurrency_test | 2 | | | common | 3 | | | compression_test | 1 | | +| console_smoke_test | 1 | ✅ | | content_encoding_test | 3 | ✅ | | copy_object_metadata_test | 1 | ✅ | | copy_object_version_restore_test | 1 | | @@ -50,7 +54,8 @@ | mc_mirror_small_bucket_test | 1 | | | multipart_auth_test | 109 | | | namespace_lock_quorum_test | 2 | | -| negative_sigv4_test | 6 | | +| negative_sigv4_test | 6 | ✅ | +| notification_webhook_test | 1 | ✅ | | object_lambda_test | 16 | | | object_lock | 33 | | | overwrite_cleanup_regression_test | 1 | | @@ -58,14 +63,15 @@ | protocols | 16 | | | quota_test | 13 | | | reliability_disk_fault_test | 3 | | -| reliant | 9 | 3 ✅ | -| replication_extension_test | 44 | 20 ✅ +24 🌙 | +| reliant | 10 | 4 ✅ | +| replication_extension_test | 47 | 20 ✅ +27 🌙 | | security_boundary_test | 4 | | | server_startup_failfast_test | 1 | | | snowball_auto_extract_test | 6 | | | special_chars_test | 14 | ✅ | | stale_multipart_cleanup_cluster_test | 1 | | | tls_gen | 3 | | +| tls_hot_reload_test | 1 | ✅ | | version_id_regression_test | 10 | ✅ | -**Total listed: 425 tests across 52 modules · PR smoke subset: 93 tests / 20 modules** (18 full modules + 3 `reliant::lifecycle` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 24 tests** · generated 2026-07-15. +**Total listed: 435 tests across 56 modules · PR smoke subset: 108 tests / 26 modules** (24 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-15.