test(e2e): restore tier and inline full-suite checks (#6794)

This commit is contained in:
Zhengchao An
2026-08-28 21:08:30 +08:00
committed by GitHub
parent 7eddd1cf83
commit 847fbd2a8b
2 changed files with 21 additions and 12 deletions
@@ -15,9 +15,9 @@
//! Four-node EC regression gate for inline storage and the inline GET reader. //! Four-node EC regression gate for inline storage and the inline GET reader.
//! //!
//! The storage decision is based on shard bytes (256 KiB / 32 KiB objects for //! The storage decision is based on shard bytes (256 KiB / 32 KiB objects for
//! the default EC 2+2 geometry), while the GET fast path has its own object-size //! the default EC 2+2 geometry), and the GET fast path follows the persisted
//! limits (128 KiB / 16 KiB). A local OTLP/HTTP collector observes the existing //! inline marker. A local OTLP/HTTP collector observes the existing reader-path
//! reader-path counter without adding a scrape endpoint or production logging. //! counter without adding a scrape endpoint or production logging.
//! One S3 GET can select readers on multiple EC nodes, so the counter tracks //! One S3 GET can select readers on multiple EC nodes, so the counter tracks
//! distributed reader selection rather than HTTP request count. //! distributed reader selection rather than HTTP request count.
@@ -89,6 +89,7 @@ const MPU_PART_1_SIZE: usize = 5 * 1024 * 1024;
const MPU_PART_2_SIZE: usize = 16 * KIB; const MPU_PART_2_SIZE: usize = 16 * KIB;
const TIER_BUCKET: &str = "inline-fallback-cold-tier"; const TIER_BUCKET: &str = "inline-fallback-cold-tier";
const TIER_PREFIX: &str = "tiered"; const TIER_PREFIX: &str = "tiered";
const ALLOW_LOOPBACK_TIER_ENDPOINT_ENV: &str = "RUSTFS_TIER_RUSTFS_ALLOW_LOOPBACK_ENDPOINT";
const MSGPACK_FALLBACK_CONTROL_SERIES: [(&str, &str); 4] = [ const MSGPACK_FALLBACK_CONTROL_SERIES: [(&str, &str); 4] = [
(FALLBACK_REQUEST_DIRECTION, "ReadMultipleReq"), (FALLBACK_REQUEST_DIRECTION, "ReadMultipleReq"),
(FALLBACK_RESPONSE_DIRECTION, "ReadMultipleResp"), (FALLBACK_RESPONSE_DIRECTION, "ReadMultipleResp"),
@@ -791,12 +792,12 @@ fn metric_attribute(key: &str, value: &str) -> KeyValue {
} }
fn boundary_cases(state: VersionState) -> Vec<BoundaryCase> { fn boundary_cases(state: VersionState) -> Vec<BoundaryCase> {
let (fast_limit, storage_limit) = match state { let storage_limit = match state {
VersionState::Enabled => (16 * KIB, 32 * KIB), VersionState::Enabled => 32 * KIB,
VersionState::Unversioned => (128 * KIB, 256 * KIB), VersionState::Unversioned => 256 * KIB,
// A suspended bucket stores its null version using the unversioned // A suspended bucket stores its null version using the unversioned
// shard threshold, while ObjectInfo keeps version-aware GET semantics. // shard threshold, while ObjectInfo keeps version-aware GET semantics.
VersionState::Suspended => (16 * KIB, 256 * KIB), VersionState::Suspended => 256 * KIB,
}; };
let mut sizes = vec![0, 16 * KIB - 1, 16 * KIB, 16 * KIB + 1, 32 * KIB - 1, 32 * KIB, 32 * KIB + 1]; let mut sizes = vec![0, 16 * KIB - 1, 16 * KIB, 16 * KIB + 1, 32 * KIB - 1, 32 * KIB, 32 * KIB + 1];
if !matches!(state, VersionState::Enabled) { if !matches!(state, VersionState::Enabled) {
@@ -817,7 +818,7 @@ fn boundary_cases(state: VersionState) -> Vec<BoundaryCase> {
stored_inline: size <= storage_limit, stored_inline: size <= storage_limit,
expected_reader_path: if size == 0 { expected_reader_path: if size == 0 {
EMPTY EMPTY
} else if size <= fast_limit { } else if size <= storage_limit {
INLINE_DIRECT INLINE_DIRECT
} else { } else {
LEGACY_DUPLEX LEGACY_DUPLEX
@@ -2100,6 +2101,7 @@ async fn four_node_add_tier_converges() -> TestResult {
cold.create_s3_client().create_bucket().bucket(TIER_BUCKET).send().await?; cold.create_s3_client().create_bucket().bucket(TIER_BUCKET).send().await?;
let mut hot = RustFSTestClusterEnvironment::new(4).await?; let mut hot = RustFSTestClusterEnvironment::new(4).await?;
hot.set_env(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV, "true");
hot.start().await?; hot.start().await?;
let tier_name = unique_tier_name(); let tier_name = unique_tier_name();
@@ -2118,6 +2120,7 @@ async fn four_node_add_tier_converges_after_offline_node_restart_without_second_
cold.create_s3_client().create_bucket().bucket(TIER_BUCKET).send().await?; cold.create_s3_client().create_bucket().bucket(TIER_BUCKET).send().await?;
let mut hot = RustFSTestClusterEnvironment::new(4).await?; let mut hot = RustFSTestClusterEnvironment::new(4).await?;
hot.set_env(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV, "true");
hot.start().await?; hot.start().await?;
let tier_name = unique_tier_name(); let tier_name = unique_tier_name();
@@ -2214,6 +2217,7 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?; cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
let mut hot = RustFSTestClusterEnvironment::new(4).await?; let mut hot = RustFSTestClusterEnvironment::new(4).await?;
hot.set_env(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV, "true");
hot.set_env("RUSTFS_SCANNER_ENABLED", "false"); hot.set_env("RUSTFS_SCANNER_ENABLED", "false");
hot.set_env("RUSTFS_SCANNER_CYCLE", "3600"); hot.set_env("RUSTFS_SCANNER_CYCLE", "3600");
hot.set_env("RUSTFS_MAX_TRANSITION_WORKERS", "1"); hot.set_env("RUSTFS_MAX_TRANSITION_WORKERS", "1");
@@ -2356,6 +2360,7 @@ async fn four_node_manual_transition_rollout_non_empty_restart_readback() -> Tes
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?; cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
let mut hot = RustFSTestClusterEnvironment::new(4).await?; let mut hot = RustFSTestClusterEnvironment::new(4).await?;
hot.set_env(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV, "true");
hot.set_env("RUSTFS_SCANNER_ENABLED", "false"); hot.set_env("RUSTFS_SCANNER_ENABLED", "false");
hot.set_env("RUSTFS_SCANNER_CYCLE", "3600"); hot.set_env("RUSTFS_SCANNER_CYCLE", "3600");
hot.set_env("RUSTFS_MAX_TRANSITION_WORKERS", "2"); hot.set_env("RUSTFS_MAX_TRANSITION_WORKERS", "2");
@@ -2460,6 +2465,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
let collector = OtlpMetricCollector::start().await?; let collector = OtlpMetricCollector::start().await?;
let mut hot = RustFSTestClusterEnvironment::new(4).await?; let mut hot = RustFSTestClusterEnvironment::new(4).await?;
hot.set_env(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV, "true");
configure_mixed_msgpack_cluster(&mut hot, &collector)?; configure_mixed_msgpack_cluster(&mut hot, &collector)?;
hot.set_env("RUSTFS_SCANNER_CYCLE", "1"); hot.set_env("RUSTFS_SCANNER_CYCLE", "1");
hot.set_env("RUSTFS_ILM_PROCESS_TIME", "1"); hot.set_env("RUSTFS_ILM_PROCESS_TIME", "1");
@@ -2571,6 +2577,7 @@ async fn four_node_transitioned_inline_fallback() -> TestResult {
let collector = OtlpMetricCollector::start().await?; let collector = OtlpMetricCollector::start().await?;
let mut hot = RustFSTestClusterEnvironment::new(4).await?; let mut hot = RustFSTestClusterEnvironment::new(4).await?;
hot.set_env(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV, "true");
configure_reader_metric_cluster(&mut hot, &collector); configure_reader_metric_cluster(&mut hot, &collector);
hot.set_env("RUSTFS_SCANNER_CYCLE", "1"); hot.set_env("RUSTFS_SCANNER_CYCLE", "1");
hot.set_env("RUSTFS_ILM_PROCESS_TIME", "1"); hot.set_env("RUSTFS_ILM_PROCESS_TIME", "1");
@@ -66,6 +66,7 @@ const SURVIVOR_KEY: &str = "keep/object.bin";
const TIER_NAME: &str = "KMSCOLD"; const TIER_NAME: &str = "KMSCOLD";
const TIER_BUCKET: &str = "kms-ilm-cold-tier"; const TIER_BUCKET: &str = "kms-ilm-cold-tier";
const TIER_PREFIX: &str = "tiered"; const TIER_PREFIX: &str = "tiered";
const ALLOW_LOOPBACK_TIER_ENDPOINT_ENV: (&str, &str) = ("RUSTFS_TIER_RUSTFS_ALLOW_LOOPBACK_ENDPOINT", "true");
const TRANSITION_BUCKET: &str = "kms-ilm-transition"; const TRANSITION_BUCKET: &str = "kms-ilm-transition";
const TRANSITION_KEY: &str = "tier/object.bin"; const TRANSITION_KEY: &str = "tier/object.bin";
@@ -80,7 +81,7 @@ const ILM_DEADLINE: StdDuration = StdDuration::from_secs(90);
/// `--kms-default-key-id`, insecure dev defaults). The lifecycle env matches /// `--kms-default-key-id`, insecure dev defaults). The lifecycle env matches
/// `reliant/lifecycle.rs::fast_lifecycle_env` plus `RUSTFS_ILM_DEBUG_DAY_SECS=2`, /// `reliant/lifecycle.rs::fast_lifecycle_env` plus `RUSTFS_ILM_DEBUG_DAY_SECS=2`,
/// so a `Days=1` rule is due about two seconds after the write. /// so a `Days=1` rule is due about two seconds after the write.
async fn start_enforcing_ilm_server(env: &mut LocalKMSTestEnvironment) -> TestResult { async fn start_enforcing_ilm_server(env: &mut LocalKMSTestEnvironment, extra_env: &[(&str, &str)]) -> TestResult {
create_key_with_specific_id(&env.kms_keys_dir, SSE_KEY).await?; create_key_with_specific_id(&env.kms_keys_dir, SSE_KEY).await?;
let key_dir = env.kms_keys_dir.clone(); let key_dir = env.kms_keys_dir.clone();
@@ -94,13 +95,14 @@ async fn start_enforcing_ilm_server(env: &mut LocalKMSTestEnvironment) -> TestRe
SSE_KEY, SSE_KEY,
]; ];
let envs = [ let mut envs = vec![
("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"), ("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"),
("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "true"), ("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "true"),
("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_CYCLE", "1"),
("RUSTFS_ILM_PROCESS_TIME", "1"), ("RUSTFS_ILM_PROCESS_TIME", "1"),
("RUSTFS_ILM_DEBUG_DAY_SECS", "2"), ("RUSTFS_ILM_DEBUG_DAY_SECS", "2"),
]; ];
envs.extend_from_slice(extra_env);
env.base_env.start_rustfs_server_with_env(args, &envs).await?; env.base_env.start_rustfs_server_with_env(args, &envs).await?;
Ok(()) Ok(())
@@ -427,7 +429,7 @@ async fn ilm_expiration_on_sse_kms_bucket_under_enforcement() -> TestResult {
init_logging(); init_logging();
let mut env = LocalKMSTestEnvironment::new().await?; let mut env = LocalKMSTestEnvironment::new().await?;
start_enforcing_ilm_server(&mut env).await?; start_enforcing_ilm_server(&mut env, &[]).await?;
env.base_env.create_test_bucket(EXPIRY_BUCKET).await?; env.base_env.create_test_bucket(EXPIRY_BUCKET).await?;
let client = env.base_env.create_s3_client(); let client = env.base_env.create_s3_client();
@@ -499,7 +501,7 @@ async fn ilm_transition_on_sse_kms_bucket_under_enforcement_reads_back() -> Test
// Hot server: Local KMS + enforcement + accelerated lifecycle clock. // Hot server: Local KMS + enforcement + accelerated lifecycle clock.
let mut env = LocalKMSTestEnvironment::new().await?; let mut env = LocalKMSTestEnvironment::new().await?;
start_enforcing_ilm_server(&mut env).await?; start_enforcing_ilm_server(&mut env, &[ALLOW_LOOPBACK_TIER_ENDPOINT_ENV]).await?;
let hot_client = env.base_env.create_s3_client(); let hot_client = env.base_env.create_s3_client();
add_rustfs_tier(&env.base_env, &cold.base_env).await?; add_rustfs_tier(&env.base_env, &cold.base_env).await?;