From 1585308f0fa277f9f3b4306b01d6822c1864b56e Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 28 Aug 2026 08:41:36 +0800 Subject: [PATCH] fix(test): restore #[serial] markers the fallback runner still needs (#6767) --- crates/lifecycle/Cargo.toml | 2 +- crates/lifecycle/src/core.rs | 45 +++++++++++++++ crates/scanner/Cargo.toml | 3 +- crates/scanner/src/lib.rs | 5 ++ crates/scanner/src/runtime_config.rs | 14 +++++ crates/scanner/src/scanner/tests.rs | 56 +++++++++++++++++++ crates/scanner/src/scanner_folder/tests.rs | 34 +++++++++++ crates/scanner/src/scanner_io/tests.rs | 27 +++++++++ crates/scanner/src/sleeper.rs | 5 ++ .../tests/lifecycle_integration_test.rs | 47 ++++++++++++++-- 10 files changed, 232 insertions(+), 6 deletions(-) diff --git a/crates/lifecycle/Cargo.toml b/crates/lifecycle/Cargo.toml index df5fc1de4..6bb187f03 100644 --- a/crates/lifecycle/Cargo.toml +++ b/crates/lifecycle/Cargo.toml @@ -66,7 +66,7 @@ uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnos [dev-dependencies] metrics-util = { workspace = true, features = ["debugging"] } proptest = "1" -serial_test = { workspace = true } +serial_test.workspace = true temp-env.workspace = true tokio = { workspace = true, features = ["macros", "fs", "rt-multi-thread"] } diff --git a/crates/lifecycle/src/core.rs b/crates/lifecycle/src/core.rs index 2eadd6064..4a2e9a4b3 100644 --- a/crates/lifecycle/src/core.rs +++ b/crates/lifecycle/src/core.rs @@ -1568,6 +1568,7 @@ mod tests { } #[tokio::test] + #[serial] async fn abort_incomplete_multipart_upload_due_accepts_zero_days() { let initiated = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { @@ -1628,6 +1629,7 @@ mod tests { } #[tokio::test] + #[serial] async fn predict_expiration_selects_closest_expiry_for_put_object() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { @@ -1874,6 +1876,7 @@ mod tests { } #[tokio::test] + #[serial] async fn empty_transition_vectors_are_not_active_or_due() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -1939,6 +1942,7 @@ mod tests { } #[tokio::test] + #[serial] async fn eval_inner_keeps_latest_object_before_days_due() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { @@ -1972,6 +1976,7 @@ mod tests { } #[tokio::test] + #[serial] async fn eval_inner_transitions_latest_object_after_days_due() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { @@ -2009,6 +2014,7 @@ mod tests { } #[tokio::test] + #[serial] async fn eval_inner_transitions_latest_object_after_date_due() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let transition_date = base_time - Duration::days(1); @@ -2048,6 +2054,7 @@ mod tests { } #[tokio::test] + #[serial] async fn eval_inner_selects_earliest_due_among_multiple_past_due_events() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); // Two enabled rules both yield a past-due DeleteAction and a third yields a @@ -2161,6 +2168,7 @@ mod tests { } #[tokio::test] + #[serial] async fn eval_inner_expires_noncurrent_version_after_due() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { @@ -2198,6 +2206,7 @@ mod tests { } #[tokio::test] + #[serial] async fn eval_inner_skips_noncurrent_expiration_without_successor() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("valid fixed test timestamp"); let lc = BucketLifecycleConfiguration { @@ -2233,6 +2242,7 @@ mod tests { } #[tokio::test] + #[serial] async fn eval_inner_missing_successor_does_not_skip_noncurrent_transition() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("valid fixed test timestamp"); let lc = BucketLifecycleConfiguration { @@ -2275,6 +2285,7 @@ mod tests { } #[tokio::test] + #[serial] async fn eval_inner_noncurrent_expiration_one_day_respects_due_boundary() { let successor_time = datetime!(2025-06-15 12:00:00 UTC); let due = expected_expiry_time(successor_time, 1); @@ -2316,6 +2327,7 @@ mod tests { } #[tokio::test] + #[serial] async fn eval_inner_expires_noncurrent_version_immediately_when_zero_days() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { @@ -2353,6 +2365,7 @@ mod tests { } #[tokio::test] + #[serial] async fn eval_inner_transitions_noncurrent_version_after_due() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { @@ -2428,6 +2441,7 @@ mod tests { } #[tokio::test] + #[serial] async fn evaluator_honors_newer_noncurrent_versions_retention_count() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = Arc::new(BucketLifecycleConfiguration { @@ -2716,6 +2730,7 @@ mod tests { } #[tokio::test] + #[serial] async fn expired_object_delete_marker_ignores_marker_with_noncurrent_versions_present() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { @@ -2792,6 +2807,7 @@ mod tests { } #[tokio::test] + #[serial] async fn expired_object_delete_marker_deletes_only_delete_marker_immediately() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { @@ -2869,6 +2885,7 @@ mod tests { } #[tokio::test] + #[serial] async fn expiration_days_deletes_only_expired_delete_marker_when_due() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { @@ -2919,6 +2936,7 @@ mod tests { } #[tokio::test] + #[serial] async fn expiration_days_uses_earliest_due_rule_for_expired_delete_marker() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let make_rule = |id: &str, days| LifecycleRule { @@ -3249,6 +3267,7 @@ mod tests { } #[tokio::test] + #[serial] async fn del_marker_expiration_deletes_marker_and_older_versions_when_due() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("fixed timestamp should be valid"); let lc = BucketLifecycleConfiguration { @@ -3288,6 +3307,7 @@ mod tests { // --- TASK-003 tests: Round up to next UTC processing boundary --- #[test] + #[serial] fn expected_expiry_time_rounds_up_to_next_midnight_utc() { with_default_ilm_process_time(|| { // Object created at 2025-01-15T10:30:45Z, expire in 30 days @@ -3303,6 +3323,7 @@ mod tests { } #[test] + #[serial] fn expected_expiry_time_immediate_expiry_returns_epoch() { with_default_ilm_process_time(|| { let mod_time = datetime!(2025-06-01 12:00:00 UTC); @@ -3312,6 +3333,7 @@ mod tests { } #[test] + #[serial] fn expected_expiry_time_preserves_exact_midnight_boundary() { with_default_ilm_process_time(|| { let mod_time = datetime!(2025-03-01 00:00:00 UTC); @@ -3321,6 +3343,7 @@ mod tests { } #[test] + #[serial] fn expected_expiry_time_rounds_end_of_day_to_following_midnight() { with_default_ilm_process_time(|| { let mod_time = datetime!(2025-06-15 23:59:59 UTC); @@ -3330,6 +3353,7 @@ mod tests { } #[test] + #[serial] fn expected_expiry_time_uses_canonical_process_time_boundary() { let mod_time = datetime!(2025-01-15 10:30:45 UTC); @@ -3342,6 +3366,7 @@ mod tests { } #[test] + #[serial] fn expected_expiry_time_uses_deprecated_process_time_alias() { let mod_time = datetime!(2025-01-15 10:30:45 UTC); @@ -3354,6 +3379,7 @@ mod tests { } #[test] + #[serial] fn expected_expiry_time_uses_default_boundary_when_process_time_is_zero_or_invalid() { let mod_time = datetime!(2025-01-15 10:30:45 UTC); @@ -3376,6 +3402,7 @@ mod tests { // (a) Default path (env unset) is byte-identical: one day == 86400s. #[test] + #[serial] fn ilm_day_secs_defaults_to_86400_when_unset() { temp_env::with_var_unset(ENV_ILM_DEBUG_DAY_SECS, || { assert_eq!(ilm_day_secs(), DEFAULT_ILM_DAY_SECS); @@ -3404,6 +3431,7 @@ mod tests { // (b) End-to-end env read scales the day length. #[test] + #[serial] fn ilm_day_secs_scales_when_env_set() { temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("2"), || { assert_eq!(ilm_day_secs(), 2); @@ -3412,6 +3440,7 @@ mod tests { // (c) Invalid env value falls back to 86400. #[test] + #[serial] fn ilm_day_secs_falls_back_on_invalid_env() { temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("bogus"), || { assert_eq!(ilm_day_secs(), DEFAULT_ILM_DAY_SECS); @@ -3424,6 +3453,7 @@ mod tests { // Deadline math scales: with a 1s day and PROCESS_TIME unset, a Days=1 rule is // due 1s after mod_time (rounded up to the next 1s boundary => same instant). #[test] + #[serial] fn expected_expiry_time_scales_with_debug_day_secs() { let mod_time = datetime!(2025-01-15 10:30:45 UTC); temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("1"), || { @@ -3439,6 +3469,7 @@ mod tests { // days == 0 still yields the immediate-expiry sentinel regardless of the switch. #[test] + #[serial] fn expected_expiry_time_zero_days_ignores_debug_day_secs() { let mod_time = datetime!(2025-06-01 12:00:00 UTC); temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("2"), || { @@ -3449,6 +3480,7 @@ mod tests { // (③) Interaction with an explicit RUSTFS_ILM_PROCESS_TIME: the deadline offset // uses the accelerated day length, but the rounding boundary honors PROCESS_TIME. #[test] + #[serial] fn expected_expiry_time_debug_day_secs_respects_explicit_process_time() { let mod_time = datetime!(2025-01-15 10:30:00 UTC); // day == 10s, but round up to the next 60s (PROCESS_TIME) boundary. @@ -3465,6 +3497,7 @@ mod tests { // (③) With the switch unset, an explicit PROCESS_TIME behaves exactly as before. #[test] + #[serial] fn expected_expiry_time_unset_debug_day_secs_matches_legacy_process_time() { let mod_time = datetime!(2025-01-15 10:30:45 UTC); temp_env::with_var_unset(ENV_ILM_DEBUG_DAY_SECS, || { @@ -3492,6 +3525,7 @@ mod tests { // The abort-incomplete-multipart deadline path also scales through the switch. #[test] + #[serial] fn abort_incomplete_multipart_due_scales_with_debug_day_secs() { use s3s::dto::AbortIncompleteMultipartUpload; let initiated = datetime!(2025-01-15 10:30:45 UTC); @@ -3536,6 +3570,7 @@ mod tests { // (⑤ evaluator seam) A Days=1 rule fires under RUSTFS_ILM_DEBUG_DAY_SECS=1 once // `now` advances a few seconds past a mod_time only ~seconds in the past. #[test] + #[serial] fn eval_inner_expires_days_one_rule_under_debug_day_secs() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -3584,6 +3619,7 @@ mod tests { // Absolute Date-based rules must NOT scale with the switch (regression guard). #[test] + #[serial] fn eval_inner_date_rule_ignores_debug_day_secs() { let expiry_date = datetime!(2025-06-01 00:00:00 UTC); let lc = BucketLifecycleConfiguration { @@ -3866,6 +3902,7 @@ mod tests { } #[tokio::test] + #[serial] async fn eval_inner_triggers_delete_all_versions_when_expired_object_all_versions_set() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { @@ -3904,6 +3941,7 @@ mod tests { } #[tokio::test] + #[serial] async fn expired_object_all_versions_does_not_apply_to_current_delete_marker() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("fixed timestamp should be valid"); let lc = BucketLifecycleConfiguration { @@ -3933,6 +3971,7 @@ mod tests { } #[tokio::test] + #[serial] async fn eval_inner_uses_delete_action_when_all_versions_not_set() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { @@ -4051,6 +4090,7 @@ mod tests { use super::*; use proptest::prelude::*; use s3s::dto::{NoncurrentVersionExpiration, Tag}; + use serial_test::serial; const DAY_SECS: i64 = 86400; @@ -4281,6 +4321,7 @@ mod tests { /// combination, and must be deterministic: the same input /// evaluated twice yields an identical event. #[test] + #[serial] fn eval_inner_never_panics_and_is_deterministic( rules in prop::collection::vec(arb_rule(), 0..4), obj in arb_object_opts(), @@ -4420,6 +4461,7 @@ mod tests { /// candidate set — earliest due wins, ties prefer delete-class — /// and must be `NoneAction` exactly when that set is empty. #[test] + #[serial] fn eval_inner_winner_matches_selection_oracle( rules in prop::collection::vec(arb_selection_rule(), 0..5), mod_off in 0i64..(2 * DAY_SECS), @@ -4473,6 +4515,7 @@ mod tests { /// non-decreasing in `days` (days == 0 maps to UNIX_EPOCH, below /// any post-1970 deadline). #[test] + #[serial] fn expected_expiry_time_is_monotonic_in_days( mod_off in 0i64..(3650 * DAY_SECS), d1 in 0i32..2000, @@ -4494,6 +4537,7 @@ mod tests { /// to the next whole-day boundary: the result is day-aligned, not /// before `mod_time + days`, and less than one boundary beyond it. #[test] + #[serial] fn expected_expiry_time_lands_on_default_day_boundary( mod_off in 0i64..(3650 * DAY_SECS), days in 1i32..2000, @@ -4511,6 +4555,7 @@ mod tests { /// to that boundary instead: aligned to it, never early, and less /// than one boundary late. #[test] + #[serial] fn expected_expiry_time_lands_on_explicit_process_boundary( mod_off in 0i64..(365 * DAY_SECS), days in 1i32..400, diff --git a/crates/scanner/Cargo.toml b/crates/scanner/Cargo.toml index 91f35715f..904fb0913 100644 --- a/crates/scanner/Cargo.toml +++ b/crates/scanner/Cargo.toml @@ -105,7 +105,8 @@ hex-simd.workspace = true [dev-dependencies] tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] } -temp-env = { workspace = true, features = ["async_closure"] } +serial_test = { workspace = true } +temp-env = { workspace = true } tempfile = { workspace = true } uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] } tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] } diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index 2c78928b5..d089454f4 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -951,8 +951,10 @@ impl ScannerConfigObjectDelete for SetDisks { #[cfg(test)] mod tests { use super::*; + use serial_test::serial; #[tokio::test] + #[serial] async fn runtime_tier_names_serves_cached_arc_within_ttl() { reset_tier_name_cache_for_test(); // The tier config manager is unconfigured in unit tests, so the @@ -1028,6 +1030,7 @@ mod tests { } #[test] + #[serial] fn foreground_read_guard_tracks_stream_lifetime() { reset_foreground_read_activity_for_test(); assert_eq!(current_foreground_read_activity(), 0); @@ -1041,6 +1044,7 @@ mod tests { } #[test] + #[serial] fn foreground_read_activity_keeps_larger_signal() { reset_foreground_read_activity_for_test(); let _guard = ForegroundReadGuard::new(); @@ -1053,6 +1057,7 @@ mod tests { } #[test] + #[serial] fn scanner_runtime_guard_tracks_runtime_lifetime() { reset_scanner_runtime_instances_for_test(); assert!(!scanner_runtime_initialized()); diff --git a/crates/scanner/src/runtime_config.rs b/crates/scanner/src/runtime_config.rs index 2fef6f9f4..cdace6a8f 100644 --- a/crates/scanner/src/runtime_config.rs +++ b/crates/scanner/src/runtime_config.rs @@ -896,6 +896,7 @@ mod tests { SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed, }; + use serial_test::serial; use std::collections::HashMap; use std::time::Duration; use temp_env::{with_var, with_var_unset}; @@ -943,6 +944,7 @@ mod tests { } #[test] + #[serial] fn scanner_runtime_config_uses_persisted_values_when_env_is_unset() { let config = server_config_with_scanner(&[ (SCANNER_SPEED, "slow"), @@ -1014,6 +1016,7 @@ mod tests { } #[test] + #[serial] fn scanner_runtime_config_normalizes_persisted_default_speed() { let config = server_config_with_scanner(&[(SCANNER_SPEED, "default")]); @@ -1029,6 +1032,7 @@ mod tests { } #[test] + #[serial] fn scanner_runtime_config_prefers_env_over_persisted_config() { let config = server_config_with_scanner(&[(SCANNER_SPEED, "slowest"), (SCANNER_CYCLE, "600")]); @@ -1045,6 +1049,7 @@ mod tests { } #[test] + #[serial] fn scanner_runtime_config_prefers_heal_bitrot_cycle_over_scanner_compat_config() { let config = server_config_with_scanner_and_heal(&[(SCANNER_BITROT_CYCLE, "3600")], &[(HEAL_BITROT_CYCLE, "off")]); @@ -1057,6 +1062,7 @@ mod tests { } #[test] + #[serial] fn scanner_runtime_config_marks_scanner_bitrot_cycle_as_compat_source() { let config = server_config_with_scanner(&[(SCANNER_BITROT_CYCLE, "3600")]); @@ -1073,6 +1079,7 @@ mod tests { } #[test] + #[serial] fn scanner_runtime_config_normalizes_persisted_default_bitrot_cycles() { let default_cycle = DEFAULT_HEAL_BITROT_CYCLE_SECS.to_string(); for config in [ @@ -1097,6 +1104,7 @@ mod tests { } #[test] + #[serial] fn scanner_runtime_config_validation_rejects_invalid_persisted_speed_with_env_override() { let config = server_config_with_scanner(&[(SCANNER_SPEED, "warp")]); @@ -1130,6 +1138,7 @@ mod tests { } #[test] + #[serial] fn scanner_runtime_config_uses_derived_delay_for_excessive_env_override() { let config = server_config_with_scanner(&[(SCANNER_SPEED, "slow")]); @@ -1150,6 +1159,7 @@ mod tests { } #[test] + #[serial] fn scanner_runtime_config_status_reports_value_sources() { let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_OBJECTS, "100"), (SCANNER_CACHE_SAVE_TIMEOUT, "5")]); @@ -1170,6 +1180,7 @@ mod tests { } #[test] + #[serial] fn applied_runtime_config_is_the_authoritative_scheduler_state() { let config = server_config_with_scanner(&[(SCANNER_CYCLE, "321")]); @@ -1186,6 +1197,7 @@ mod tests { } #[test] + #[serial] fn scanner_runtime_config_status_reports_persisted_pacing_overrides() { let config = server_config_with_scanner(&[("delay", "3.5"), ("max_wait", "7")]); @@ -1207,6 +1219,7 @@ mod tests { } #[test] + #[serial] fn scanner_runtime_config_status_prefers_env_pacing_overrides() { let config = server_config_with_scanner(&[("delay", "3.5"), ("max_wait", "7")]); @@ -1228,6 +1241,7 @@ mod tests { } #[test] + #[serial] fn scanner_runtime_config_status_preserves_subsecond_max_wait() { let config = server_config_with_scanner(&[(SCANNER_SPEED, "fast")]); diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index e0da552b1..96629b4d3 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -21,6 +21,7 @@ use crate::{ ScannerPutObjReader as PutObjReader, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx, }; +use serial_test::serial; use std::collections::{HashMap, HashSet}; use std::io::Cursor; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; @@ -587,6 +588,7 @@ fn test_initial_scanner_delay_uses_configured_start_delay() { } #[test] +#[serial] fn test_initial_scanner_delay_uses_cycle_without_explicit_start_delay() { with_var(ENV_SCANNER_CYCLE, Some("120"), || { crate::runtime_config::refresh_scanner_runtime_config_for_tests(); @@ -633,6 +635,7 @@ fn test_initial_scanner_delay_keeps_delay_for_replication_without_buckets() { } #[test] +#[serial] fn test_scanner_cycle_max_duration_uses_env() { with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("42"), || { assert_eq!(scanner_cycle_max_duration(), Some(Duration::from_secs(42))); @@ -676,6 +679,7 @@ async fn test_scanner_cycle_budget_drop_cancels_child_without_elapsed() { } #[test] +#[serial] fn test_scanner_cycle_budget_config_uses_work_budget_env() { with_var(ENV_SCANNER_CYCLE_MAX_OBJECTS, Some("100"), || { with_var(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, Some("25"), || { @@ -687,6 +691,7 @@ fn test_scanner_cycle_budget_config_uses_work_budget_env() { } #[test] +#[serial] fn test_scanner_cycle_budget_config_disables_zero_work_budgets() { with_var(ENV_SCANNER_CYCLE_MAX_OBJECTS, Some("0"), || { with_var(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, Some("0"), || { @@ -729,6 +734,7 @@ fn test_scan_cycle_partial_source_maps_budget_reason() { } #[tokio::test] +#[serial] async fn test_mark_scan_cycle_idle_clears_published_cycle_state() { let mut cycle_info = CurrentCycle { current: 12, @@ -757,6 +763,7 @@ async fn test_mark_scan_cycle_idle_clears_published_cycle_state() { } #[tokio::test] +#[serial] async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() { let cycle_started = Utc::now() - chrono::Duration::seconds(5); let mut cycle_info = CurrentCycle { @@ -783,6 +790,7 @@ async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() { } #[tokio::test] +#[serial] async fn scanner_cycle_metrics_guard_keeps_active_cycle_published_during_finalization() { let mut cycle_info = CurrentCycle { current: 12, @@ -807,6 +815,7 @@ async fn scanner_cycle_metrics_guard_keeps_active_cycle_published_during_finaliz } #[tokio::test] +#[serial] async fn scanner_cycle_metrics_guard_drop_clears_activity() { let guard = ScannerCycleMetricsGuard::new(CurrentCycle { current: 12, @@ -824,6 +833,7 @@ async fn scanner_cycle_metrics_guard_drop_clears_activity() { } #[tokio::test] +#[serial] async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() { let (_temp_dir, store) = setup_scanner_cycle_store().await; let ctx = CancellationToken::new(); @@ -874,6 +884,7 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() { } #[tokio::test] +#[serial] async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() { let store = Arc::new(MemoryConfigStore::default()); let ctx = CancellationToken::new(); @@ -909,6 +920,7 @@ async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() { } #[tokio::test] +#[serial] async fn scanner_cycle_recovers_to_newer_durable_cache_floor() { let store = Arc::new(MemoryConfigStore::default()); let ctx = CancellationToken::new(); @@ -948,6 +960,7 @@ async fn scanner_cycle_recovers_to_newer_durable_cache_floor() { } #[tokio::test] +#[serial] async fn scanner_cycle_rejects_invalid_cache_floor() { let store = Arc::new(MemoryConfigStore::default()); let ctx = CancellationToken::new(); @@ -2369,6 +2382,7 @@ async fn scanner_usage_bootstrap_allows_first_bucket_to_win_startup() { } #[tokio::test] +#[serial] async fn scanner_usage_backup_uses_durable_cycle_cadence_across_tasks() { let store = Arc::new(MemoryConfigStore::default()); let ctx = CancellationToken::new(); @@ -2489,6 +2503,7 @@ fn scanner_cycle_advance_fails_before_reserved_exhausted_value() { } #[tokio::test] +#[serial] async fn test_finalize_partial_scan_cycle_reports_persist_failure() { let store = Arc::new(MemoryConfigStore::default()); let ctx = CancellationToken::new(); @@ -2512,6 +2527,7 @@ async fn test_finalize_partial_scan_cycle_reports_persist_failure() { } #[tokio::test] +#[serial] async fn test_persist_scanner_cycle_state_reconciles_newer_winner() { let store = Arc::new(MemoryConfigStore::default()); let ctx = CancellationToken::new(); @@ -3167,6 +3183,7 @@ async fn test_observational_usage_defers_when_authoritative_baseline_is_missing( } #[tokio::test] +#[serial] async fn test_usage_route_barrier_precedes_durable_reconciliation() { let store = Arc::new(MemoryConfigStore::default()); let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); @@ -3235,6 +3252,7 @@ async fn coordinator_does_not_put_after_remote_generation_flip() { } #[tokio::test] +#[serial] async fn test_deferred_usage_save_keeps_last_real_save_metric() { let metrics = global_metrics(); metrics.record_scanner_usage_save_result(ScannerUsageSaveResult::Success); @@ -4307,6 +4325,7 @@ fn scanner_cycle_cache_floor_stays_pending_during_deferred_usage_publication() { } #[test] +#[serial] fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() { crate::scanner_io::clear_dirty_usage_bucket("photos"); crate::scanner_io::record_dirty_usage_bucket("photos"); @@ -4333,6 +4352,7 @@ fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() { } #[test] +#[serial] fn finalizing_a_deferred_usage_save_keeps_dirty_work_pending() { crate::scanner_io::clear_dirty_usage_bucket("photos"); crate::scanner_io::record_dirty_usage_bucket("photos"); @@ -4375,6 +4395,7 @@ async fn scanner_cycle_keeps_remote_pending_acknowledgement() { } #[test] +#[serial] fn finalizing_an_already_durable_cycle_acknowledges_its_exact_dirty_snapshot() { crate::scanner_io::clear_dirty_usage_bucket("photos"); crate::scanner_io::record_dirty_usage_bucket("photos"); @@ -4389,6 +4410,7 @@ fn finalizing_an_already_durable_cycle_acknowledges_its_exact_dirty_snapshot() { } #[test] +#[serial] fn finalizing_a_prior_same_cycle_snapshot_keeps_new_dirty_work_pending() { crate::scanner_io::clear_dirty_usage_bucket("photos"); crate::scanner_io::record_dirty_usage_bucket("photos"); @@ -4404,6 +4426,7 @@ fn finalizing_a_prior_same_cycle_snapshot_keeps_new_dirty_work_pending() { } #[test] +#[serial] fn finalizing_a_durable_superseded_snapshot_keeps_dirty_work_pending() { crate::scanner_io::clear_dirty_usage_bucket("photos"); crate::scanner_io::record_dirty_usage_bucket("photos"); @@ -4419,6 +4442,7 @@ fn finalizing_a_durable_superseded_snapshot_keeps_dirty_work_pending() { } #[test] +#[serial] fn data_usage_persist_wait_covers_cache_retries_and_backup() { with_var(rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, Some("7"), || { crate::runtime_config::refresh_scanner_runtime_config_for_tests(); @@ -4471,6 +4495,7 @@ async fn maintenance_feature_inspection_preserves_base_cycle_after_timeout() { } #[tokio::test(start_paused = true)] +#[serial] async fn stable_maintenance_detection_preserves_base_cycle_after_timeout() { let ctx = CancellationToken::new(); @@ -4536,6 +4561,7 @@ async fn maintenance_feature_inspection_stops_on_cancellation() { } #[test] +#[serial] fn test_cycle_interval_prefers_explicit_cycle_override() { with_var(ENV_SCANNER_SPEED, Some("slowest"), || { with_var(ENV_SCANNER_CYCLE, Some("42"), || { @@ -4545,6 +4571,7 @@ fn test_cycle_interval_prefers_explicit_cycle_override() { } #[test] +#[serial] fn test_cycle_interval_prefers_explicit_cycle_over_default_cycle() { let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS); @@ -4554,6 +4581,7 @@ fn test_cycle_interval_prefers_explicit_cycle_over_default_cycle() { } #[test] +#[serial] fn test_cycle_interval_uses_scanner_default_speed_override_when_unconfigured() { let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest); @@ -4563,6 +4591,7 @@ fn test_cycle_interval_uses_scanner_default_speed_override_when_unconfigured() { } #[test] +#[serial] fn test_cycle_interval_prefers_explicit_speed_over_default_speed_override() { let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest); @@ -4580,6 +4609,7 @@ fn test_cycle_interval_prefers_explicit_speed_over_default_speed_override() { } #[test] +#[serial] fn test_cycle_interval_uses_default_cycle_override_when_unconfigured() { let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS); @@ -4761,6 +4791,7 @@ fn scanner_cycle_wait_plan_drives_growth_resets_and_bitrot_cap() { } #[test] +#[serial] fn scanner_cycle_schedule_status_reports_effective_backoff() { record_scanner_cycle_schedule(Duration::from_millis(86_400_001), true, 2_048, true, 7); @@ -5077,6 +5108,7 @@ fn dirty_usage_wakes_are_disabled_for_explicit_cycle_policy() { } #[test] +#[serial] fn clean_idle_cap_preserves_default_bitrot_coverage_window() { let config = ScannerRuntimeConfig { bitrot_cycle: Some(Duration::from_secs(30 * 24 * 60 * 60)), @@ -5106,6 +5138,7 @@ fn clean_idle_cap_allows_policy_max_when_bitrot_is_disabled() { } #[test] +#[serial] fn clean_idle_cap_never_shortens_the_base_cycle() { let config = ScannerRuntimeConfig { bitrot_cycle: Some(Duration::from_secs(60)), @@ -5119,6 +5152,7 @@ fn clean_idle_cap_never_shortens_the_base_cycle() { } #[test] +#[serial] fn test_cycle_interval_keeps_default_cycle_with_explicit_speed() { let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS); @@ -5136,6 +5170,7 @@ fn test_cycle_interval_keeps_default_cycle_with_explicit_speed() { } #[test] +#[serial] fn test_cycle_interval_prefers_explicit_start_delay_over_default_cycle() { let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS); @@ -5149,6 +5184,7 @@ fn test_cycle_interval_prefers_explicit_start_delay_over_default_cycle() { } #[test] +#[serial] fn test_cycle_interval_supports_minio_speed_alias() { with_var_unset(ENV_SCANNER_SPEED, || { with_var_unset(ENV_SCANNER_CYCLE, || { @@ -5162,6 +5198,7 @@ fn test_cycle_interval_supports_minio_speed_alias() { } #[test] +#[serial] fn test_cycle_interval_supports_minio_cycle_alias() { with_var_unset(ENV_SCANNER_CYCLE, || { with_var_unset(ENV_SCANNER_START_DELAY_SECS, || { @@ -5181,6 +5218,7 @@ fn test_randomized_cycle_delay_handles_small_start_delay() { } #[tokio::test] +#[serial] async fn test_wait_for_next_scanner_cycle_wakes_for_dirty_usage() { crate::scanner_io::clear_dirty_usage_buckets_for_tests(); @@ -5206,6 +5244,7 @@ async fn test_wait_for_next_scanner_cycle_wakes_for_dirty_usage() { } #[tokio::test] +#[serial] async fn test_wait_for_next_scanner_cycle_sees_unattempted_dirty_usage() { crate::scanner_io::clear_dirty_usage_buckets_for_tests(); let dirty_generation = crate::scanner_io::dirty_usage_generation(); @@ -5227,6 +5266,7 @@ async fn test_wait_for_next_scanner_cycle_sees_unattempted_dirty_usage() { } #[tokio::test(start_paused = true)] +#[serial] async fn test_wait_for_next_scanner_cycle_retries_stable_dirty_usage_on_timer() { crate::scanner_io::clear_dirty_usage_buckets_for_tests(); crate::scanner_io::record_dirty_usage_bucket("photos"); @@ -5248,6 +5288,7 @@ async fn test_wait_for_next_scanner_cycle_retries_stable_dirty_usage_on_timer() } #[tokio::test(start_paused = true)] +#[serial] async fn test_wait_for_next_scanner_cycle_can_defer_dirty_wakes_until_timer() { crate::scanner_io::clear_dirty_usage_buckets_for_tests(); let ctx = CancellationToken::new(); @@ -5266,6 +5307,7 @@ async fn test_wait_for_next_scanner_cycle_can_defer_dirty_wakes_until_timer() { } #[tokio::test] +#[serial] async fn test_wait_for_next_scanner_cycle_wakes_for_repeated_dirty_bucket() { crate::scanner_io::clear_dirty_usage_buckets_for_tests(); crate::scanner_io::record_dirty_usage_bucket("photos"); @@ -5291,6 +5333,7 @@ async fn test_wait_for_next_scanner_cycle_wakes_for_repeated_dirty_bucket() { } #[tokio::test] +#[serial] async fn test_wait_for_next_scanner_cycle_reschedules_for_runtime_config() { crate::scanner_io::clear_dirty_usage_buckets_for_tests(); let observed_generation = crate::runtime_config::scanner_runtime_config_generation(); @@ -5318,6 +5361,7 @@ async fn test_wait_for_next_scanner_cycle_reschedules_for_runtime_config() { } #[tokio::test] +#[serial] async fn test_wait_for_next_scanner_cycle_reschedules_for_maintenance_change() { crate::scanner_io::clear_dirty_usage_buckets_for_tests(); let observed_generation = crate::scanner_io::scanner_maintenance_generation(); @@ -5629,6 +5673,7 @@ fn scanner_activity_after_a_cycle_restores_the_base_interval() { } #[tokio::test(start_paused = true)] +#[serial] async fn distributed_clean_idle_wait_wakes_at_base_interval_for_remote_activity() { crate::scanner_io::clear_dirty_usage_buckets_for_tests(); let ctx = CancellationToken::new(); @@ -5656,6 +5701,7 @@ async fn distributed_clean_idle_wait_wakes_at_base_interval_for_remote_activity( } #[tokio::test(start_paused = true)] +#[serial] async fn superseded_retry_wait_defers_dirty_cluster_activity_until_timer() { crate::scanner_io::clear_dirty_usage_buckets_for_tests(); let ctx = CancellationToken::new(); @@ -5751,6 +5797,7 @@ async fn superseded_retry_wait_wakes_when_remote_restart_clears_movement_state() } #[tokio::test(start_paused = true)] +#[serial] async fn distributed_clean_idle_wait_blocks_backoff_for_unpropagated_maintenance() { crate::scanner_io::clear_dirty_usage_buckets_for_tests(); let ctx = CancellationToken::new(); @@ -5777,6 +5824,7 @@ async fn distributed_clean_idle_wait_blocks_backoff_for_unpropagated_maintenance } #[tokio::test(start_paused = true)] +#[serial] async fn distributed_clean_idle_wait_fails_closed_when_a_peer_is_unverifiable() { crate::scanner_io::clear_dirty_usage_buckets_for_tests(); let ctx = CancellationToken::new(); @@ -5803,6 +5851,7 @@ async fn distributed_clean_idle_wait_fails_closed_when_a_peer_is_unverifiable() } #[tokio::test(start_paused = true)] +#[serial] async fn distributed_clean_idle_wait_keeps_the_extended_deadline_when_peers_are_clean() { crate::scanner_io::clear_dirty_usage_buckets_for_tests(); let ctx = CancellationToken::new(); @@ -5830,6 +5879,7 @@ async fn distributed_clean_idle_wait_keeps_the_extended_deadline_when_peers_are_ } #[tokio::test(start_paused = true)] +#[serial] async fn scanner_activity_probe_wait_is_cancellation_aware() { crate::scanner_io::clear_dirty_usage_buckets_for_tests(); let ctx = CancellationToken::new(); @@ -5860,6 +5910,7 @@ async fn scanner_activity_probe_wait_is_cancellation_aware() { } #[tokio::test(start_paused = true)] +#[serial] async fn scanner_activity_probe_wait_stops_after_leader_lock_loss() { crate::scanner_io::clear_dirty_usage_buckets_for_tests(); let ctx = CancellationToken::new(); @@ -5891,6 +5942,7 @@ async fn scanner_activity_probe_wait_stops_after_leader_lock_loss() { } #[test] +#[serial] fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() { with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || { let mode = get_cycle_scan_mode(10, 0, Some(Utc::now()), bitrot_scan_cycle()); @@ -5899,6 +5951,7 @@ fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() { } #[test] +#[serial] fn test_get_cycle_scan_mode_respects_elapsed_bitrot_cycle() { with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || { let recent = Utc::now() - chrono::Duration::minutes(30); @@ -5910,6 +5963,7 @@ fn test_get_cycle_scan_mode_respects_elapsed_bitrot_cycle() { } #[test] +#[serial] fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() { with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("off"), || { assert_eq!(get_cycle_scan_mode(1, 0, None, bitrot_scan_cycle()), HealScanMode::Normal); @@ -5917,6 +5971,7 @@ fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() { } #[test] +#[serial] fn test_background_heal_info_for_scan_start_marks_deep_active() { let now = Utc::now(); let info = @@ -5942,6 +5997,7 @@ fn background_heal_read_failures_never_become_initializable_defaults() { } #[test] +#[serial] fn test_background_heal_info_for_scan_start_keeps_deep_window_start() { with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || { let started_at = Utc::now(); diff --git a/crates/scanner/src/scanner_folder/tests.rs b/crates/scanner/src/scanner_folder/tests.rs index 3708db8af..ade37c88c 100644 --- a/crates/scanner/src/scanner_folder/tests.rs +++ b/crates/scanner/src/scanner_folder/tests.rs @@ -18,6 +18,7 @@ use super::*; use crate::storage_api::VersionPurgeStatusType; use crate::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, TierStats, new_disk, storageclass}; use rustfs_filemeta::{FileInfo, FileMeta, MetadataResolutionParams}; +use serial_test::serial; use std::io::Write; #[cfg(unix)] use std::os::unix::fs::{PermissionsExt, symlink}; @@ -375,6 +376,7 @@ impl Drop for TestGuard { } #[tokio::test] +#[serial] async fn test_should_skip_failed_respects_ttl() { let (mut scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir); @@ -396,6 +398,7 @@ async fn test_should_skip_failed_respects_ttl() { } #[tokio::test] +#[serial] async fn test_record_failed_ttl_zero_noop() { let (mut scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard::new(0, 100, &mut scanner, temp_dir); @@ -549,6 +552,7 @@ fn test_should_account_replication_stats_only_for_live_object_versions() { } #[tokio::test] +#[serial] async fn test_heal_replication_only_queues_pending_null_deletes() { async fn replication_skipped_count() -> u64 { global_metrics() @@ -797,6 +801,7 @@ async fn test_scanner_heal_admission_accounting_maps_deep_scan_to_bitrot() { } #[test] +#[serial] fn test_excessive_version_alert_thresholds_use_env() { with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_VERSIONS, Some("3"), || { with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_VERSION_SIZE, Some("100"), || { @@ -811,6 +816,7 @@ fn test_excessive_version_alert_thresholds_use_env() { } #[test] +#[serial] fn test_excessive_folders_threshold_uses_env() { with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_FOLDERS, Some("3"), || { crate::runtime_config::refresh_scanner_runtime_config_for_tests(); @@ -820,6 +826,7 @@ fn test_excessive_folders_threshold_uses_env() { } #[test] +#[serial] fn test_excessive_folders_threshold_default_supports_pbs_layout() { with_var_unset(rustfs_config::ENV_SCANNER_ALERT_EXCESS_FOLDERS, || { crate::runtime_config::refresh_scanner_runtime_config_for_tests(); @@ -829,6 +836,7 @@ fn test_excessive_folders_threshold_default_supports_pbs_layout() { } #[test] +#[serial] fn test_scanner_yield_every_n_objects_uses_env() { with_var(rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS, Some("32"), || { crate::runtime_config::refresh_scanner_runtime_config_for_tests(); @@ -838,6 +846,7 @@ fn test_scanner_yield_every_n_objects_uses_env() { } #[test] +#[serial] fn test_scanner_yield_every_n_objects_uses_default() { with_var_unset(rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS, || { crate::runtime_config::refresh_scanner_runtime_config_for_tests(); @@ -964,6 +973,7 @@ fn test_order_folders_for_resume_reports_stale_hint() { } #[tokio::test] +#[serial] async fn test_record_failed_prunes_to_max_entries() { let (mut scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard::new(1000, 2, &mut scanner, temp_dir); @@ -995,6 +1005,7 @@ async fn test_record_failed_prunes_to_max_entries() { } #[tokio::test] +#[serial] async fn test_prune_failed_objects_cache_drops_expired() { let (mut scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard::new(5, 10, &mut scanner, temp_dir); @@ -1018,6 +1029,7 @@ async fn test_prune_failed_objects_cache_drops_expired() { } #[tokio::test] +#[serial] async fn test_prune_failed_objects_max_zero_keeps_fresh() { let (mut scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard::new(60, 0, &mut scanner, temp_dir); @@ -1840,6 +1852,7 @@ async fn test_heal_actions_returns_actual_size_without_inline_heal() { #[tokio::test] #[cfg(unix)] +#[serial] async fn test_scan_folder_skips_unreadable_child_directory() { let (mut scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard::new(60, 0, &mut scanner, temp_dir.clone()); @@ -1871,6 +1884,7 @@ async fn test_scan_folder_skips_unreadable_child_directory() { } #[tokio::test] +#[serial] async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() { let (mut scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone()); @@ -1984,6 +1998,7 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() { } #[tokio::test] +#[serial] async fn test_scan_folder_xl_meta_named_directory_uses_namespace_descent() { let (mut scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone()); @@ -2029,6 +2044,7 @@ async fn test_scan_folder_xl_meta_named_directory_uses_namespace_descent() { } #[tokio::test(flavor = "current_thread")] +#[serial] async fn test_scan_folder_corrupt_xl_meta_stops_erasure_data_dir_descent() { let logs = CapturedLogs::default(); let subscriber = tracing_subscriber::fmt() @@ -2190,6 +2206,7 @@ async fn test_scan_folder_corrupt_xl_meta_stops_erasure_data_dir_descent() { } #[tokio::test] +#[serial] async fn test_scan_folder_missing_xl_meta_stops_erasure_data_dir_descent() { let (mut scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone()); @@ -2267,6 +2284,7 @@ async fn test_scan_folder_missing_xl_meta_stops_erasure_data_dir_descent() { } #[tokio::test] +#[serial] async fn test_scan_folder_uuid_namespace_part_name_directory_is_not_data_dir() { let (mut scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone()); @@ -2328,6 +2346,7 @@ async fn test_scan_folder_uuid_namespace_part_name_directory_is_not_data_dir() { } #[tokio::test] +#[serial] async fn test_scan_folder_non_erasure_metadata_keeps_namespace_descent() { let (mut scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone()); @@ -2369,6 +2388,7 @@ async fn test_scan_folder_non_erasure_metadata_keeps_namespace_descent() { } #[tokio::test] +#[serial] async fn test_scan_folder_compacted_parent_sends_partial_update() { let (mut scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone()); @@ -2410,6 +2430,7 @@ async fn test_scan_folder_compacted_parent_sends_partial_update() { } #[tokio::test] +#[serial] async fn test_scan_data_folder_cancelled_before_scan_clears_current_path() { let (scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard { @@ -2454,6 +2475,7 @@ async fn test_scan_data_folder_cancelled_before_scan_clears_current_path() { } #[tokio::test] +#[serial] async fn test_scan_data_folder_returns_partial_cache_on_budget_cancel() { let (mut scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone()); @@ -2509,6 +2531,7 @@ async fn test_scan_data_folder_returns_partial_cache_on_budget_cancel() { } #[tokio::test] +#[serial] async fn test_scan_data_folder_reports_invalid_checkpoint_ignored_once() { let (scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard { @@ -2553,6 +2576,7 @@ async fn test_scan_data_folder_reports_invalid_checkpoint_ignored_once() { } #[tokio::test] +#[serial] async fn test_scan_data_folder_resume_hint_prioritizes_next_existing_folder() { let (scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard { @@ -2626,6 +2650,7 @@ async fn test_scan_data_folder_resume_hint_prioritizes_next_existing_folder() { } #[tokio::test] +#[serial] async fn scan_data_folder_missing_bucket_returns_partial() { let (scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard { @@ -2677,6 +2702,7 @@ async fn scan_data_folder_missing_bucket_returns_partial() { } #[tokio::test] +#[serial] async fn scan_data_folder_missing_scan_root_returns_partial() { let (scanner, temp_dir) = build_test_scanner().await; tokio::fs::remove_dir_all(&temp_dir) @@ -2722,6 +2748,7 @@ async fn scan_data_folder_missing_scan_root_returns_partial() { } #[tokio::test] +#[serial] async fn test_scan_data_folder_resume_hint_orders_across_new_and_existing_folders() { let (scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard { @@ -2790,6 +2817,7 @@ async fn test_scan_data_folder_resume_hint_orders_across_new_and_existing_folder } #[tokio::test] +#[serial] async fn test_scan_data_folder_partial_object_budget_accumulates_progress() { let (scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard { @@ -2872,6 +2900,7 @@ async fn test_scan_data_folder_partial_object_budget_accumulates_progress() { } #[tokio::test] +#[serial] async fn test_partial_compacted_entry_does_not_carry_children() { let (mut scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard { @@ -2917,6 +2946,7 @@ async fn test_partial_compacted_entry_does_not_carry_children() { } #[tokio::test] +#[serial] async fn test_partial_entry_does_not_carry_missing_old_child() { let (mut scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard { @@ -2949,6 +2979,7 @@ async fn test_partial_entry_does_not_carry_missing_old_child() { } #[tokio::test] +#[serial] async fn test_legacy_windows_cache_rebuilds_and_round_trips_portable_keys() { let (scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard { @@ -3015,6 +3046,7 @@ async fn test_legacy_windows_cache_rebuilds_and_round_trips_portable_keys() { } #[tokio::test] +#[serial] async fn test_scan_data_folder_success_clears_resume_hint() { let (scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard { @@ -3057,6 +3089,7 @@ async fn test_scan_data_folder_success_clears_resume_hint() { } #[tokio::test] +#[serial] async fn test_scan_data_folder_keeps_unresolved_objects_partial() { let (scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard { @@ -3104,6 +3137,7 @@ async fn test_scan_data_folder_keeps_unresolved_objects_partial() { #[tokio::test] #[cfg(unix)] +#[serial] async fn test_scan_folder_ignores_symlinked_child_directory() { let (mut scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard::new(60, 0, &mut scanner, temp_dir.clone()); diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index bfa25301d..ef0e98063 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -27,6 +27,7 @@ use crate::{ init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path, }; use rustfs_filemeta::FileInfo; +use serial_test::serial; use temp_env::with_var; use time::OffsetDateTime; use uuid::Uuid; @@ -102,6 +103,7 @@ async fn setup_two_pool_scanner_store() -> (tempfile::TempDir, Arc) { } #[tokio::test] +#[serial] async fn scanner_cache_locks_block_same_source_workers() { let (_temp_dir, store) = setup_two_pool_scanner_store().await; let set = &store.pools[0].disk_set[0]; @@ -128,6 +130,7 @@ async fn scanner_cache_locks_block_same_source_workers() { } #[tokio::test] +#[serial] async fn scanner_cache_locks_allow_cross_source_workers() { let (_temp_dir, store) = setup_two_pool_scanner_store().await; let first_set = &store.pools[0].disk_set[0]; @@ -190,6 +193,7 @@ async fn scanner_set_cache_admission_tracks_owner_snapshot_and_fails_closed() { } #[tokio::test] +#[serial] async fn scanner_cycle_is_deferred_while_rebalance_is_active() { let (_temp_dir, store) = setup_two_pool_scanner_store().await; let mut pool_stats = vec![EcstoreRebalanceStats::default(); store.pools.len()]; @@ -225,6 +229,7 @@ async fn scanner_cycle_is_deferred_while_rebalance_is_active() { } #[tokio::test] +#[serial] async fn scanner_cycle_is_deferred_while_terminal_decommission_is_blocked() { let (_temp_dir, store) = setup_two_pool_scanner_store().await; for decommission in [ @@ -299,6 +304,7 @@ async fn data_usage_publish_rejects_a_second_terminal_update_without_blocking() } #[tokio::test] +#[serial] async fn multi_pool_scanner_cycle_publishes_combined_usage() { let (_temp_dir, store) = setup_two_pool_scanner_store().await; let bucket = format!("scanner-union-{}", Uuid::new_v4().simple()); @@ -346,6 +352,7 @@ async fn multi_pool_scanner_cycle_publishes_combined_usage() { } #[tokio::test] +#[serial] async fn multi_pool_scanner_cycle_zero_fills_bucket_absent_from_first_pool() { let (_temp_dir, store) = setup_two_pool_scanner_store().await; let bucket = format!("scanner-second-pool-{}", Uuid::new_v4().simple()); @@ -433,6 +440,7 @@ fn object_lock_config_enabled_accepts_enabled_only() { } #[test] +#[serial] fn dirty_usage_snapshot_clear_preserves_newer_generation() { clear_dirty_usage_buckets_for_tests(); record_dirty_usage_bucket("photos"); @@ -447,6 +455,7 @@ fn dirty_usage_snapshot_clear_preserves_newer_generation() { } #[test] +#[serial] fn dirty_usage_generation_acknowledgement_preserves_newer_mutations() { clear_dirty_usage_buckets_for_tests(); record_dirty_usage_bucket("photos"); @@ -472,6 +481,7 @@ fn dirty_usage_generation_acknowledgement_preserves_newer_mutations() { } #[test] +#[serial] fn dirty_usage_generation_acknowledgement_rejects_stale_process_and_future_generation() { clear_dirty_usage_buckets_for_tests(); record_dirty_usage_bucket("photos"); @@ -501,6 +511,7 @@ fn dirty_usage_generation_acknowledgement_rejects_stale_process_and_future_gener } #[test] +#[serial] fn dirty_usage_snapshot_detects_uncovered_generation() { clear_dirty_usage_buckets_for_tests(); record_dirty_usage_bucket("photos"); @@ -525,6 +536,7 @@ fn generation_saturates_instead_of_wrapping() { } #[test] +#[serial] fn dirty_usage_snapshot_clears_a_stably_absent_bucket_after_durable_save() { clear_dirty_usage_buckets_for_tests(); record_dirty_usage_bucket("photos"); @@ -546,6 +558,7 @@ fn dirty_usage_snapshot_clears_a_stably_absent_bucket_after_durable_save() { } #[test] +#[serial] fn dirty_usage_snapshot_preserves_an_absent_bucket_recorded_after_listing_started() { clear_dirty_usage_buckets_for_tests(); let generation_before_bucket_list = dirty_usage_generation(); @@ -560,6 +573,7 @@ fn dirty_usage_snapshot_preserves_an_absent_bucket_recorded_after_listing_starte } #[test] +#[serial] fn deleting_a_clean_bucket_invalidates_an_inflight_usage_snapshot() { clear_dirty_usage_buckets_for_tests(); let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], dirty_usage_generation()); @@ -573,6 +587,7 @@ fn deleting_a_clean_bucket_invalidates_an_inflight_usage_snapshot() { } #[test] +#[serial] fn deleting_a_bucket_during_listing_invalidates_the_resulting_usage_snapshot() { clear_dirty_usage_buckets_for_tests(); let generation_before_bucket_list = dirty_usage_generation(); @@ -586,6 +601,7 @@ fn deleting_a_bucket_during_listing_invalidates_the_resulting_usage_snapshot() { } #[test] +#[serial] fn scanner_maintenance_change_advances_generation_and_marks_usage_dirty() { clear_dirty_usage_buckets_for_tests(); let generation = scanner_maintenance_generation(); @@ -598,6 +614,7 @@ fn scanner_maintenance_change_advances_generation_and_marks_usage_dirty() { } #[test] +#[serial] fn dirty_usage_clear_excludes_failed_buckets() { clear_dirty_usage_buckets_for_tests(); record_dirty_usage_bucket("photos"); @@ -629,6 +646,7 @@ fn dirty_usage_clear_plan_excludes_cache_save_failures() { } #[test] +#[serial] fn dirty_usage_is_acknowledged_only_after_durable_usage_confirmation() { clear_dirty_usage_buckets_for_tests(); record_dirty_usage_bucket("photos"); @@ -646,6 +664,7 @@ fn dirty_usage_is_acknowledged_only_after_durable_usage_confirmation() { } #[test] +#[serial] fn clear_dirty_usage_bucket_removes_deleted_bucket_marker() { clear_dirty_usage_buckets_for_tests(); record_dirty_usage_bucket("photos"); @@ -1016,30 +1035,35 @@ async fn bucket_cache_pending_heal_reaches_cycle_maintenance_state() { } #[test] +#[serial] fn scanner_concurrency_limit_preserves_available_when_unconfigured() { crate::reset_foreground_read_activity_for_test(); assert_eq!(scanner_concurrency_limit(0, 4), 4); } #[test] +#[serial] fn scanner_concurrency_limit_caps_to_configured_value() { crate::reset_foreground_read_activity_for_test(); assert_eq!(scanner_concurrency_limit(2, 4), 2); } #[test] +#[serial] fn scanner_concurrency_limit_never_exceeds_available_work() { crate::reset_foreground_read_activity_for_test(); assert_eq!(scanner_concurrency_limit(8, 4), 4); } #[test] +#[serial] fn scanner_concurrency_limit_handles_no_available_work() { crate::reset_foreground_read_activity_for_test(); assert_eq!(scanner_concurrency_limit(2, 0), 0); } #[test] +#[serial] fn scanner_concurrency_limit_yields_to_foreground_reads() { crate::reset_foreground_read_activity_for_test(); crate::set_foreground_read_activity(8); @@ -1049,6 +1073,7 @@ fn scanner_concurrency_limit_yields_to_foreground_reads() { } #[test] +#[serial] fn scanner_concurrency_limit_yields_to_streaming_reads() { crate::reset_foreground_read_activity_for_test(); let _guard = crate::ForegroundReadGuard::new(); @@ -1072,6 +1097,7 @@ fn increment_atomic_usize_saturates_at_max() { } #[test] +#[serial] fn scanner_max_concurrent_set_scans_uses_env_cap() { with_var(ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, Some("2"), || { crate::runtime_config::refresh_scanner_runtime_config_for_tests(); @@ -1081,6 +1107,7 @@ fn scanner_max_concurrent_set_scans_uses_env_cap() { } #[test] +#[serial] fn scanner_max_concurrent_disk_scans_uses_env_cap() { with_var(ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, Some("1"), || { crate::runtime_config::refresh_scanner_runtime_config_for_tests(); diff --git a/crates/scanner/src/sleeper.rs b/crates/scanner/src/sleeper.rs index 1a554dcdd..8546cc99a 100644 --- a/crates/scanner/src/sleeper.rs +++ b/crates/scanner/src/sleeper.rs @@ -258,6 +258,7 @@ impl SleepTimer { #[cfg(test)] mod tests { use super::*; + use serial_test::serial; use temp_env::{with_var, with_var_unset}; struct ScannerDefaultSpeedGuard; @@ -325,6 +326,7 @@ mod tests { } #[test] + #[serial] fn test_refresh_from_env_applies_speed_and_idle_mode_for_next_cycle() { let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed); SCANNER_IDLE_MODE.store(true, Ordering::Relaxed); @@ -344,6 +346,7 @@ mod tests { } #[test] + #[serial] fn test_refresh_from_env_uses_default_speed_override_when_speed_unset() { let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest); let s = DynamicSleeper::new(ScannerSpeed::Default); @@ -359,6 +362,7 @@ mod tests { } #[tokio::test(start_paused = true)] + #[serial] async fn test_fastest_never_sleeps() { let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed); SCANNER_IDLE_MODE.store(true, Ordering::Relaxed); @@ -372,6 +376,7 @@ mod tests { } #[tokio::test(start_paused = true)] + #[serial] async fn test_idle_mode_off_skips_sleep() { let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed); SCANNER_IDLE_MODE.store(false, Ordering::Relaxed); diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index 22e42732e..12d928513 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -14,6 +14,7 @@ #![recursion_limit = "256"] +use futures::FutureExt; use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT; use rustfs_scanner::scanner_folder::ScannerItem; use rustfs_scanner::scanner_io::ScannerIODisk; @@ -22,8 +23,10 @@ use rustfs_scanner::{ scanner::init_data_scanner, }; use s3s::dto::RestoreRequest; +use serial_test::serial; use std::{ collections::HashMap, + env, path::{Path, PathBuf}, sync::{Arc, Once, OnceLock}, time::Duration, @@ -532,15 +535,31 @@ async fn wait_for_transition(ecstore: &Arc, bucket: &str, object: &str, } } -// Run `test_fn` with `ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT` -// set to `"1"` for its duration. `temp_env` serializes environment mutations -// globally, preventing data races when multiple tests run in parallel. +// SAFETY: this helper is used only by `#[serial]` tests and runs under the single-threaded Tokio +// runtime (`worker_threads = 1`), so no concurrent test can mutate process environment during the +// `env::set_var` / `env::remove_var` window. +#[allow(unsafe_code)] async fn with_forced_immediate_enqueue_timeout(test_fn: F) where F: FnOnce() -> Fut, Fut: std::future::Future, { - temp_env::async_with_vars([(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, Some("1"))], test_fn()).await; + let original = env::var_os(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT); + unsafe { + env::set_var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, "1"); + } + let result = std::panic::AssertUnwindSafe(test_fn()).catch_unwind().await; + match original { + Some(value) => unsafe { + env::set_var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, value); + }, + None => unsafe { + env::remove_var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT); + }, + } + if let Err(err) = result { + std::panic::resume_unwind(err); + } } mod serial_tests { @@ -573,6 +592,7 @@ mod serial_tests { /// body (GET won) or a clean object/version-not-found (expiry won). A /// tier-fetch failure -- the #3491 symptom -- is never tolerated. #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-2)"] async fn test_expire_transitioned_object_never_races_concurrent_get() { let (_disk_paths, ecstore) = setup_isolated_test_env(false).await; @@ -718,6 +738,7 @@ mod serial_tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"] async fn rejected_transition_candidate_is_recovered_from_persisted_delete_journal() { let (_disk_paths, ecstore) = setup_isolated_test_env(false).await; @@ -804,6 +825,7 @@ mod serial_tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"] async fn cancelled_before_cleanup_store_resolution_persists_journal() { let (_disk_paths, ecstore) = setup_isolated_test_env(false).await; @@ -897,6 +919,7 @@ mod serial_tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"] async fn rejected_transition_cleanup_durability_matrix() { #[derive(Clone, Copy)] @@ -1036,6 +1059,7 @@ mod serial_tests { } #[test] + #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"] fn test_transition_and_restore_flows() { std::thread::Builder::new() @@ -1361,6 +1385,7 @@ mod serial_tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"] async fn test_scanner_enqueues_free_version_cleanup_for_stale_transitioned_object() { let (disk_paths, ecstore) = setup_isolated_test_env(false).await; @@ -1421,6 +1446,7 @@ mod serial_tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"] async fn test_scanner_cleanup_still_works_after_immediate_compensation_transition() { let (disk_paths, ecstore) = setup_isolated_test_env(false).await; @@ -1478,6 +1504,7 @@ mod serial_tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"] async fn test_existing_object_backfill_is_idempotent_after_immediate_compensation_transition() { let (_disk_paths, ecstore) = setup_isolated_test_env(false).await; @@ -1520,6 +1547,7 @@ mod serial_tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] #[ignore = "FAILING on main: excluded from the serial ILM lane pending a fix, see rustfs/backlog#1148 (ilm-1 partial)"] async fn test_noncurrent_expiry_still_works_after_immediate_compensation_transition() { let (disk_paths, ecstore) = setup_isolated_test_env(true).await; @@ -1603,6 +1631,7 @@ mod serial_tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] #[ignore = "FAILING on main: excluded from the serial ILM lane pending a fix, see rustfs/backlog#1148 (ilm-1 partial)"] async fn test_noncurrent_transition_still_works_after_immediate_compensation_transition() { let (disk_paths, ecstore) = setup_isolated_test_env(true).await; @@ -1685,6 +1714,7 @@ mod serial_tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"] async fn test_modeled_versioned_delete_creates_delete_marker_after_immediate_compensation_transition() { let (_disk_paths, ecstore) = setup_isolated_test_env(true).await; @@ -1732,6 +1762,7 @@ mod serial_tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"] async fn test_modeled_delete_marker_cleanup_after_immediate_compensation_transition() { let (disk_paths, ecstore) = setup_isolated_test_env(true).await; @@ -1808,6 +1839,7 @@ mod serial_tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"] async fn test_scanner_expires_zero_day_current_version() { let (disk_paths, ecstore) = setup_isolated_test_env(false).await; @@ -1834,6 +1866,7 @@ mod serial_tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"] async fn test_put_object_immediately_enqueues_zero_day_current_expiry() { let (_disk_paths, ecstore) = setup_isolated_test_env(true).await; @@ -1871,6 +1904,7 @@ mod serial_tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"] async fn test_scanner_expires_zero_day_noncurrent_version() { let (disk_paths, ecstore) = setup_isolated_test_env(false).await; @@ -1937,6 +1971,7 @@ mod serial_tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"] async fn test_put_object_immediately_enqueues_zero_day_noncurrent_expiry() { let (_disk_paths, ecstore) = setup_isolated_test_env(true).await; @@ -1997,6 +2032,7 @@ mod serial_tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] async fn test_background_scanner_expires_zero_day_current_version() { let (_disk_paths, ecstore) = setup_isolated_test_env(true).await; @@ -2020,6 +2056,7 @@ mod serial_tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"] async fn test_background_scanner_expires_zero_day_current_version_for_exact_key_prefix() { let (_disk_paths, ecstore) = setup_isolated_test_env(true).await; @@ -2085,6 +2122,7 @@ mod serial_tests { /// tier object is untouched (zero `remove` calls) -> GET streams from the /// tier again -> a second restore succeeds. #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-8)"] async fn test_restore_chain_local_read_expiry_keeps_remote_and_allows_re_restore() { let (_disk_paths, ecstore) = setup_test_env().await; @@ -2216,6 +2254,7 @@ mod serial_tests { /// parts) must reassemble the exact part layout: part count and sizes, /// the multipart ETag, and byte-identical content across part boundaries. #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-8)"] async fn test_multipart_restore_preserves_parts_and_etag() { let (_disk_paths, ecstore) = setup_test_env().await;