From 35456bcede213bed3d92194dc49086d4aba2b8bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Mon, 31 Aug 2026 18:20:25 +0800 Subject: [PATCH] test(scanner): serialize tests sharing process-global scanner state (#6940) Under the cargo test fallback (threads in one process), tests that touch the process-global scanner cycle recovery status or the global usage-save metrics raced each other and failed randomly in full-suite runs. Mark all touchers with #[serial] per docs/testing/README.md: - 22 tests reading or writing scanner_cycle_recovery_status() via load_scanner_cycle_state_for_startup / reset_scanner_cycle_recovery - 24 tests mutating global_metrics() usage-save counters via store_data_usage_in_backend*, which raced the existing serial test_deferred_usage_save_keeps_last_real_save_metric No-op under nextest, which isolates each test in its own process. --- crates/scanner/src/scanner/tests.rs | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index 1f2b217b4..033032f4d 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -1287,6 +1287,7 @@ fn scanner_startup_fails_closed_on_nonempty_corrupt_cycle_state() { } #[tokio::test] +#[serial] async fn corrupt_cycle_state_is_quarantined_once() { let store = Arc::new(MemoryConfigStore::default()); let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); @@ -1337,6 +1338,7 @@ async fn corrupt_cycle_state_is_quarantined_once() { } #[tokio::test] +#[serial] async fn empty_cycle_state_object_is_quarantined_as_corrupt() { let store = Arc::new(MemoryConfigStore::default()); let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); @@ -1357,6 +1359,7 @@ async fn empty_cycle_state_object_is_quarantined_as_corrupt() { } #[tokio::test] +#[serial] async fn future_cycle_state_schema_is_recovery_required() { let store = Arc::new(MemoryConfigStore::default()); let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); @@ -1375,6 +1378,7 @@ async fn future_cycle_state_schema_is_recovery_required() { } #[tokio::test] +#[serial] async fn concurrent_leaders_cannot_quarantine_newer_cycle_state() { let store = Arc::new(MemoryConfigStore::default()); let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); @@ -1401,6 +1405,7 @@ async fn concurrent_leaders_cannot_quarantine_newer_cycle_state() { } #[tokio::test] +#[serial] async fn cleanup_pending_marker_blocks_a_rewritten_primary_after_restart() { let store = Arc::new(MemoryConfigStore::default()); let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); @@ -1476,6 +1481,7 @@ fn full_rescan_reset_accepts_unknown_marker_fields_without_trusting_cursor() { } #[tokio::test] +#[serial] async fn full_rescan_reset_rebuilds_after_malformed_marker_without_trusting_cursor() { let (_temp_dir, store) = setup_scanner_cycle_store().await; save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0xff, 0x00, 0x01]) @@ -1502,6 +1508,7 @@ async fn full_rescan_reset_rebuilds_after_malformed_marker_without_trusting_curs } #[tokio::test] +#[serial] async fn full_rescan_reset_ignores_epoch_from_malformed_future_primary() { let (_temp_dir, store) = setup_scanner_cycle_store().await; let mut future_primary = vec![0; 24]; @@ -1562,6 +1569,7 @@ async fn ecstore_exact_recovery_marker_delete_honors_etag() { } #[tokio::test] +#[serial] async fn full_rescan_reset_rejects_corrupt_primary_under_stale_blocked_marker() { let (_temp_dir, store) = setup_scanner_cycle_store().await; let corrupt_primary = vec![0xff, 0x00, 0x01]; @@ -1612,6 +1620,7 @@ async fn full_rescan_reset_rejects_corrupt_primary_under_stale_blocked_marker() } #[tokio::test] +#[serial] async fn full_rescan_reset_preserves_valid_primary_when_marker_is_malformed() { let (_temp_dir, store) = setup_scanner_cycle_store().await; let primary = CurrentCycle { @@ -1683,6 +1692,7 @@ async fn full_rescan_reset_preserves_valid_primary_when_marker_is_malformed() { } #[tokio::test] +#[serial] async fn full_rescan_reset_resumes_cleanup_pending_preserved_primary() { let (_temp_dir, store) = setup_scanner_cycle_store().await; let completed_at = Utc::now(); @@ -1762,6 +1772,7 @@ async fn full_rescan_reset_resumes_cleanup_pending_preserved_primary() { } #[tokio::test] +#[serial] async fn full_rescan_reset_rebuilds_oversized_regular_primary_with_malformed_marker() { let (_temp_dir, store) = setup_scanner_cycle_store().await; save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0; 1024 * 1024 + 1]) @@ -1788,6 +1799,7 @@ async fn full_rescan_reset_rebuilds_oversized_regular_primary_with_malformed_mar } #[tokio::test] +#[serial] async fn full_rescan_reset_rebuilds_oversized_primary_after_cleanup_marker() { let (_temp_dir, store) = setup_scanner_cycle_store().await; save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0; 1024 * 1024 + 1]) @@ -1838,6 +1850,7 @@ async fn full_rescan_reset_rebuilds_oversized_primary_after_cleanup_marker() { } #[tokio::test] +#[serial] async fn full_rescan_reset_rebuilds_with_oversized_marker() { let (_temp_dir, store) = setup_scanner_cycle_store().await; save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0xff, 0x00, 0x01]) @@ -1863,6 +1876,7 @@ async fn full_rescan_reset_rebuilds_with_oversized_marker() { } #[tokio::test] +#[serial] async fn full_rescan_reset_rebuilds_with_empty_marker() { let (_temp_dir, store) = setup_scanner_cycle_store().await; save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0xff, 0x00, 0x01]) @@ -1888,6 +1902,7 @@ async fn full_rescan_reset_rebuilds_with_empty_marker() { } #[tokio::test] +#[serial] async fn full_rescan_reset_keeps_cleanup_marker_when_preserved_epoch_is_exhausted() { let (_temp_dir, store) = setup_scanner_cycle_store().await; let primary = CurrentCycle { @@ -1927,6 +1942,7 @@ async fn full_rescan_reset_keeps_cleanup_marker_when_preserved_epoch_is_exhauste } #[tokio::test] +#[serial] async fn full_rescan_reset_rejects_preserved_epoch_that_would_be_terminal() { let (_temp_dir, store) = setup_scanner_cycle_store().await; let primary = CurrentCycle { @@ -1963,6 +1979,7 @@ async fn full_rescan_reset_rejects_preserved_epoch_that_would_be_terminal() { } #[tokio::test] +#[serial] async fn full_rescan_reset_rejects_usage_floor_that_would_be_terminal() { let (_temp_dir, store) = setup_scanner_cycle_store().await; save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), vec![0xff, 0x00, 0x01]) @@ -1998,6 +2015,7 @@ async fn full_rescan_reset_rejects_usage_floor_that_would_be_terminal() { } #[tokio::test] +#[serial] async fn full_rescan_reset_rebuilds_empty_primary_with_malformed_marker() { let (_temp_dir, store) = setup_scanner_cycle_store().await; save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), Vec::new()) @@ -2024,6 +2042,7 @@ async fn full_rescan_reset_rebuilds_empty_primary_with_malformed_marker() { } #[tokio::test] +#[serial] async fn full_rescan_reset_rebuilds_when_primary_cycle_state_is_missing() { let (_temp_dir, store) = setup_scanner_cycle_store().await; let marker = ScannerCycleRecoveryMarker { @@ -2064,6 +2083,7 @@ async fn full_rescan_reset_rebuilds_when_primary_cycle_state_is_missing() { } #[tokio::test] +#[serial] async fn corrupt_cycle_state_rename_or_marker_failure_stays_recovery_required() { let store = Arc::new(MemoryConfigStore::default()); let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); @@ -2089,6 +2109,7 @@ async fn corrupt_cycle_state_rename_or_marker_failure_stays_recovery_required() } #[tokio::test] +#[serial] async fn oversized_or_symlinked_cycle_state_is_rejected() { let store = Arc::new(MemoryConfigStore::default()); let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); @@ -3328,6 +3349,7 @@ fn fenced_usage_bootstrap_retains_partial_cycle_progress() { } #[tokio::test] +#[serial] async fn missing_usage_floor_rebuilds_persisted_cycle_before_leadership_claim() { let store = Arc::new(MemoryConfigStore::default()); let state_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); @@ -4283,6 +4305,7 @@ async fn cycle_budget_lease_takeover_rejects_old_generation() { } #[tokio::test] +#[serial] async fn test_store_data_usage_in_backend_preserves_newer_snapshot() { let store = Arc::new(MemoryConfigStore::default()); let (sender, receiver) = mpsc::channel(2); @@ -4309,6 +4332,7 @@ async fn test_store_data_usage_in_backend_preserves_newer_snapshot() { } #[tokio::test] +#[serial] async fn test_usage_save_object_not_found_defers_only_with_a_fresh_route_barrier() { for (route_blocked, expected) in [ (true, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement)), @@ -4368,6 +4392,7 @@ async fn test_usage_save_object_not_found_defers_only_with_a_fresh_route_barrier } #[tokio::test] +#[serial] async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() { for observational in [false, true] { let store = Arc::new(MemoryConfigStore::default()); @@ -4407,6 +4432,7 @@ async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() { } #[tokio::test] +#[serial] async fn test_observational_usage_defers_when_authoritative_baseline_is_missing() { let store = Arc::new(MemoryConfigStore::default()); let (sender, receiver) = mpsc::channel(1); @@ -4436,6 +4462,7 @@ async fn test_observational_usage_defers_when_authoritative_baseline_is_missing( } #[tokio::test] +#[serial] async fn test_observational_usage_uses_fenced_backup_when_v2_primary_has_no_identity() { let store = Arc::new(MemoryConfigStore::default()); let primary = DataUsageInfo { @@ -4537,6 +4564,7 @@ async fn test_usage_route_barrier_precedes_durable_reconciliation() { } #[tokio::test] +#[serial] async fn coordinator_does_not_put_after_remote_generation_flip() { let store = Arc::new(MemoryConfigStore::default()); let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); @@ -4578,6 +4606,7 @@ async fn coordinator_does_not_put_after_remote_generation_flip() { } #[tokio::test] +#[serial] async fn coordinator_classifies_an_expired_publication_lease() { let store = Arc::new(MemoryConfigStore::default()); let (sender, receiver) = mpsc::channel(1); @@ -4696,6 +4725,7 @@ async fn test_deferred_usage_save_keeps_last_real_save_metric() { } #[tokio::test] +#[serial] async fn test_store_data_usage_in_backend_fences_interleaving_newer_writer() { let store = Arc::new(MemoryConfigStore::default()); let (sender, receiver) = mpsc::channel(1); @@ -4725,6 +4755,7 @@ async fn test_store_data_usage_in_backend_fences_interleaving_newer_writer() { } #[tokio::test] +#[serial] async fn test_store_data_usage_in_backend_does_not_resurrect_deleted_bucket_after_conflict() { let store = Arc::new(MemoryConfigStore::default()); let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); @@ -4796,6 +4827,7 @@ async fn test_store_data_usage_in_backend_does_not_resurrect_deleted_bucket_afte } #[tokio::test] +#[serial] async fn test_store_data_usage_in_backend_updates_backup_with_new_bucket() { let store = Arc::new(MemoryConfigStore::default()); let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); @@ -4856,6 +4888,7 @@ async fn test_store_data_usage_in_backend_updates_backup_with_new_bucket() { } #[tokio::test] +#[serial] async fn test_store_data_usage_in_backend_repairs_backup_after_primary_only_commit() { let store = Arc::new(MemoryConfigStore::default()); let main_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); @@ -4885,6 +4918,7 @@ async fn test_store_data_usage_in_backend_repairs_backup_after_primary_only_comm } #[tokio::test] +#[serial] async fn test_store_data_usage_in_backend_copies_concurrent_bucket_removal_to_backup() { let store = Arc::new(MemoryConfigStore::default()); let main_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); @@ -4951,6 +4985,7 @@ async fn test_store_data_usage_in_backend_copies_concurrent_bucket_removal_to_ba } #[tokio::test] +#[serial] async fn test_store_data_usage_in_backend_retries_after_stale_interleaving_writer() { let store = Arc::new(MemoryConfigStore::default()); let (sender, receiver) = mpsc::channel(1); @@ -4991,6 +5026,7 @@ async fn test_store_data_usage_in_backend_retries_after_stale_interleaving_write } #[tokio::test] +#[serial] async fn test_store_data_usage_in_backend_rejects_untimestamped_complete_snapshot() { let store = Arc::new(MemoryConfigStore::default()); let (sender, receiver) = mpsc::channel(2); @@ -5023,6 +5059,7 @@ async fn test_store_data_usage_in_backend_rejects_untimestamped_complete_snapsho } #[tokio::test] +#[serial] async fn test_store_data_usage_in_backend_recognizes_already_durable_snapshot() { let store = Arc::new(MemoryConfigStore::default()); let (sender, receiver) = mpsc::channel(1); @@ -5053,6 +5090,7 @@ async fn test_store_data_usage_in_backend_recognizes_already_durable_snapshot() } #[tokio::test] +#[serial] async fn test_store_data_usage_in_backend_advances_past_changed_same_epoch_cycle() { let store = Arc::new(MemoryConfigStore::default()); let (sender, receiver) = mpsc::channel(1); @@ -5102,6 +5140,7 @@ async fn test_store_data_usage_in_backend_advances_past_changed_same_epoch_cycle } #[tokio::test] +#[serial] async fn test_store_data_usage_in_backend_orders_scanner_cycles_before_wall_clock() { let store = Arc::new(MemoryConfigStore::default()); let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); @@ -5162,6 +5201,7 @@ async fn test_store_data_usage_in_backend_orders_scanner_cycles_before_wall_cloc } #[tokio::test] +#[serial] async fn test_store_data_usage_in_backend_orders_leader_epochs_before_cycles() { let store = Arc::new(MemoryConfigStore::default()); let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); @@ -5224,6 +5264,7 @@ async fn test_store_data_usage_in_backend_orders_leader_epochs_before_cycles() { } #[tokio::test] +#[serial] async fn test_store_data_usage_in_backend_keeps_first_same_cycle_snapshot() { let store = Arc::new(MemoryConfigStore::default()); let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); @@ -5270,6 +5311,7 @@ async fn test_store_data_usage_in_backend_keeps_first_same_cycle_snapshot() { } #[tokio::test] +#[serial] async fn test_store_data_usage_in_backend_rejects_incomplete_snapshot() { let store = Arc::new(MemoryConfigStore::default()); let (sender, receiver) = mpsc::channel(2); @@ -5302,6 +5344,7 @@ async fn test_store_data_usage_in_backend_rejects_incomplete_snapshot() { } #[tokio::test] +#[serial] async fn test_store_data_usage_in_backend_preserves_superseded_status() { let store = Arc::new(MemoryConfigStore::default()); let authoritative_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); @@ -5353,6 +5396,7 @@ async fn test_store_data_usage_in_backend_preserves_superseded_status() { } #[tokio::test] +#[serial] async fn test_store_data_usage_in_backend_removes_observed_after_authoritative_save() { let store = Arc::new(MemoryConfigStore::default()); let authoritative_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); @@ -5542,6 +5586,7 @@ fn test_stale_data_usage_update_reason_preserves_none_handling() { } #[tokio::test] +#[serial] async fn test_store_data_usage_in_backend_keeps_backup_when_primary_save_fails() { let store = Arc::new(MemoryConfigStore::default()); let (sender, receiver) = mpsc::channel(11); @@ -5584,6 +5629,7 @@ async fn test_store_data_usage_in_backend_keeps_backup_when_primary_save_fails() } #[tokio::test] +#[serial] async fn test_store_data_usage_in_backend_reports_missing_snapshot() { let store = Arc::new(MemoryConfigStore::default()); let (sender, receiver) = mpsc::channel(1);